Skip to content

Commit 19a504d

Browse files
EvilBeaverEvilBeaver
authored
Merge pull request EvilBeaver#1670 from johnnyshut/fix/1667
feat(binary): лимит памяти для ДвоичныеДанные из конфигурации (EvilBeaver#1667)
2 parents d018b18 + 3e487c9 commit 19a504d

22 files changed

Lines changed: 493 additions & 88 deletions

‎src/1Script.sln‎

Lines changed: 135 additions & 0 deletions
Large diffs are not rendered by default.

‎src/OneScript.StandardLibrary/Binary/BinaryDataConstants.cs‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
/*----------------------------------------------------------
2+
This Source Code Form is subject to the terms of the
3+
Mozilla Public License, v.2.0. If a copy of the MPL
4+
was not distributed with this file, You can obtain one
5+
at http://mozilla.org/MPL/2.0/.
6+
----------------------------------------------------------*/
7+
8+
using System;
9+
10+
namespace OneScript.StandardLibrary.Binary
11+
{
12+
public static class BinaryDataConstants
13+
{
14+
/// <summary>
15+
/// Максимальный размер массива, доступный в среде выполнения.
16+
/// Де-факто он чуть меньше 2Гб, он же Int32.MaxValue, поэтому используется системная константа <see cref="Array.MaxLength"/>
17+
/// </summary>
18+
public static readonly int SYSTEM_IN_MEMORY_LIMIT = Array.MaxLength;
19+
20+
/// <summary>
21+
/// Размер двоичных данных, хранимый в памяти по умолчанию.
22+
/// </summary>
23+
public const int DEFAULT_IN_MEMORY_LIMIT = 1024 * 1024 * 50;
24+
}
25+
}

‎src/OneScript.StandardLibrary/Binary/BinaryDataContext.cs‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,32 +24,32 @@ public sealed class BinaryDataContext : AutoContext<BinaryDataContext>, IDisposa
2424
private byte[] _buffer;
2525
private BackingTemporaryFile _backingFile;
2626

27-
public BinaryDataContext(string filename)
27+
public BinaryDataContext(string filename, int inMemLimit)
2828
{
2929
using var fs = new FileStream(filename, FileMode.Open, FileAccess.Read);
30-
ReadFromStream(fs);
30+
ReadFromStream(fs, inMemLimit);
3131
}
3232

3333
public BinaryDataContext(byte[] buffer)
3434
{
3535
_buffer = buffer ?? throw new ArgumentNullException(nameof(buffer));
3636
}
3737

38-
public BinaryDataContext(Stream stream)
38+
public BinaryDataContext(Stream stream, int inMemLimit)
3939
{
4040
long pos = 0;
41-
ReadFromStream(stream);
41+
ReadFromStream(stream, inMemLimit);
4242
stream.Position = pos;
4343
}
44-
44+
4545
/// <summary>
4646
/// Признак хранения данных в памяти
4747
/// </summary>
4848
public bool InMemory => _backingFile == null;
4949

50-
private void ReadFromStream(Stream stream)
50+
private void ReadFromStream(Stream stream, int inMemLimit)
5151
{
52-
if (stream.Length < FileBackingConstants.DEFAULT_MEMORY_LIMIT)
52+
if (stream.Length < inMemLimit)
5353
{
5454
LoadToBuffer(stream);
5555
}
@@ -197,9 +197,9 @@ public GenericStream OpenStreamForRead()
197197
}
198198

199199
[ScriptConstructor(Name = "На основании файла")]
200-
public static BinaryDataContext Constructor(string filename)
200+
public static BinaryDataContext Constructor(TypeActivationContext ctx, string filename)
201201
{
202-
return new BinaryDataContext(filename);
202+
return new BinaryDataContext(filename, ctx.Services.Resolve<IBinaryDataMemoryLimit>().MaxBytesInMemory);
203203
}
204204

205205
public override bool Equals(BslValue other)

‎src/OneScript.StandardLibrary/Binary/BinaryDataOptions.cs‎

Lines changed: 101 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,101 @@
1+
/*----------------------------------------------------------
2+
This Source Code Form is subject to the terms of the
3+
Mozilla Public License, v.2.0. If a copy of the MPL
4+
was not distributed with this file, You can obtain one
5+
at http://mozilla.org/MPL/2.0/.
6+
----------------------------------------------------------*/
7+
8+
using System.Globalization;
9+
using ScriptEngine;
10+
using ScriptEngine.Hosting;
11+
12+
namespace OneScript.StandardLibrary.Binary
13+
{
14+
/// <summary>
15+
/// Инкапсулирует логику чтения и хранения настроек двоичных данных
16+
/// </summary>
17+
public class BinaryDataOptions : IBinaryDataMemoryLimit
18+
{
19+
public const string IN_MEMORY_LIMIT_KEY_NAME = "binaryData.inMemoryMaxSize";
20+
public const string IN_MEMORY_MAX_MAGIC = "max";
21+
22+
public BinaryDataOptions(KeyValueConfig config)
23+
{
24+
var configValue = config[IN_MEMORY_LIMIT_KEY_NAME];
25+
MaxBytesInMemory = ResolveFromConfigString(configValue);
26+
}
27+
28+
public int MaxBytesInMemory { get; }
29+
30+
private static int ResolveFromConfigString(string rawValue)
31+
{
32+
if (string.IsNullOrWhiteSpace(rawValue))
33+
return BinaryDataConstants.DEFAULT_IN_MEMORY_LIMIT;
34+
35+
if (rawValue.Trim() == IN_MEMORY_MAX_MAGIC)
36+
return BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT;
37+
38+
if (!TryParseByteSize(rawValue.Trim(), out var bytes))
39+
{
40+
SystemLogger.Write($"Invalid value for {IN_MEMORY_LIMIT_KEY_NAME}: {rawValue}");
41+
return BinaryDataConstants.DEFAULT_IN_MEMORY_LIMIT;
42+
}
43+
44+
if (bytes <= 0 || bytes >= BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT)
45+
{
46+
SystemLogger.Write($"Value for {IN_MEMORY_LIMIT_KEY_NAME} must be between 1 and {BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT - 1}: {bytes}");
47+
return BinaryDataConstants.DEFAULT_IN_MEMORY_LIMIT;
48+
}
49+
50+
return (int)bytes;
51+
}
52+
53+
private static bool TryParseByteSize(string value, out long bytes)
54+
{
55+
bytes = 0;
56+
57+
if (value.Length == 0)
58+
return false;
59+
60+
var suffix = value[value.Length - 1];
61+
long multiplier = 1;
62+
var numberPart = value;
63+
64+
const long KILOBYTES = 1024L;
65+
const long MEGABYTES = KILOBYTES * 1024L;
66+
const long GIGABYTES = MEGABYTES * 1024L;
67+
68+
switch (suffix)
69+
{
70+
case 'k':
71+
case 'K':
72+
multiplier = KILOBYTES;
73+
numberPart = value.Substring(0, value.Length - 1).TrimEnd();
74+
break;
75+
case 'm':
76+
case 'M':
77+
multiplier = MEGABYTES;
78+
numberPart = value.Substring(0, value.Length - 1).TrimEnd();
79+
break;
80+
case 'g':
81+
case 'G':
82+
multiplier = GIGABYTES;
83+
numberPart = value.Substring(0, value.Length - 1).TrimEnd();
84+
break;
85+
}
86+
87+
if (numberPart.Length == 0)
88+
return false;
89+
90+
if (!long.TryParse(numberPart, NumberStyles.AllowLeadingWhite | NumberStyles.AllowTrailingWhite, CultureInfo.InvariantCulture, out var number))
91+
return false;
92+
93+
if (number <= 0)
94+
return false;
95+
96+
bytes = number * multiplier;
97+
98+
return true;
99+
}
100+
}
101+
}

‎src/OneScript.StandardLibrary/Binary/FileBackingConstants.cs‎

Lines changed: 0 additions & 17 deletions
This file was deleted.

‎src/OneScript.StandardLibrary/Binary/FileBackingStream.cs‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -20,13 +20,9 @@ public sealed class FileBackingStream : Stream
2020

2121
private string _backingFileName;
2222

23-
public FileBackingStream() : this(FileBackingConstants.DEFAULT_MEMORY_LIMIT)
24-
{
25-
}
26-
2723
public FileBackingStream(int inMemoryLimit, int capacity = 0)
2824
{
29-
if (inMemoryLimit == FileBackingConstants.SYSTEM_IN_MEMORY_LIMIT)
25+
if (inMemoryLimit == BinaryDataConstants.SYSTEM_IN_MEMORY_LIMIT)
3026
throw new ArgumentException("Use MemoryStream instead");
3127

3228
_inMemoryLimit = inMemoryLimit;
@@ -85,6 +81,7 @@ public override long Position
8581
public bool HasBackingFile => _backingFileName != null;
8682

8783
public string FileName => _backingFileName;
84+
public int InMemoryThreshold => _inMemoryLimit;
8885

8986
public void SwitchToMemory()
9087
{

‎src/OneScript.StandardLibrary/Binary/GlobalBinaryData.cs‎

Lines changed: 12 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ This Source Code Form is subject to the terms of the
1111
using System.Text;
1212
using OneScript.Contexts;
1313
using OneScript.Exceptions;
14+
using OneScript.Types;
1415
using ScriptEngine.Machine;
1516
using ScriptEngine.Machine.Contexts;
1617
namespace OneScript.StandardLibrary.Binary
@@ -21,6 +22,13 @@ namespace OneScript.StandardLibrary.Binary
2122
[GlobalContext(Category = "Процедуры и функции работы с двоичными данными")]
2223
public sealed class GlobalBinaryData : GlobalContextBase<GlobalBinaryData>
2324
{
25+
private readonly int _memoryLimitMaxBytesInMemory;
26+
27+
private GlobalBinaryData(int memoryLimitMaxBytesInMemory)
28+
{
29+
_memoryLimitMaxBytesInMemory = memoryLimitMaxBytesInMemory;
30+
}
31+
2432
private static byte[] HexStringToByteArray(string hex)
2533
{
2634
var newHex = System.Text.RegularExpressions.Regex.Replace(hex, @"[^0-9A-Fa-f]", "");
@@ -142,9 +150,9 @@ private static void CheckAndThrowIfNull<T>(AutoContext<T> obj, int argNumber, st
142150
throw RuntimeException.InvalidArgumentType(argNumber, argName);
143151
}
144152

145-
public static IAttachableContext CreateInstance()
153+
public static IAttachableContext CreateInstance(IBinaryDataMemoryLimit memoryLimit)
146154
{
147-
return new GlobalBinaryData();
155+
return new GlobalBinaryData(memoryLimit.MaxBytesInMemory);
148156
}
149157

150158
/// <summary>
@@ -174,7 +182,7 @@ public BinaryDataContext ConcatBinaryData(ArrayImpl array)
174182
}
175183
stream.Position = 0;
176184

177-
return new BinaryDataContext(stream);
185+
return new BinaryDataContext(stream, _memoryLimitMaxBytesInMemory);
178186
}
179187

180188
/// <summary>
@@ -244,7 +252,7 @@ public BinaryDataContext GetBinaryDataFromString(string str, IValue encoding = n
244252
stream.Write(inputString, 0, inputString.Length);
245253
stream.Position = 0;
246254

247-
return new BinaryDataContext(stream);
255+
return new BinaryDataContext(stream, _memoryLimitMaxBytesInMemory);
248256
}
249257

250258
/// <summary>

‎src/OneScript.StandardLibrary/Binary/IBinaryDataMemoryLimit.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
/*----------------------------------------------------------
2+
This Source Code Form is subject to the terms of the
3+
Mozilla Public License, v.2.0. If a copy of the MPL
4+
was not distributed with this file, You can obtain one
5+
at http://mozilla.org/MPL/2.0/.
6+
----------------------------------------------------------*/
7+
8+
namespace OneScript.StandardLibrary.Binary
9+
{
10+
/// <summary>
11+
/// Лимит объёма данных в памяти для объектов «ДвоичныеДанные» и смежных потоков
12+
/// до выгрузки во временный файл (байты).
13+
/// </summary>
14+
public interface IBinaryDataMemoryLimit
15+
{
16+
int MaxBytesInMemory { get; }
17+
}
18+
}

‎src/OneScript.StandardLibrary/EngineBuilderExtensions.cs‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ This Source Code Form is subject to the terms of the
55
at http://mozilla.org/MPL/2.0/.
66
----------------------------------------------------------*/
77

8+
using OneScript.StandardLibrary.Binary;
89
using OneScript.StandardLibrary.Collections;
910
using ScriptEngine.Hosting;
1011
using ScriptEngine.Machine;
@@ -13,9 +14,15 @@ namespace OneScript.StandardLibrary
1314
{
1415
public static class EngineBuilderExtensions
1516
{
17+
public static IEngineBuilder UseBinaryDataOptions(this IEngineBuilder builder)
18+
{
19+
builder.Services.RegisterSingleton<IBinaryDataMemoryLimit, BinaryDataOptions>();
20+
return builder;
21+
}
22+
1623
public static ExecutionContext AddStandardLibrary(this ExecutionContext env)
1724
{
1825
return env.AddAssembly(typeof(ArrayImpl).Assembly);
1926
}
2027
}
21-
}
28+
}

‎src/OneScript.StandardLibrary/Http/HttpRequestBodyBinary.cs‎

Lines changed: 6 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -17,42 +17,19 @@ namespace OneScript.StandardLibrary.Http
1717
{
1818
class HttpRequestBodyBinary : IHttpRequestBody
1919
{
20-
private readonly FileBackingStream _storage = new FileBackingStream();
20+
private readonly FileBackingStream _storage;
2121

22-
public HttpRequestBodyBinary()
22+
internal HttpRequestBodyBinary(int inMemoryBodyLimit)
2323
{
24+
_storage = new FileBackingStream(inMemoryBodyLimit);
2425
}
2526

26-
public HttpRequestBodyBinary(BinaryDataContext data)
27+
internal HttpRequestBodyBinary(int inMemoryBodyLimit, BinaryDataContext data)
2728
{
29+
_storage = new FileBackingStream(inMemoryBodyLimit);
2830
data.CopyTo(_storage);
2931
}
3032

31-
public HttpRequestBodyBinary(string body, IValue encoding = null,
32-
ByteOrderMarkUsageEnum bomUsage = ByteOrderMarkUsageEnum.Auto)
33-
{
34-
var useBom = bomUsage == ByteOrderMarkUsageEnum.Auto ||
35-
bomUsage == ByteOrderMarkUsageEnum.Use;
36-
37-
Encoding encoder;
38-
if (encoding == null)
39-
{
40-
encoder = new UTF8Encoding(useBom);
41-
}
42-
else if (encoding.SystemType == BasicTypes.String)
43-
{
44-
var utfs = new List<string> {"utf-16", "utf-32"};
45-
encoder = TextEncodingEnum.GetEncoding(encoding, utfs.Contains(encoding.ToString()) && useBom);
46-
}
47-
else
48-
{
49-
encoder = TextEncodingEnum.GetEncoding(encoding);
50-
}
51-
52-
var byteArray = encoder.GetBytes(body);
53-
_storage.Write(byteArray, 0, byteArray.Length);
54-
}
55-
5633
public IValue GetAsString()
5734
{
5835
_storage.Seek(0, SeekOrigin.Begin);
@@ -63,7 +40,7 @@ public IValue GetAsString()
6340
public IValue GetAsBinary()
6441
{
6542
_storage.Seek(0, SeekOrigin.Begin);
66-
return new BinaryDataContext(_storage);
43+
return new BinaryDataContext(_storage, _storage.InMemoryThreshold);
6744
}
6845

6946
public IValue GetAsFilename()

0 commit comments

Comments
 (0)