Skip to content

Commit b6f7a59

Browse files
loop8ackloop8ack
committed
Multi-process lock for download cache access
1 parent 1c33ef8 commit b6f7a59

13 files changed

Lines changed: 192 additions & 29 deletions

File tree

‎src/PackScan.Analyzer/Core/Diagnostics.cs‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ private enum Id
1818
OptionNoParsedBool,
1919
OptionNoParsedValue,
2020
OptionNoParsedSize,
21+
OptionNoParsedTimeSpan,
2122

2223
// Licenses Analyzer
2324
LicenseNotAllowed = 200,
@@ -96,6 +97,23 @@ public static Diagnostic Create(string optionName, string value)
9697
}
9798
}
9899

100+
public static class OptionNotParsedTimeSpan
101+
{
102+
public static DiagnosticDescriptor Descriptor { get; }
103+
= new DiagnosticDescriptor(
104+
id: $"{Prefix}{(int)Id.OptionNoParsedTimeSpan:000}",
105+
title: "Not supported TimeSpan value",
106+
messageFormat: "Could not parse '{0}' value '{1}' as TimeSpan.",
107+
category: Category,
108+
DiagnosticSeverity.Error,
109+
isEnabledByDefault: true);
110+
111+
public static Diagnostic Create(string optionName, string value)
112+
{
113+
return Diagnostic.Create(Descriptor, Location.None, optionName, value);
114+
}
115+
}
116+
99117
public static class OptionNotParsedValue
100118
{
101119
public static DiagnosticDescriptor Descriptor { get; }

‎src/PackScan.Analyzer/Core/Options/AnalyzerConfigOptionsExtensions.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,19 @@ public static OptionValue<TEnum> GetOptionEnum<TEnum>(this AnalyzerConfigOptions
5151
return new(name, Diagnostics.OptionNotFound.Create(name));
5252
}
5353

54+
public static OptionValue<TimeSpan> GetOptionTimeSpan(this AnalyzerConfigOptions options, string name)
55+
{
56+
if (options.TryGetValue(Prefix + name, out string? str))
57+
{
58+
if (TimeSpan.TryParse(str, out TimeSpan value))
59+
return new(name, value);
60+
61+
return new(name, Diagnostics.OptionNotParsedTimeSpan.Create(name, str));
62+
}
63+
64+
return new(name, Diagnostics.OptionNotFound.Create(name));
65+
}
66+
5467
public static OptionValue<Size?> GetOptionNullableSKSize(this AnalyzerConfigOptions options, string name)
5568
{
5669
if (options.TryGetValue(Prefix + name, out string? str))

‎src/PackScan.Analyzer/Core/Services/PackagesProviderGeneratorService.cs‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,10 @@ internal record class PackagesProviderGeneratorService
3333
private OptionValue<ContentLoadMode> LicenseContentLoadMode { get; set; }
3434
private OptionValue<ContentLoadMode> ReadMeContentLoadMode { get; set; }
3535
private OptionValue<ContentLoadMode> ReleaseNotesContentLoadMode { get; set; }
36-
36+
private OptionValue<string> DownloadCacheFolder { get; set; }
37+
private OptionValue<TimeSpan> DownloadCacheAccessTimeout { get; set; }
38+
private OptionValue<TimeSpan> DownloadCacheAccessRetryDelay { get; set; }
39+
3740
public PackagesProviderGeneratorService(AnalyzerConfigOptions options)
3841
{
3942
_readerService = new(options);
@@ -50,6 +53,9 @@ public PackagesProviderGeneratorService(AnalyzerConfigOptions options)
5053
LicenseContentLoadMode = options.GetOptionEnum<ContentLoadMode>("PackagesProviderLicenseContentLoadMode");
5154
ReadMeContentLoadMode = options.GetOptionEnum<ContentLoadMode>("PackagesProviderReadMeContentLoadMode");
5255
ReleaseNotesContentLoadMode = options.GetOptionEnum<ContentLoadMode>("PackagesProviderReleaseNotesContentLoadMode");
56+
DownloadCacheFolder = options.GetOptionString("PackagesProviderDownloadCacheFolder");
57+
DownloadCacheAccessTimeout = options.GetOptionTimeSpan("PackagesProviderDownloadCacheAccessTimeout");
58+
DownloadCacheAccessRetryDelay = options.GetOptionTimeSpan("PackagesProviderDownloadCacheAccessRetryDelay");
5359
}
5460

5561
public void Generate(SourceProductionContext context)
@@ -111,6 +117,9 @@ private void CoreGenerate(SourceProductionContext context)
111117
ReadMeContentLoadMode = ReadMeContentLoadMode,
112118
ReleaseNotesContentLoadMode = ReleaseNotesContentLoadMode,
113119
ProductInfoProvider = new AssemblyProductInfoProvider(Assembly.GetExecutingAssembly()),
120+
DownloadCacheFolder = DownloadCacheFolder,
121+
DownloadCacheAccessTimeout = DownloadCacheAccessTimeout,
122+
DownloadCacheAccessRetryDelay = DownloadCacheAccessRetryDelay,
114123
}.WriteCode(packagesData, context.CancellationToken);
115124

116125
foreach (IPackagesProviderFile file in files.Files)

‎src/PackScan.Analyzer/PackScan.Analyzer.targets‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
<CompilerVisibleProperty Include="PackagesProviderReadMeContentLoadMode" />
3636
<CompilerVisibleProperty Include="PackagesProviderReleaseNotesContentLoadMode" />
3737
<CompilerVisibleProperty Include="PackagesProviderDownloadCacheFolder" />
38+
<CompilerVisibleProperty Include="PackagesProviderDownloadCacheAccessTimeout" />
39+
<CompilerVisibleProperty Include="PackagesProviderDownloadCacheAccessRetryDelay" />
3840
<CompilerVisibleProperty Include="PackagesProviderWorkingDirectory" />
3941
<CompilerVisibleProperty Include="PackagesProviderIconContentMaxSize" />
4042

‎src/PackScan.Defaults/PackScan.Defaults.targets‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -104,7 +104,13 @@
104104

105105
<!-- Download cache folder -->
106106
<PackagesProviderDownloadCacheFolder Condition="'$(PackagesProviderDownloadCacheFolder)' == ''"></PackagesProviderDownloadCacheFolder>
107-
107+
108+
<!-- Default timeout to access the download cache -->
109+
<PackagesProviderDownloadCacheAccessTimeout Condition="'$(PackagesProviderDownloadCacheAccessTimeout)' == ''">00:00:01</PackagesProviderDownloadCacheAccessTimeout>
110+
111+
<!-- Default retry delay to access the download cache -->
112+
<PackagesProviderDownloadCacheAccessRetryDelay Condition="'$(PackagesProviderDownloadCacheAccessRetryDelay)' == ''">00:00:00.01</PackagesProviderDownloadCacheAccessRetryDelay>
113+
108114
</PropertyGroup>
109115

110116
<!-- ==================================================== -->

‎src/PackScan.PackagesProvider.Generator/PackageContents/Core/Loader/PackageContentLoader.cs‎

Lines changed: 26 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -21,18 +21,18 @@ internal abstract class PackageContentLoader<TContent, TType> : IPackageContentL
2121
private readonly Dictionary<string, AsyncLazy<IPackageContent<TContent, TType>?>> _runningDownloads = new(StringComparer.OrdinalIgnoreCase);
2222
private readonly IPackagesProviderFilesManager _filesManager;
2323
private readonly IHttpClientFactory _httpClientFactory;
24-
private readonly string _downloadCacheFolder;
2524
private readonly IPackagesProviderFileModification? _modification;
25+
private readonly PackageContentLoaderOptions _options;
2626

2727
protected abstract IReadOnlyDictionary<string, TType> MimeTypeTypeMapping { get; }
2828
protected abstract IReadOnlyDictionary<string, TType> FileExtensionTypeMapping { get; }
2929

30-
protected PackageContentLoader(IPackagesProviderFilesManager filesManager, IHttpClientFactory httpClientFactory, string downloadCacheFolder, IPackagesProviderFileModification? modification)
30+
protected PackageContentLoader(IPackagesProviderFilesManager filesManager, IHttpClientFactory httpClientFactory, IPackagesProviderFileModification? modification, PackageContentLoaderOptions options)
3131
{
3232
_filesManager = filesManager;
3333
_httpClientFactory = httpClientFactory;
34-
_downloadCacheFolder = downloadCacheFolder;
3534
_modification = modification;
35+
_options = options;
3636
}
3737

3838
public IPackageContent<TContent, TType>? TryLoad(ContentLoadMode loadMode, IPackageContentData? contentData, CancellationToken cancellationToken)
@@ -145,26 +145,31 @@ private static bool HasUrl([NotNullWhen(true)] Uri? url)
145145
private async Task<IPackageContent<TContent, TType>?> TryDownloadAsync(Uri url, CancellationToken cancellationToken)
146146
{
147147
string urlMD5Hash = CalcMD5Hash(url);
148-
string tempFilePath = Path.Combine(_downloadCacheFolder, urlMD5Hash);
148+
string tempFilePath = Path.Combine(_options.DownloadCacheFolder, urlMD5Hash);
149149

150-
if (!Directory.Exists(_downloadCacheFolder))
151-
Directory.CreateDirectory(_downloadCacheFolder);
150+
Directory.CreateDirectory(_options.DownloadCacheFolder);
152151

153-
lock (urlMD5Hash)
152+
if (!File.Exists(tempFilePath))
154153
{
155-
if (File.Exists(tempFilePath))
156-
{
157-
cancellationToken.ThrowIfCancellationRequested();
154+
TimeSpan timeout = _options.DownloadCacheAccessTimeout;
155+
TimeSpan retryDelay = _options.DownloadCacheAccessRetryDelay;
158156

159-
TType type = GetContentType(tempFilePath);
157+
using (await LockFile.LockAsync(tempFilePath, timeout, retryDelay, cancellationToken))
158+
{
159+
if (!File.Exists(tempFilePath))
160+
{
161+
cancellationToken.ThrowIfCancellationRequested();
160162

161-
return AddFileAndCreateContent(tempFilePath, type);
163+
return await TryDownloadToTempFileAsync(url, tempFilePath, cancellationToken);
164+
}
162165
}
163166
}
164167

165168
cancellationToken.ThrowIfCancellationRequested();
166169

167-
return await TryDownloadAsync(url, tempFilePath, cancellationToken);
170+
TType type = GetContentType(tempFilePath);
171+
172+
return AddFileAndCreateContent(tempFilePath, type);
168173
}
169174
private static string CalcMD5Hash(Uri uri)
170175
{
@@ -179,7 +184,7 @@ private static string CalcMD5Hash(Uri uri)
179184
.ToLower();
180185
}
181186
}
182-
private async Task<IPackageContent<TContent, TType>?> TryDownloadAsync(Uri url, string tempFilePath, CancellationToken cancellationToken)
187+
private async Task<IPackageContent<TContent, TType>?> TryDownloadToTempFileAsync(Uri url, string tempFilePath, CancellationToken cancellationToken)
183188
{
184189
using HttpRequestMessage message = new(HttpMethod.Get, url);
185190

@@ -196,8 +201,13 @@ private static string CalcMD5Hash(Uri uri)
196201

197202
cancellationToken.ThrowIfCancellationRequested();
198203

199-
using (Stream tempStream = File.Create(tempFilePath))
200-
await response.Content.CopyToAsync(tempStream);
204+
string uniqueTempFilePath = Path.Combine(Path.GetDirectoryName(tempFilePath), Guid.NewGuid().ToString("N"));
205+
206+
using (Stream fileStream = File.Create(uniqueTempFilePath))
207+
await response.Content.CopyToAsync(fileStream);
208+
209+
File.Delete(tempFilePath);
210+
File.Move(uniqueTempFilePath, tempFilePath);
201211

202212
TType type = GetContentType(response, tempFilePath);
203213

‎src/PackScan.PackagesProvider.Generator/PackageContents/Core/Loader/PackageContentLoaderOptions.cs‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
namespace PackScan.PackagesProvider.Generator.PackageContents.Core.Loader;
2+
3+
internal sealed class PackageContentLoaderOptions
4+
{
5+
public required string DownloadCacheFolder { get; init; }
6+
public required TimeSpan DownloadCacheAccessTimeout { get; init; }
7+
public required TimeSpan DownloadCacheAccessRetryDelay { get; init; }
8+
}

‎src/PackScan.PackagesProvider.Generator/PackageContents/Core/Loader/PackageImageContentLoader.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -7,8 +7,8 @@ internal sealed class PackageImageContentLoader : PackageContentLoader<byte[], I
77
protected override IReadOnlyDictionary<string, ImageType> MimeTypeTypeMapping => FileExtensionMappings.ImageTypeByMimeType;
88
protected override IReadOnlyDictionary<string, ImageType> FileExtensionTypeMapping => FileExtensionMappings.ImageTypeByExtension;
99

10-
public PackageImageContentLoader(IPackagesProviderFilesManager filesManager, IHttpClientFactory httpClientFactory, string downloadCacheFolder, IPackagesProviderFileModification? modification)
11-
: base(filesManager, httpClientFactory, downloadCacheFolder, modification)
10+
public PackageImageContentLoader(IPackagesProviderFilesManager filesManager, IHttpClientFactory httpClientFactory, IPackagesProviderFileModification? modification, PackageContentLoaderOptions options)
11+
: base(filesManager, httpClientFactory, modification, options)
1212
{
1313
}
1414

‎src/PackScan.PackagesProvider.Generator/PackageContents/Core/Loader/PackageTextContentLoader.cs‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ internal sealed class PackageTextContentLoader : PackageContentLoader<string, Te
1010
protected override IReadOnlyDictionary<string, TextType> MimeTypeTypeMapping => FileExtensionMappings.TextTypeByMimeType;
1111
protected override IReadOnlyDictionary<string, TextType> FileExtensionTypeMapping => FileExtensionMappings.TextTypeByExtension;
1212

13-
public PackageTextContentLoader(IPackagesProviderFilesManager filesManager, IHttpClientFactory httpClientFactory, string downloadCacheFolder, IPackagesProviderFileModification? modification)
14-
: base(filesManager, httpClientFactory, downloadCacheFolder, modification)
13+
public PackageTextContentLoader(IPackagesProviderFilesManager filesManager, IHttpClientFactory httpClientFactory, IPackagesProviderFileModification? modification, PackageContentLoaderOptions options)
14+
: base(filesManager, httpClientFactory, modification, options)
1515
{
1616
}
1717

‎src/PackScan.PackagesProvider.Generator/PackagesProviderGenerator.cs‎

Lines changed: 15 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,8 @@ public string ClassName
4141
public ContentLoadMode ReadMeContentLoadMode { get; set; }
4242
public ContentLoadMode ReleaseNotesContentLoadMode { get; set; }
4343
public string? DownloadCacheFolder { get; set; }
44+
public TimeSpan? DownloadCacheAccessTimeout { get; set; }
45+
public TimeSpan? DownloadCacheAccessRetryDelay { get; set; }
4446

4547
public IPackagesProviderFiles WriteCode(IEnumerable<IPackageData> packages, CancellationToken cancellationToken = default)
4648
{
@@ -65,9 +67,15 @@ public IPackagesProviderFiles WriteCode(IEnumerable<IPackageData> packages, Canc
6567

6668
private IPackageContentManager CreateAndLoadContentManager(HttpClientFactory httpClientFactory, IPackagesProviderFilesManager filesManager, IPackageData[] packagesData, CancellationToken cancellationToken)
6769
{
68-
string downloadCacheFolderPath = DownloadCacheFolder is null or { Length: 0 }
69-
? Path.Combine(Path.GetTempPath(), "PackScan.PackagesProvider.Writer", "DownloadCache")
70-
: Environment.ExpandEnvironmentVariables(DownloadCacheFolder);
70+
PackageContentLoaderOptions contentLoaderOptions = new()
71+
{
72+
DownloadCacheFolder = DownloadCacheFolder is null or { Length: 0 }
73+
? Path.Combine(Path.GetTempPath(), "PackScan.PackagesProvider.Writer", "DownloadCache")
74+
: Environment.ExpandEnvironmentVariables(DownloadCacheFolder),
75+
76+
DownloadCacheAccessTimeout = DownloadCacheAccessTimeout ?? TimeSpan.FromSeconds(1),
77+
DownloadCacheAccessRetryDelay = DownloadCacheAccessRetryDelay ?? TimeSpan.FromMilliseconds(10)!,
78+
};
7179

7280
IPackagesProviderFileModification? iconContentModification = IconContentMaxSize is not null
7381
? new ReduceImageContentSizeModification(IconContentMaxSize.Value)
@@ -76,16 +84,16 @@ private IPackageContentManager CreateAndLoadContentManager(HttpClientFactory htt
7684
PackageContentManager contentManager = new()
7785
{
7886
IconLoadMode = IconContentLoadMode,
79-
IconContentLoader = new PackageImageContentLoader(filesManager, httpClientFactory, downloadCacheFolderPath, iconContentModification),
87+
IconContentLoader = new PackageImageContentLoader(filesManager, httpClientFactory, iconContentModification, contentLoaderOptions),
8088

8189
LicenseLoadMode = LicenseContentLoadMode,
82-
LicenseContentLoader = new PackageTextContentLoader(filesManager, httpClientFactory, downloadCacheFolderPath, modification: null),
90+
LicenseContentLoader = new PackageTextContentLoader(filesManager, httpClientFactory, modification: null, contentLoaderOptions),
8391

8492
ReadMeLoadMode = ReadMeContentLoadMode,
85-
ReleaseNotesContentLoader = new PackageTextContentLoader(filesManager, httpClientFactory, downloadCacheFolderPath, modification: null),
93+
ReleaseNotesContentLoader = new PackageTextContentLoader(filesManager, httpClientFactory, modification: null, contentLoaderOptions),
8694

8795
ReleaseNotesLoadMode = ReleaseNotesContentLoadMode,
88-
ReadMeContentLoader = new PackageTextContentLoader(filesManager, httpClientFactory, downloadCacheFolderPath, modification: null),
96+
ReadMeContentLoader = new PackageTextContentLoader(filesManager, httpClientFactory, modification: null, contentLoaderOptions),
8997
};
9098

9199
contentManager.LoadAll(packagesData, LoadContentParallel, cancellationToken);

0 commit comments

Comments
 (0)