Skip to content

Commit 8ab05d0

Browse files
PathoschildPathoschild
committed
add new CurseForge export API
1 parent 74bb0e8 commit 8ab05d0

18 files changed

Lines changed: 533 additions & 48 deletions

‎docs/release-notes.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,10 @@
1414
* Fixed `content.Load` ignoring language override in recent SMAPI builds.
1515
* Fixed player sprites and building paint masks not always propagated on change.
1616

17+
* For the update check server:
18+
* Rewrote update checks for mods on CurseForge to use a new CurseForge API endpoint.
19+
_This should result in much faster update checks for CurseForge, and less chance of update-check errors when the CurseForge servers are under heavy load._
20+
1721
* For the web UI:
1822
* Updated JSON validator for Content Patcher 2.1.0.
1923
* Fixed the log parser showing the wrong game folder path if the `Mods` folder path was customized.

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/CurseForgeExportApiClient.cs‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
using System.Threading.Tasks;
2+
using Pathoschild.Http.Client;
3+
using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
4+
5+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport
6+
{
7+
/// <inheritdoc cref="ICurseForgeExportApiClient" />
8+
public class CurseForgeExportApiClient : ICurseForgeExportApiClient
9+
{
10+
/*********
11+
** Fields
12+
*********/
13+
/// <summary>The underlying HTTP client.</summary>
14+
private readonly IClient Client;
15+
16+
17+
/*********
18+
** Public methods
19+
*********/
20+
/// <summary>Construct an instance.</summary>
21+
/// <param name="userAgent">The user agent for the CurseForge export API.</param>
22+
/// <param name="baseUrl">The base URL for the CurseForge export API.</param>
23+
public CurseForgeExportApiClient(string userAgent, string baseUrl)
24+
{
25+
this.Client = new FluentClient(baseUrl).SetUserAgent(userAgent);
26+
}
27+
28+
/// <inheritdoc />
29+
public async Task<CurseForgeFullExport> FetchExportAsync()
30+
{
31+
return await this.Client
32+
.GetAsync("")
33+
.As<CurseForgeFullExport>();
34+
}
35+
36+
/// <inheritdoc />
37+
public void Dispose()
38+
{
39+
this.Client.Dispose();
40+
}
41+
}
42+
}

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ICurseForgeExportApiClient.cs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
using System;
2+
using System.Threading.Tasks;
3+
using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
4+
5+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport
6+
{
7+
/// <summary>An HTTP client for fetching the mod export from the CurseForge export API.</summary>
8+
public interface ICurseForgeExportApiClient : IDisposable
9+
{
10+
/// <summary>Fetch the latest export file from the CurseForge export API.</summary>
11+
public Task<CurseForgeFullExport> FetchExportAsync();
12+
}
13+
}

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ResponseModels/CurseForgeAuthorExport.cs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels
2+
{
3+
/// <summary>The metadata for a user who manages a mod from the CurseForge export API.</summary>
4+
public class CurseForgeAuthorExport
5+
{
6+
/// <summary>The author's user ID.</summary>
7+
public uint Id { get; set; }
8+
9+
/// <summary>The author's display name.</summary>
10+
public string? Name { get; set; }
11+
}
12+
}

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ResponseModels/CurseForgeFileExport.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics.CodeAnalysis;
4+
using Newtonsoft.Json;
5+
6+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels
7+
{
8+
/// <summary>The metadata for an uploaded file for a mod from the CurseForge export API.</summary>
9+
public class CurseForgeFileExport
10+
{
11+
/// <summary>The file identifier.</summary>
12+
public long Id { get; set; }
13+
14+
/// <summary>The file's display name.</summary>
15+
public string? DisplayName { get; set; }
16+
17+
/// <summary>The file internal name.</summary>
18+
public string? FileName { get; set; }
19+
20+
/// <summary>The game version for which it was uploaded.</summary>
21+
public string? GameVersion { get; set; }
22+
23+
/// <summary>The file release type.</summary>
24+
public int ReleaseType { get; set; }
25+
26+
/// <summary>The group the file is listed under, or <c>null</c> if the file predates file groups.</summary>
27+
public CurseForgeFileGroupType? FileGroupType { get; set; }
28+
29+
/// <summary>The file version type (e.g. release or beta).</summary>
30+
public int VersionTypeId { get; set; }
31+
32+
/// <summary>When the file was uploaded.</summary>
33+
public DateTimeOffset FileDate { get; set; }
34+
35+
/// <summary>The extra fields returned by the export API, if any.</summary>
36+
[JsonExtensionData]
37+
[SuppressMessage("ReSharper", "UnusedMember.Global", Justification = "Used to track any new data provided by the API.")]
38+
public Dictionary<string, object>? OtherFields;
39+
}
40+
}

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ResponseModels/CurseForgeFileGroupType.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels
2+
{
3+
/// <summary>The group a file is listed under.</summary>
4+
public enum CurseForgeFileGroupType
5+
{
6+
/// <summary>Unknown or invalid group. This is usually a file which predates file group types.</summary>
7+
None = 0,
8+
9+
/// <summary>A primary download for the mod (e.g. the version most players should install).</summary>
10+
Main = 1,
11+
12+
/// <summary>An optional secondary download.</summary>
13+
Optional = 2,
14+
15+
/// <summary>An old version of the mod that was originally in the <see cref="Main"/> category.</summary>
16+
OldMain = 3,
17+
18+
/// <summary>An old version of the mod that was originally in the <see cref="Optional"/> category.</summary>
19+
OldOptional = 4
20+
}
21+
}

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ResponseModels/CurseForgeFullExport.cs‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
using System.Collections.Generic;
2+
3+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels
4+
{
5+
/// <summary>The metadata for all Stardew Valley from the CurseForge export API.</summary>
6+
public class CurseForgeFullExport
7+
{
8+
/// <summary>The mod data indexed by public mod ID.</summary>
9+
public Dictionary<uint, CurseForgeModExport> Mods { get; set; } = new();
10+
}
11+
}

‎src/SMAPI.Toolkit/Framework/Clients/CurseForgeExport/ResponseModels/CurseForgeModExport.cs‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
using System;
2+
using System.Collections.Generic;
3+
using System.Diagnostics.CodeAnalysis;
4+
using Newtonsoft.Json;
5+
6+
namespace StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels
7+
{
8+
/// <summary>The metadata for a mod from the CurseForge export API.</summary>
9+
public class CurseForgeModExport
10+
{
11+
/// <summary>The mod ID.</summary>
12+
public long Id { get; set; }
13+
14+
/// <summary>The mod's display name.</summary>
15+
public string? Name { get; set; }
16+
17+
/// <summary>The URL to the mod's web page on CurseForge.</summary>
18+
public string? ModPageUrl { get; set; }
19+
20+
/// <summary>The authors of the mod.</summary>
21+
public CurseForgeAuthorExport[] Authors { get; set; } = Array.Empty<CurseForgeAuthorExport>();
22+
23+
/// <summary>When the mod was created.</summary>
24+
public DateTimeOffset DateCreated { get; set; }
25+
26+
/// <summary>When the mod became public.</summary>
27+
public DateTimeOffset DateReleased { get; set; }
28+
29+
/// <summary>When the mod was last modified.</summary>
30+
public DateTimeOffset DateModified { get; set; }
31+
32+
/// <summary>The files uploaded for the mod.</summary>
33+
public CurseForgeFileExport[] Files { get; set; } = Array.Empty<CurseForgeFileExport>();
34+
35+
/// <summary>The extra fields returned by the export API, if any.</summary>
36+
[JsonExtensionData]
37+
[SuppressMessage("ReSharper", "UnusedMember.Global", Justification = "Used to track any new data provided by the API.")]
38+
public Dictionary<string, object>? OtherFields;
39+
}
40+
}

‎src/SMAPI.Web/BackgroundService.cs‎

Lines changed: 50 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,16 @@
66
using Microsoft.Extensions.Hosting;
77
using Microsoft.Extensions.Options;
88
using StardewModdingAPI.Toolkit;
9+
using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport;
10+
using StardewModdingAPI.Toolkit.Framework.Clients.CurseForgeExport.ResponseModels;
911
using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport;
1012
using StardewModdingAPI.Toolkit.Framework.Clients.NexusExport.ResponseModels;
1113
using StardewModdingAPI.Toolkit.Framework.Clients.Wiki;
14+
using StardewModdingAPI.Web.Framework.Caching.CurseForgeExport;
1215
using StardewModdingAPI.Web.Framework.Caching.Mods;
1316
using StardewModdingAPI.Web.Framework.Caching.NexusExport;
1417
using StardewModdingAPI.Web.Framework.Caching.Wiki;
18+
using StardewModdingAPI.Web.Framework.Clients.CurseForge;
1519
using StardewModdingAPI.Web.Framework.Clients.Nexus;
1620
using StardewModdingAPI.Web.Framework.ConfigModels;
1721

@@ -33,6 +37,12 @@ internal class BackgroundService : IHostedService, IDisposable
3337
/// <summary>The cache in which to store mod data.</summary>
3438
private static IModCacheRepository? ModCache;
3539

40+
/// <summary>The HTTP client for fetching the mod export from the CurseForge export API.</summary>
41+
private static ICurseForgeExportApiClient? CurseForgeExportApiClient;
42+
43+
/// <summary>The HTTP client for fetching the mod export from the CurseForge export API.</summary>
44+
private static ICurseForgeExportCacheRepository? CurseForgeExportCache;
45+
3646
/// <summary>The cache in which to store mod data from the Nexus export API.</summary>
3747
private static INexusExportCacheRepository? NexusExportCache;
3848

@@ -43,11 +53,20 @@ internal class BackgroundService : IHostedService, IDisposable
4353
private static IOptions<ModUpdateCheckConfig>? UpdateCheckConfig;
4454

4555
/// <summary>Whether the service has been started.</summary>
46-
[MemberNotNullWhen(true, nameof(BackgroundService.JobServer), nameof(BackgroundService.ModCache), nameof(NexusExportApiClient), nameof(NexusExportCache), nameof(BackgroundService.UpdateCheckConfig), nameof(BackgroundService.WikiCache))]
56+
[MemberNotNullWhen(true,
57+
nameof(BackgroundService.JobServer),
58+
nameof(BackgroundService.ModCache),
59+
nameof(BackgroundService.CurseForgeExportApiClient),
60+
nameof(BackgroundService.CurseForgeExportCache),
61+
nameof(BackgroundService.NexusExportApiClient),
62+
nameof(BackgroundService.NexusExportCache),
63+
nameof(BackgroundService.UpdateCheckConfig),
64+
nameof(BackgroundService.WikiCache)
65+
)]
4766
private static bool IsStarted { get; set; }
4867

49-
/// <summary>The number of minutes the Nexus export should be considered valid based on its last-updated date before it's ignored.</summary>
50-
private static int NexusExportStaleAge => (BackgroundService.UpdateCheckConfig?.Value.SuccessCacheMinutes ?? 0) + 10;
68+
/// <summary>The number of minutes a site export should be considered valid based on its last-updated date before it's ignored.</summary>
69+
private static int ExportStaleAge => (BackgroundService.UpdateCheckConfig?.Value.SuccessCacheMinutes ?? 0) + 10;
5170

5271

5372
/*********
@@ -59,20 +78,24 @@ internal class BackgroundService : IHostedService, IDisposable
5978
/// <summary>Construct an instance.</summary>
6079
/// <param name="wikiCache">The cache in which to store wiki metadata.</param>
6180
/// <param name="modCache">The cache in which to store mod data.</param>
81+
/// <param name="curseForgeExportCache">The cache in which to store mod data from the CurseForge export API.</param>
82+
/// <param name="curseForgeExportApiClient">The HTTP client for fetching the mod export from the CurseForge export API.</param>
6283
/// <param name="nexusExportCache">The cache in which to store mod data from the Nexus export API.</param>
6384
/// <param name="nexusExportApiClient">The HTTP client for fetching the mod export from the Nexus Mods export API.</param>
6485
/// <param name="hangfireStorage">The Hangfire storage implementation.</param>
6586
/// <param name="updateCheckConfig">The config settings for mod update checks.</param>
6687
[SuppressMessage("ReSharper", "UnusedParameter.Local", Justification = "The Hangfire reference forces it to initialize first, since it's needed by the background service.")]
67-
public BackgroundService(IWikiCacheRepository wikiCache, IModCacheRepository modCache, INexusExportCacheRepository nexusExportCache, INexusExportApiClient nexusExportApiClient, JobStorage hangfireStorage, IOptions<ModUpdateCheckConfig> updateCheckConfig)
88+
public BackgroundService(IWikiCacheRepository wikiCache, IModCacheRepository modCache, ICurseForgeExportCacheRepository curseForgeExportCache, ICurseForgeExportApiClient curseForgeExportApiClient, INexusExportCacheRepository nexusExportCache, INexusExportApiClient nexusExportApiClient, JobStorage hangfireStorage, IOptions<ModUpdateCheckConfig> updateCheckConfig)
6889
{
6990
BackgroundService.WikiCache = wikiCache;
7091
BackgroundService.ModCache = modCache;
92+
BackgroundService.CurseForgeExportApiClient = curseForgeExportApiClient;
93+
BackgroundService.CurseForgeExportCache = curseForgeExportCache;
7194
BackgroundService.NexusExportCache = nexusExportCache;
7295
BackgroundService.NexusExportApiClient = nexusExportApiClient;
7396
BackgroundService.UpdateCheckConfig = updateCheckConfig;
7497

75-
_ = hangfireStorage; // this parameter is only received so it's initialized before the background service
98+
_ = hangfireStorage; // parameter is only received to initialize it before the background service
7699
}
77100

78101
/// <summary>Start the service.</summary>
@@ -81,16 +104,21 @@ public Task StartAsync(CancellationToken cancellationToken)
81104
{
82105
this.TryInit();
83106

107+
bool enableCurseForgeExport = BackgroundService.CurseForgeExportApiClient is not DisabledCurseForgeExportApiClient;
84108
bool enableNexusExport = BackgroundService.NexusExportApiClient is not DisabledNexusExportApiClient;
85109

86110
// set startup tasks
87111
BackgroundJob.Enqueue(() => BackgroundService.UpdateWikiAsync());
112+
if (enableCurseForgeExport)
113+
BackgroundJob.Enqueue(() => BackgroundService.UpdateCurseForgeExportAsync());
88114
if (enableNexusExport)
89115
BackgroundJob.Enqueue(() => BackgroundService.UpdateNexusExportAsync());
90116
BackgroundJob.Enqueue(() => BackgroundService.RemoveStaleModsAsync());
91117

92118
// set recurring tasks
93119
RecurringJob.AddOrUpdate("update wiki data", () => BackgroundService.UpdateWikiAsync(), "*/10 * * * *"); // every 10 minutes
120+
if (enableCurseForgeExport)
121+
RecurringJob.AddOrUpdate("update CurseForge export", () => BackgroundService.UpdateCurseForgeExportAsync(), "*/10 * * * *");
94122
if (enableNexusExport)
95123
RecurringJob.AddOrUpdate("update Nexus export", () => BackgroundService.UpdateNexusExportAsync(), "*/10 * * * *");
96124
RecurringJob.AddOrUpdate("remove stale mods", () => BackgroundService.RemoveStaleModsAsync(), "2/10 * * * *"); // offset by 2 minutes so it runs after updates (e.g. 00:02, 00:12, etc)
@@ -132,6 +160,21 @@ public static async Task UpdateWikiAsync()
132160
BackgroundService.WikiCache.SaveWikiData(wikiCompatList.StableVersion, wikiCompatList.BetaVersion, wikiCompatList.Mods);
133161
}
134162

163+
/// <summary>Update the cached CurseForge mod dump.</summary>
164+
[AutomaticRetry(Attempts = 3, DelaysInSeconds = new[] { 30, 60, 120 })]
165+
public static async Task UpdateCurseForgeExportAsync()
166+
{
167+
if (!BackgroundService.IsStarted)
168+
throw new InvalidOperationException($"Must call {nameof(BackgroundService.StartAsync)} before scheduling tasks.");
169+
170+
CurseForgeFullExport data = await BackgroundService.CurseForgeExportApiClient.FetchExportAsync();
171+
172+
var cache = BackgroundService.CurseForgeExportCache;
173+
cache.SetData(data);
174+
if (cache.IsStale(BackgroundService.ExportStaleAge))
175+
cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead
176+
}
177+
135178
/// <summary>Update the cached Nexus mod dump.</summary>
136179
[AutomaticRetry(Attempts = 3, DelaysInSeconds = new[] { 30, 60, 120 })]
137180
public static async Task UpdateNexusExportAsync()
@@ -143,7 +186,7 @@ public static async Task UpdateNexusExportAsync()
143186

144187
var cache = BackgroundService.NexusExportCache;
145188
cache.SetData(data);
146-
if (cache.IsStale(BackgroundService.NexusExportStaleAge))
189+
if (cache.IsStale(BackgroundService.ExportStaleAge))
147190
cache.SetData(null); // if the export is too old, fetch fresh mod data from the site/API instead
148191
}
149192

@@ -157,7 +200,7 @@ public static Task RemoveStaleModsAsync()
157200
BackgroundService.ModCache.RemoveStaleMods(TimeSpan.FromHours(48));
158201

159202
// remove stale export cache
160-
if (BackgroundService.NexusExportCache.IsStale(BackgroundService.NexusExportStaleAge))
203+
if (BackgroundService.NexusExportCache.IsStale(BackgroundService.ExportStaleAge))
161204
BackgroundService.NexusExportCache.SetData(null);
162205

163206
return Task.CompletedTask;

0 commit comments

Comments
 (0)