diff --git a/PluginBuilder/APIModels/InstalledPluginRequest.cs b/PluginBuilder/APIModels/InstalledPluginRequest.cs index a3d420e4..a526de86 100644 --- a/PluginBuilder/APIModels/InstalledPluginRequest.cs +++ b/PluginBuilder/APIModels/InstalledPluginRequest.cs @@ -1,3 +1,5 @@ +using Newtonsoft.Json; + namespace PluginBuilder.APIModels; public sealed record InstalledPluginRequest(string Identifier, string Version); diff --git a/PluginBuilder/Controllers/ApiController.cs b/PluginBuilder/Controllers/ApiController.cs index bf742a2c..25b9a901 100644 --- a/PluginBuilder/Controllers/ApiController.cs +++ b/PluginBuilder/Controllers/ApiController.cs @@ -28,7 +28,8 @@ public class ApiController( UserManager userManager, UserVerifiedLogic userVerifiedLogic, IHttpClientFactory httpClientFactory, - ServerEnvironment serverEnvironment) + ServerEnvironment serverEnvironment, + TelemetryService telemetryService) : ControllerBase { private sealed class BuildRow @@ -200,6 +201,38 @@ ORDER BY lv.ver DESC return Ok(versions); } + [AllowAnonymous] + [HttpPost("telemetry/plugins")] + [EnableRateLimiting(Policies.PublicApiRateLimit)] + public async Task ReportInstalledPlugins([FromBody] List plugins) + { + if (plugins is null || plugins.Count == 0) + return Ok(); + + var userAgent = Request.Headers.UserAgent.ToString(); + var remoteIp = HttpContext.Connection.RemoteIpAddress?.ToString(); + var xForwardedFor = Request.Headers["X-Forwarded-For"].ToString(); + var xOriginalFor = Request.Headers["X-Original-For"].ToString(); + + var pluginReports = plugins.Where(p => !string.IsNullOrWhiteSpace(p.Identifier) && !string.IsNullOrWhiteSpace(p.Version)) + .Select(p => new PluginReport(p.Identifier, p.Version)).ToList(); + + if (pluginReports.Count == 0) + return Ok(); + + _ = telemetryService.RecordServerSnapshot(remoteIp, userAgent, pluginReports, xOriginalFor, xForwardedFor); + return Ok(); + } + + [AllowAnonymous] + [HttpGet("plugins/{pluginSlug}/stats")] + [EnableRateLimiting(Policies.PublicApiRateLimit)] + public async Task GetPluginStats(string pluginSlug) + { + var stats = await telemetryService.GetStats(pluginSlug); + return Ok(stats); + } + [AllowAnonymous] [HttpGet("plugins/{pluginSlug}/versions/{version}")] [EnableRateLimiting(Policies.PublicApiRateLimit)] @@ -249,6 +282,10 @@ public async Task Download( await using var conn = await connectionFactory.Open(); await conn.InsertEvent("Download", new JObject { ["pluginSlug"] = pluginSlug.ToString(), ["version"] = version.ToString() }); + + _ = telemetryService.RecordPluginDownload(pluginSlug.ToString(), version.ToString(), Request.Headers.UserAgent.ToString(), + HttpContext.Connection.RemoteIpAddress?.ToString(), Request.Headers["X-Original-For"].ToString(), Request.Headers["X-Forwarded-For"].ToString()); + if (serverEnvironment.EnableLocalArtifactDownloadProxy && Uri.TryCreate(url, UriKind.Absolute, out var artifactUri) && artifactUri.IsLoopback) { return RedirectToAction( diff --git a/PluginBuilder/Data/Scripts/22.IncludePluginStatesTable.sql b/PluginBuilder/Data/Scripts/22.IncludePluginStatesTable.sql new file mode 100644 index 00000000..fb62a2f4 --- /dev/null +++ b/PluginBuilder/Data/Scripts/22.IncludePluginStatesTable.sql @@ -0,0 +1,16 @@ +CREATE TABLE IF NOT EXISTS plugin_server_installs +( + hashed_ip TEXT NOT NULL, + plugin_slug TEXT NOT NULL, + version TEXT NOT NULL, + btcpay_version TEXT NOT NULL, + install_count BIGINT NOT NULL DEFAULT 1, + installed_at TIMESTAMPTZ NOT NULL, + updated_at TIMESTAMPTZ NOT NULL, + uninstalled_at TIMESTAMPTZ NULL, + PRIMARY KEY (hashed_ip, plugin_slug) +); + +CREATE INDEX IF NOT EXISTS ix_plugin_server_installs_plugin_slug ON plugin_server_installs (plugin_slug); + +CREATE INDEX IF NOT EXISTS ix_plugin_server_installs_plugin_slug_uninstalled ON plugin_server_installs (plugin_slug, uninstalled_at); diff --git a/PluginBuilder/DataModels/PluginDownload.cs b/PluginBuilder/DataModels/PluginDownload.cs new file mode 100644 index 00000000..fe7e589d --- /dev/null +++ b/PluginBuilder/DataModels/PluginDownload.cs @@ -0,0 +1,23 @@ +namespace PluginBuilder.DataModels; + +public class PluginDownload +{ + public long Id { get; set; } + public string PluginSlug { get; set; } = null!; + public string Version { get; set; } = null!; + public DateTimeOffset Timestamp { get; set; } + public string HashedIp { get; set; } = null!; + public string BTCPayVersion { get; set; } = null!; +} + +public class PluginServerInstall +{ + public string HashedIp { get; set; } = null!; + public string PluginSlug { get; set; } = null!; + public string Version { get; set; } = null!; + public string BTCPayVersion { get; set; } = null!; + public long InstallCount { get; set; } + public DateTimeOffset InstalledAt { get; set; } + public DateTimeOffset UpdatedAt { get; set; } + public DateTimeOffset? UninstalledAt { get; set; } +} diff --git a/PluginBuilder/Program.cs b/PluginBuilder/Program.cs index 176cf8a6..791c366f 100644 --- a/PluginBuilder/Program.cs +++ b/PluginBuilder/Program.cs @@ -188,6 +188,7 @@ public void AddServices(IConfiguration configuration, IServiceCollection service services.AddHostedService(); services.AddSingleton(); + services.AddScoped(); services.AddScoped(); services.AddScoped(); services.AddSingleton(); diff --git a/PluginBuilder/Services/TelemetryService.cs b/PluginBuilder/Services/TelemetryService.cs new file mode 100644 index 00000000..26d5a12f --- /dev/null +++ b/PluginBuilder/Services/TelemetryService.cs @@ -0,0 +1,296 @@ +using System.Net; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using Dapper; +using PluginBuilder.DataModels; + +namespace PluginBuilder.Services; + +public class TelemetryService(DBConnectionFactory connectionFactory, ILogger logger) +{ + private static readonly Regex BTCPayUserAgentRegex = new(@"^BTCPayServer/(\d+\.\d+\.\d+)", RegexOptions.Compiled); + + public async Task RecordPluginDownload(string pluginSlug, string version, string? userAgent, string? remoteIp, + string? xOriginalFor = null, string? xForwardedFor = null) + { + try + { + if (!TryParseBTCPayUserAgent(userAgent, out var btcpayVersion)) + return; + + if (!TryGetPublicIp(remoteIp, xOriginalFor, xForwardedFor, out var ip)) + return; + + var hashedIp = HashIp(ip!); + var now = DateTimeOffset.UtcNow; + + await using var conn = await connectionFactory.Open(); + var existing = await conn.QueryFirstOrDefaultAsync(""" + SELECT * FROM plugin_server_installs WHERE hashed_ip = @HashedIp AND plugin_slug = @PluginSlug + """, + new { HashedIp = hashedIp, PluginSlug = pluginSlug }); + + if (existing is null) + { + await conn.ExecuteAsync(""" + INSERT INTO plugin_server_installs + (hashed_ip, plugin_slug, version, btcpay_version, installed_at, updated_at, uninstalled_at, install_count) + VALUES + (@HashedIp, @PluginSlug, @Version, @BTCPayVersion, @Now, @Now, NULL, 1) + """, + new { HashedIp = hashedIp, PluginSlug = pluginSlug, Version = version, BTCPayVersion = btcpayVersion, Now = now }); + } + else if (existing.UninstalledAt != null) + { + await conn.ExecuteAsync(""" + UPDATE plugin_server_installs + SET version = @Version, + btcpay_version = @BTCPayVersion, + installed_at = @Now, + updated_at = @Now, + uninstalled_at = NULL, + install_count = install_count + 1 + WHERE hashed_ip = @HashedIp AND plugin_slug = @PluginSlug + """, + new { HashedIp = hashedIp, PluginSlug = pluginSlug, Version = version, BTCPayVersion = btcpayVersion, Now = now }); + } + else + { + await conn.ExecuteAsync(""" + UPDATE plugin_server_installs + SET version = @Version, btcpay_version = @BTCPayVersion, updated_at = @Now + WHERE hashed_ip = @HashedIp AND plugin_slug = @PluginSlug + """, + new { HashedIp = hashedIp, PluginSlug = pluginSlug, Version = version, BTCPayVersion = btcpayVersion, Now = now }); + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to record download telemetry for {PluginSlug} {Version}", pluginSlug, version); + } + } + + + public async Task RecordServerSnapshot(string? remoteIp, string userAgent, IEnumerable plugins, + string? xOriginalFor = null, string? xForwardedFor = null) + { + try + { + var pluginList = plugins.GroupBy(p => p.Slug, StringComparer.OrdinalIgnoreCase).Select(g => g.Last()).ToList(); + if (!TryParseBTCPayUserAgent(userAgent, out var btcpayVersion)) + return; + + if (!TryGetPublicIp(remoteIp, xOriginalFor, xForwardedFor, out var ip)) + return; + + var hashedIp = HashIp(ip!); + var now = DateTimeOffset.UtcNow; + await using var conn = await connectionFactory.Open(); + + var resolvedPluginList = (await conn.QueryAsync(""" + SELECT p.slug AS Slug, i.version AS Version + FROM unnest(@Identifiers::text[], @Versions::text[]) AS i(identifier, version) + JOIN plugins p ON p.identifier = i.identifier + WHERE i.identifier IS NOT NULL AND i.version IS NOT NULL + """, + new + { + Identifiers = pluginList.Select(p => p.Slug).ToArray(), + Versions = pluginList.Select(p => p.Version).ToArray() + })).ToList(); + + if (resolvedPluginList.Count == 0) + return; + + + var existing = (await conn.QueryAsync(""" + SELECT * FROM plugin_server_installs WHERE hashed_ip = @HashedIp + """, new { HashedIp = hashedIp })).ToList(); + + var reportedSlugs = resolvedPluginList.Select(p => p.Slug).ToHashSet(StringComparer.OrdinalIgnoreCase); + var existingBySlug = existing.ToDictionary(x => x.PluginSlug, StringComparer.OrdinalIgnoreCase); + + foreach (var plugin in resolvedPluginList) + { + if (existingBySlug.TryGetValue(plugin.Slug, out var existingInstall)) + { + if (existingInstall.UninstalledAt != null) + { + await conn.ExecuteAsync(""" + UPDATE plugin_server_installs + SET version = @Version, btcpay_version = @BTCPayVersion, installed_at = @Now, updated_at = @Now, uninstalled_at = NULL, install_count = install_count + 1 + WHERE hashed_ip = @HashedIp AND plugin_slug = @PluginSlug + """, + new { HashedIp = hashedIp, PluginSlug = plugin.Slug, Version = plugin.Version, BTCPayVersion = btcpayVersion, Now = now }); + } + else + { + await conn.ExecuteAsync(""" + UPDATE plugin_server_installs + SET version = @Version, btcpay_version = @BTCPayVersion, updated_at = @Now + WHERE hashed_ip = @HashedIp AND plugin_slug = @PluginSlug + """, + new { HashedIp = hashedIp, PluginSlug = plugin.Slug, Version = plugin.Version, BTCPayVersion = btcpayVersion, Now = now }); + } + } + else + { + await conn.ExecuteAsync(""" + INSERT INTO plugin_server_installs + (hashed_ip, plugin_slug, version, btcpay_version, installed_at, updated_at, uninstalled_at, install_count) + VALUES + (@HashedIp, @PluginSlug, @Version, @BTCPayVersion, @Now, @Now, NULL, 1) + """, + new { HashedIp = hashedIp, PluginSlug = plugin.Slug, Version = plugin.Version, BTCPayVersion = btcpayVersion, Now = now }); + } + } + + foreach (var install in existing.Where(x => x.UninstalledAt == null)) + { + if (!reportedSlugs.Contains(install.PluginSlug)) + { + await conn.ExecuteAsync(""" + UPDATE plugin_server_installs SET uninstalled_at = @Now, install_count = GREATEST(0, install_count - 1) + WHERE hashed_ip = @HashedIp AND plugin_slug = @PluginSlug + """, + new { HashedIp = hashedIp, PluginSlug = install.PluginSlug, Now = now }); + } + } + } + catch (Exception ex) + { + logger.LogWarning(ex, "Failed to record server snapshot telemetry"); + } + } + + public async Task GetStats(string pluginSlug) + { + await using var conn = await connectionFactory.Open(); + + var installStats = await conn.QueryFirstOrDefaultAsync<(int TotalInstalls, int ActiveInstalls, int TotalUninstalls)>(""" + SELECT + COALESCE(SUM(install_count), 0) AS TotalInstalls, + COALESCE(COUNT(*) FILTER (WHERE uninstalled_at IS NULL), 0) AS ActiveInstalls, + COALESCE(COUNT(*) FILTER (WHERE uninstalled_at IS NOT NULL), 0) AS TotalUninstalls + FROM plugin_server_installs WHERE plugin_slug = @PluginSlug + """, new { PluginSlug = pluginSlug }); + + var totalInstalls = installStats == default ? 0 : installStats.TotalInstalls; + var activeInstalls = installStats == default ? 0 : installStats.ActiveInstalls; + var totalUninstalls = installStats == default ? 0 : installStats.TotalUninstalls; + + var versionBreakdown = (await conn.QueryAsync(""" + SELECT COALESCE(version, 'unknown') AS Version, COALESCE(COUNT(*), 0) AS Count + FROM plugin_server_installs + WHERE plugin_slug = @PluginSlug AND uninstalled_at IS NULL AND version IS NOT NULL + GROUP BY version + ORDER BY Count DESC + """, new { PluginSlug = pluginSlug })).ToList(); + + var btcpayVersionBreakdown = (await conn.QueryAsync(""" + SELECT COALESCE(btcpay_version, 'unknown') AS Version, COALESCE(COUNT(*), 0) AS Count + FROM plugin_server_installs + WHERE plugin_slug = @PluginSlug AND uninstalled_at IS NULL AND btcpay_version IS NOT NULL + GROUP BY btcpay_version + ORDER BY Count DESC + """, new { PluginSlug = pluginSlug })).ToList(); + + return new PluginStats( + ActiveInstalls: installStats.ActiveInstalls, + TotalUninstalls: installStats.TotalUninstalls, + VersionBreakdown: versionBreakdown, + BTCPayVersionBreakdown: btcpayVersionBreakdown + ); + } + + private static bool TryParseBTCPayUserAgent(string? userAgent, out string btcpayVersion) + { + btcpayVersion = string.Empty; + if (string.IsNullOrWhiteSpace(userAgent)) + return false; + + var match = BTCPayUserAgentRegex.Match(userAgent); + if (!match.Success) + return false; + + btcpayVersion = match.Groups[1].Value; + return true; + } + + private static bool TryGetPublicIp(string? remoteIp, string? xOriginalFor, string? xForwardedFor, out string? result) + { + result = null; + var candidates = new[] { xOriginalFor, xForwardedFor, remoteIp }; + foreach (var candidate in candidates) + { + if (string.IsNullOrWhiteSpace(candidate)) + continue; + + var raw = candidate.Split(',')[0].Trim(); + + if (raw.Contains("]:")) + raw = raw.Substring(1, raw.IndexOf(']') - 1); + else if (raw.Contains(':') && !raw.Contains('.') && raw.Count(c => c == ':') == 1) + raw = raw.Split(':')[0]; + + if (!IPAddress.TryParse(raw, out var ip)) + continue; + + if (IsPrivateOrLoopback(ip)) + continue; + + result = raw; + return true; + } + return false; + } + + + private static bool IsPrivateOrLoopback(IPAddress ip) + { + if (IPAddress.IsLoopback(ip)) + return true; + + if (ip.IsIPv4MappedToIPv6) + ip = ip.MapToIPv4(); + + var bytes = ip.GetAddressBytes(); + if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetwork) + { + if (bytes[0] == 10) + return true; + if (bytes[0] == 172 && bytes[1] >= 16 && bytes[1] <= 31) + return true; + if (bytes[0] == 192 && bytes[1] == 168) + return true; + if (bytes[0] == 169 && bytes[1] == 254) + return true; + } + if (ip.AddressFamily == System.Net.Sockets.AddressFamily.InterNetworkV6) + { + if ((bytes[0] & 0xFE) == 0xFC) + return true; + if (bytes[0] == 0xFE && (bytes[1] & 0xC0) == 0x80) + return true; + } + return false; + } + + private static string HashIp(string ip) + { + var bytes = Encoding.UTF8.GetBytes(ip); + var hash = SHA256.HashData(bytes); + return Convert.ToHexString(hash).ToLowerInvariant(); + } +} + +public record PluginReport(string Slug, string Version); +public record PluginStats( + int ActiveInstalls, + int TotalUninstalls, + List VersionBreakdown, + List BTCPayVersionBreakdown + ); + +public record VersionStat(string Version, long Count); diff --git a/PluginBuilder/ViewModels/Plugin/PluginTelemetryRequest.cs b/PluginBuilder/ViewModels/Plugin/PluginTelemetryRequest.cs new file mode 100644 index 00000000..689045ca --- /dev/null +++ b/PluginBuilder/ViewModels/Plugin/PluginTelemetryRequest.cs @@ -0,0 +1,12 @@ +namespace PluginBuilder.ViewModels.Plugin; + +public class PluginTelemetryRequest +{ + public List Plugins { get; set; } = new(); +} + +public class PluginTelemetryItem +{ + public string? Slug { get; set; } + public string? Version { get; set; } +}