Skip to content

Commit 2ca1bb8

Browse files
mateofmateof
authored
Merge pull request #95 from mateof/feature/configurable-strm-streaming-mode
feat: configurable STRM streaming mode with progressive disk cache
2 parents 0b8072b + 128776a commit 2ca1bb8

5 files changed

Lines changed: 443 additions & 63 deletions

File tree

‎TelegramDownloader/Controllers/FileController.cs‎

Lines changed: 267 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,13 +31,22 @@ public class FileController : ControllerBase
3131
string root = FileService.RELATIVELOCALDIR;
3232

3333
private static SemaphoreSlim semaphoreSlim = new SemaphoreSlim(1);
34+
35+
// Limit concurrent direct range fetches against Telegram (seeks ahead of the cache)
36+
private static readonly SemaphoreSlim telegramRangeSemaphore = new SemaphoreSlim(4);
37+
38+
// If the requested range starts within this distance of the background download
39+
// position, wait for the download instead of opening a duplicate Telegram fetch
40+
private const long WAIT_PROXIMITY_BYTES = 8 * 1024 * 1024;
41+
3442
IDbService _db { get; set; }
3543
ITelegramService _ts { get; set; }
3644
IFileService _fs { get; set; }
3745
TransactionInfoService _tis { get; set; }
46+
private readonly IProgressiveDownloadService _progressiveDownload;
3847
private ILogger<IFileService> _logger { get; set; }
3948

40-
public FileController(IDbService db, ITelegramService ts, IFileService fs, TransactionInfoService tis, ILogger<IFileService> logger)
49+
public FileController(IDbService db, ITelegramService ts, IFileService fs, TransactionInfoService tis, IProgressiveDownloadService progressiveDownload, ILogger<IFileService> logger)
4150
{
4251
this.basePath = Environment.CurrentDirectory;
4352
if (!System.IO.Directory.Exists(Path.Combine(basePath, root)))
@@ -48,6 +57,7 @@ public FileController(IDbService db, ITelegramService ts, IFileService fs, Trans
4857
_ts = ts;
4958
_db = db;
5059
_tis = tis;
60+
_progressiveDownload = progressiveDownload;
5161
_logger = logger;
5262

5363
this.operation = new PhysicalFileProvider();
@@ -662,6 +672,262 @@ public async Task<IActionResult> GetFileStream(string idChannel, string idFile,
662672

663673
}
664674

675+
/// <summary>
676+
/// Stream file from Telegram while caching it to disk in the background (progressive streaming).
677+
/// Playback starts immediately; once fully cached, ranges are served from disk.
678+
/// Supports split (multi-message) files: parts are cached sequentially into a single file
679+
/// and seeks ahead of the cache are fetched from the part that contains the requested range.
680+
/// </summary>
681+
/// <param name="idChannel">Telegram channel ID</param>
682+
/// <param name="idFile">TFM database file ID</param>
683+
/// <param name="name">File name (used for mime type / Content-Disposition)</param>
684+
[HttpGet]
685+
[Route("GetFileStreamCached/{idChannel}/{idFile}/{name}")]
686+
[ProducesResponseType(typeof(FileStreamResult), 200)]
687+
[ProducesResponseType(206)]
688+
[ProducesResponseType(404)]
689+
[ProducesResponseType(416)]
690+
public async Task<IActionResult> GetFileStreamCached(string idChannel, string idFile, string name)
691+
{
692+
var mimeType = FileService.getMimeType(name.Split(".").Last());
693+
694+
var dbFile = await _fs.getItemById(idChannel, idFile);
695+
if (dbFile == null)
696+
{
697+
return NotFound();
698+
}
699+
700+
// Ordered Telegram messages that make up the file (several for split files)
701+
var messageIds = ProgressiveDownloadService.GetMessageIds(dbFile);
702+
if (messageIds == null || messageIds.Count == 0)
703+
{
704+
return NotFound();
705+
}
706+
707+
var fileName = dbFile.Name;
708+
var cacheFileName = $"{idChannel}-{(dbFile.MessageId != null ? dbFile.MessageId.ToString() : dbFile.Id)}-{fileName}";
709+
var tempPath = Path.Combine(FileService.TEMPDIR, "_temp");
710+
var filePath = Path.Combine(tempPath, cacheFileName);
711+
Directory.CreateDirectory(tempPath);
712+
713+
long totalLength = dbFile.Size;
714+
715+
// Fully cached: serve straight from disk with full range support
716+
if (System.IO.File.Exists(filePath) && new FileInfo(filePath).Length >= totalLength)
717+
{
718+
_logger.LogDebug("GetFileStreamCached - serving fully cached file {FileName}", fileName);
719+
Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(fileName)}\"";
720+
return PhysicalFile(filePath, mimeType, enableRangeProcessing: true);
721+
}
722+
723+
// Parse range header (RFC 7233: bytes=X-, bytes=X-Y, bytes=-N)
724+
var rangeHeader = Request.Headers["Range"].ToString();
725+
long from = 0;
726+
long? to = null;
727+
bool hasRange = false;
728+
729+
if (!string.IsNullOrEmpty(rangeHeader) && rangeHeader.StartsWith("bytes=") && !rangeHeader.Contains(','))
730+
{
731+
var parts = rangeHeader.Substring("bytes=".Length).Split('-');
732+
if (parts.Length == 2)
733+
{
734+
if (string.IsNullOrEmpty(parts[0]))
735+
{
736+
// Suffix range: last N bytes
737+
if (long.TryParse(parts[1], out var suffixLength) && suffixLength > 0)
738+
{
739+
from = Math.Max(0, totalLength - suffixLength);
740+
to = totalLength - 1;
741+
hasRange = true;
742+
}
743+
}
744+
else if (long.TryParse(parts[0], out var parsedFrom) && parsedFrom >= 0)
745+
{
746+
from = parsedFrom;
747+
hasRange = true;
748+
if (!string.IsNullOrEmpty(parts[1]) && long.TryParse(parts[1], out var parsedTo))
749+
to = parsedTo;
750+
}
751+
}
752+
// Malformed ranges fall through as "no range" (initial chunk)
753+
}
754+
755+
if (hasRange && (from >= totalLength || (to.HasValue && to.Value < from)))
756+
{
757+
Response.Headers["Content-Range"] = $"bytes */{totalLength}";
758+
return StatusCode(StatusCodes.Status416RangeNotSatisfiable);
759+
}
760+
761+
// Initial request without Range: return the first chunk as 206 with total size info
762+
if (!hasRange)
763+
{
764+
from = 0;
765+
to = Math.Min(6 * 1024 * 1024, totalLength - 1);
766+
}
767+
768+
// Open-ended or oversized ranges: cap the response so the player keeps asking
769+
long rangeEnd = (!to.HasValue || to.Value >= totalLength)
770+
? Math.Min(from + (4 * 1024 * 1024), totalLength - 1)
771+
: to.Value;
772+
773+
// Kick off (or attach to) the background download that fills the disk cache,
774+
// so the file is downloaded from Telegram only once
775+
ProgressiveDownloadInfo downloadInfo = null;
776+
try
777+
{
778+
downloadInfo = await _progressiveDownload.StartOrGetDownloadAsync(cacheFileName, idChannel, dbFile, filePath);
779+
}
780+
catch (Exception ex)
781+
{
782+
_logger.LogWarning(ex, "Could not start background cache download for {FileName}", fileName);
783+
}
784+
785+
long CachedBytes()
786+
{
787+
long onDisk = System.IO.File.Exists(filePath) ? new FileInfo(filePath).Length : 0;
788+
return Math.Max(onDisk, downloadInfo?.DownloadedBytes ?? 0);
789+
}
790+
791+
try
792+
{
793+
// If the background download is nearby, wait briefly for it to cover the range
794+
// start instead of opening a duplicate Telegram fetch (normal sequential playback)
795+
if (downloadInfo != null && downloadInfo.IsDownloading &&
796+
from - CachedBytes() <= WAIT_PROXIMITY_BYTES)
797+
{
798+
var waitTarget = Math.Min(rangeEnd, from + 524288); // at least 512KB past the start
799+
var deadline = DateTime.UtcNow.AddSeconds(15);
800+
while (downloadInfo.IsDownloading &&
801+
downloadInfo.DownloadedBytes <= waitTarget &&
802+
DateTime.UtcNow < deadline)
803+
{
804+
await Task.Delay(150, HttpContext.RequestAborted);
805+
}
806+
}
807+
808+
var cachedBytes = CachedBytes();
809+
810+
// Serve from the (possibly still growing) cache file
811+
if (from < cachedBytes)
812+
{
813+
var availableEnd = Math.Min(rangeEnd, cachedBytes - 1);
814+
var length = availableEnd - from + 1;
815+
816+
_logger.LogDebug("GetFileStreamCached - serving from cache: bytes {From}-{To} of {Total}", from, availableEnd, totalLength);
817+
818+
using var cacheStream = new FileStream(filePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite);
819+
cacheStream.Seek(from, SeekOrigin.Begin);
820+
var buffer = new byte[length];
821+
var bytesRead = await cacheStream.ReadAsync(buffer, 0, (int)length, HttpContext.RequestAborted);
822+
823+
Response.StatusCode = StatusCodes.Status206PartialContent;
824+
Response.ContentType = mimeType;
825+
Response.ContentLength = bytesRead;
826+
Response.Headers["Content-Range"] = $"bytes {from}-{from + bytesRead - 1}/{totalLength}";
827+
Response.Headers["Accept-Ranges"] = "bytes";
828+
Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(fileName)}\"";
829+
830+
await Response.Body.WriteAsync(buffer, 0, bytesRead, HttpContext.RequestAborted);
831+
return new EmptyResult();
832+
}
833+
834+
// Range far ahead of the background download (seek): fetch it directly from
835+
// Telegram, streaming each 512KB chunk to the response as it arrives
836+
_logger.LogDebug("GetFileStreamCached - streaming from Telegram: bytes {From}-{To} of {Total}", from, rangeEnd, totalLength);
837+
838+
// Locate the part (Telegram message) containing `from`. Like GetFileStream,
839+
// assume uniform part sizes (the splitter uses a fixed split size)
840+
TL.Message message;
841+
long partStart = 0;
842+
long partSize;
843+
if (messageIds.Count == 1)
844+
{
845+
message = await _ts.getMessageFile(idChannel, messageIds[0]);
846+
partSize = ProgressiveDownloadService.GetDocumentSize(message);
847+
}
848+
else
849+
{
850+
var firstMessage = await _ts.getMessageFile(idChannel, messageIds[0]);
851+
long firstPartSize = ProgressiveDownloadService.GetDocumentSize(firstMessage);
852+
if (firstPartSize <= 0)
853+
{
854+
return NotFound();
855+
}
856+
int partIndex = (int)Math.Min(from / firstPartSize, messageIds.Count - 1);
857+
partStart = (long)partIndex * firstPartSize;
858+
message = partIndex == 0 ? firstMessage : await _ts.getMessageFile(idChannel, messageIds[partIndex]);
859+
partSize = ProgressiveDownloadService.GetDocumentSize(message);
860+
}
861+
862+
if (message == null || partSize <= 0)
863+
{
864+
return NotFound();
865+
}
866+
867+
// Serve only up to the end of this part; the player asks for the rest itself
868+
rangeEnd = Math.Min(rangeEnd, partStart + partSize - 1);
869+
if (rangeEnd < from)
870+
{
871+
Response.Headers["Content-Range"] = $"bytes */{totalLength}";
872+
return StatusCode(StatusCodes.Status416RangeNotSatisfiable);
873+
}
874+
875+
// Align down to 512KB for Telegram; skip the prefix when writing the response
876+
var localFrom = from - partStart;
877+
var alignedFrom = (localFrom / 524288) * 524288;
878+
var skipBytes = localFrom - alignedFrom;
879+
var responseLength = rangeEnd - from + 1;
880+
881+
await telegramRangeSemaphore.WaitAsync(HttpContext.RequestAborted);
882+
try
883+
{
884+
Response.StatusCode = StatusCodes.Status206PartialContent;
885+
Response.ContentType = mimeType;
886+
Response.ContentLength = responseLength;
887+
Response.Headers["Content-Range"] = $"bytes {from}-{rangeEnd}/{totalLength}";
888+
Response.Headers["Accept-Ranges"] = "bytes";
889+
Response.Headers["Content-Disposition"] = $"inline; filename=\"{HttpUtility.UrlEncode(fileName)}\"";
890+
891+
long remainingSkip = skipBytes;
892+
long remainingWrite = responseLength;
893+
894+
await foreach (var chunk in _ts.DownloadFileStreamChunks(
895+
message, alignedFrom, skipBytes + responseLength, HttpContext.RequestAborted))
896+
{
897+
int start = 0;
898+
int length = chunk.Length;
899+
900+
if (remainingSkip > 0)
901+
{
902+
var toSkip = (int)Math.Min(remainingSkip, length);
903+
start += toSkip;
904+
length -= toSkip;
905+
remainingSkip -= toSkip;
906+
}
907+
908+
if (length <= 0) continue;
909+
910+
var toWrite = (int)Math.Min(length, remainingWrite);
911+
await Response.Body.WriteAsync(chunk, start, toWrite, HttpContext.RequestAborted);
912+
remainingWrite -= toWrite;
913+
914+
if (remainingWrite <= 0) break;
915+
}
916+
}
917+
finally
918+
{
919+
telegramRangeSemaphore.Release();
920+
}
921+
922+
return new EmptyResult();
923+
}
924+
catch (OperationCanceledException)
925+
{
926+
_logger.LogDebug("Client closed connection during cached stream - File: {FileName}", fileName);
927+
return new EmptyResult();
928+
}
929+
}
930+
665931
/// <summary>
666932
/// Export channel database to JSON file
667933
/// </summary>

‎TelegramDownloader/Data/FileService.cs‎

Lines changed: 17 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -1252,6 +1252,21 @@ public async Task AddUploadFileFromServer(string dbName, string currentPath, Lis
12521252
_tis.CheckPendingUploadInfoTasks();
12531253
}
12541254

1255+
/// <summary>
1256+
/// Builds the URL written inside a .strm file according to the configured streaming mode.
1257+
/// Files smaller than MaxPreloadFileSizeInMb are always fully preloaded.
1258+
/// </summary>
1259+
private static string BuildStrmUrl(string host, string dbName, BsonFileManagerModel file)
1260+
{
1261+
StreamingMode mode = GeneralConfigStatic.config.GetEffectiveStreamingMode();
1262+
if (mode == StreamingMode.Preload || HelperService.bytesToMegaBytes(file.Size) < GeneralConfigStatic.config.MaxPreloadFileSizeInMb)
1263+
{
1264+
return Path.Combine(host, "api/file/GetFileByTfmId", Uri.EscapeDataString(file.Name)).Replace("\\", "/") + $"?idChannel={dbName}&idFile={file.Id}";
1265+
}
1266+
string endpoint = mode == StreamingMode.ProgressiveCache ? "GetFileStreamCached" : "GetFileStream";
1267+
return Path.Combine(host, "api/file", endpoint, dbName, file.Id, "file" + file.Type).Replace("\\", "/");
1268+
}
1269+
12551270
public async Task<String> CreateStrmFiles(string path, string dbName, string host)
12561271
{
12571272
String folderPathName = Path.GetFileName(path.TrimEnd('/'));
@@ -1300,11 +1315,7 @@ public async Task<String> CreateStrmFiles(string path, string dbName, string hos
13001315
{
13011316
if (FileExtensionTypeTest.isVideoExtension(file.Type) || FileExtensionTypeTest.isAudioExtension(file.Type))
13021317
{
1303-
string contenido = Path.Combine(host, "api/file/GetFileStream", dbName, file.Id, "file" + file.Type).Replace("\\", "/");
1304-
if (GeneralConfigStatic.config.PreloadFilesOnStream || HelperService.bytesToMegaBytes(file.Size) < GeneralConfigStatic.config.MaxPreloadFileSizeInMb)
1305-
{
1306-
contenido = Path.Combine(host, "api/file/GetFileByTfmId", Uri.EscapeDataString(file.Name)).Replace("\\", "/") + $"?idChannel={dbName}&idFile={file.Id}";
1307-
}
1318+
string contenido = BuildStrmUrl(host, dbName, file);
13081319
string pattern = $@"\.({file.Type.Replace(".", "")})$";
13091320
File.WriteAllText(Regex.Replace(filePath, pattern, ".strm"), contenido);
13101321
}
@@ -1338,11 +1349,7 @@ public async Task CreateStrmFilesToLocal(string path, string dbName, string host
13381349
{
13391350
if (FileExtensionTypeTest.isVideoExtension(file.Type) || FileExtensionTypeTest.isAudioExtension(file.Type))
13401351
{
1341-
string contenido = Path.Combine(host, "api/file/GetFileStream", dbName, file.Id, "file" + file.Type).Replace("\\", "/");
1342-
if (GeneralConfigStatic.config.PreloadFilesOnStream || HelperService.bytesToMegaBytes(file.Size) < GeneralConfigStatic.config.MaxPreloadFileSizeInMb)
1343-
{
1344-
contenido = Path.Combine(host, "api/file/GetFileByTfmId", Uri.EscapeDataString(file.Name)).Replace("\\", "/") + $"?idChannel={dbName}&idFile={file.Id}";
1345-
}
1352+
string contenido = BuildStrmUrl(host, dbName, file);
13461353
string pattern = $@"\.({file.Type.Replace(".", "")})$";
13471354
// Ensure parent directory exists
13481355
var parentDir = Path.GetDirectoryName(Regex.Replace(filePath, pattern, ".strm"));

0 commit comments

Comments
 (0)