From f3de4645fcd12b400ee1041778d2b82f14880903 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 02:25:46 -0400 Subject: [PATCH 01/11] Concat video parts with FFmpeg --- .../Tools/FfmpegConcatList.cs | 33 +++ TwitchDownloaderCore/Tools/M3U8.cs | 1 + TwitchDownloaderCore/VideoDownloader.cs | 204 ++++++++++++------ 3 files changed, 172 insertions(+), 66 deletions(-) create mode 100644 TwitchDownloaderCore/Tools/FfmpegConcatList.cs diff --git a/TwitchDownloaderCore/Tools/FfmpegConcatList.cs b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs new file mode 100644 index 00000000..d6f092e0 --- /dev/null +++ b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs @@ -0,0 +1,33 @@ +using System; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace TwitchDownloaderCore.Tools +{ + // https://www.ffmpeg.org/ffmpeg-formats.html#toc-concat-1 + public static class FfmpegConcatList + { + private const string LINE_FEED = "\u000A"; + + public static async Task SerializeAsync(string filePath, M3U8 playlist, Range videoListCrop, CancellationToken cancellationToken = default) + { + await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); + await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; + + await sw.WriteLineAsync("ffconcat version 1.0"); + + foreach (var stream in playlist.Streams.Take(videoListCrop)) + { + await sw.WriteAsync("file '"); + await sw.WriteAsync(stream.Path); + await sw.WriteLineAsync('\''); + + await sw.WriteAsync("duration "); + await sw.WriteLineAsync(stream.PartInfo.Duration.ToString(CultureInfo.InvariantCulture)); + } + } + } +} \ No newline at end of file diff --git a/TwitchDownloaderCore/Tools/M3U8.cs b/TwitchDownloaderCore/Tools/M3U8.cs index 3bc64ad5..f78c185e 100644 --- a/TwitchDownloaderCore/Tools/M3U8.cs +++ b/TwitchDownloaderCore/Tools/M3U8.cs @@ -225,6 +225,7 @@ public enum PlaylistType public uint TwitchLiveSequence { get; private set; } public decimal TwitchElapsedSeconds { get; private set; } public decimal TwitchTotalSeconds { get; private set; } + public decimal TwitchTotalSeconds { get; internal set; } // Other headers that we don't have dedicated properties for. Useful for debugging. private readonly List> _unparsedValues = new(); diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 042dfcd9..8d8ca8e8 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -86,10 +86,12 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF var playlistUrl = qualityPlaylist.Path; var baseUrl = new Uri(playlistUrl[..(playlistUrl.LastIndexOf('/') + 1)], UriKind.Absolute); - var videoLength = TimeSpan.FromSeconds(videoInfoResponse.data.video.lengthSeconds); - CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); + var videoInfo = videoInfoResponse.data.video; + var (playlist, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken); + + var videoListCrop = GetStreamListTrim(playlist.Streams, videoInfo, out var videoLength, out var startOffset, out var endOffset); - var (playlist, videoListCrop, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken); + CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); if (Directory.Exists(downloadFolder)) Directory.Delete(downloadFolder, true); @@ -103,32 +105,27 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); - _progress.SetTemplateStatus("Combining Parts {0}% [4/5]", 0); + _progress.SetStatus("Applying Trim [4/5]"); - await CombineVideoParts(downloadFolder, playlist.Streams, videoListCrop, cancellationToken); + playlist = await ApplyVideoTrim(downloadFolder, playlist, videoListCrop, startOffset, endOffset, cancellationToken); _progress.SetTemplateStatus("Finalizing Video {0}% [5/5]", 0); - var startOffset = TimeSpan.FromSeconds((double)playlist.Streams - .Take(videoListCrop.Start.Value) - .Sum(x => x.PartInfo.Duration)); - - startOffset = downloadOptions.TrimBeginningTime - startOffset; - var seekDuration = downloadOptions.TrimEndingTime - downloadOptions.TrimBeginningTime; - string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); - VideoInfo videoInfo = videoInfoResponse.data.video; await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, downloadOptions.Id.ToString(), videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, videoChapterResponse.data.video.moments.edges, cancellationToken); + var concatListPath = Path.Combine(downloadFolder, "concat.txt"); + await FfmpegConcatList.SerializeAsync(concatListPath, playlist, videoListCrop, cancellationToken); + outputFs.Close(); int ffmpegExitCode; var ffmpegRetries = 0; do { - ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, metadataPath, startOffset, seekDuration > TimeSpan.Zero ? seekDuration : videoLength), cancellationToken); + ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, videoLength), cancellationToken); if (ffmpegExitCode != 0) { _progress.LogError($"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds..."); @@ -154,6 +151,88 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d } } + private async Task ApplyVideoTrim(string downloadFolder, M3U8 playlist, Range videoListCrop, decimal startOffset, decimal endOffset, CancellationToken cancellationToken) + { + if (!downloadOptions.TrimBeginning && !downloadOptions.TrimEnding) + { + return playlist; + } + + var firstPart = playlist.Streams[videoListCrop.Start.Value]; + var lastPart = playlist.Streams[videoListCrop.End.Value - 1]; + + var sharedArgs = $"-hide_banner -y -avoid_negative_ts make_zero -analyzeduration {int.MaxValue} -probesize {int.MaxValue}"; + + if (downloadOptions.TrimBeginning && startOffset != 0) + { + var partPath = Path.Combine(downloadFolder, firstPart.Path); + var untrimmedPartPath = Path.ChangeExtension(partPath, "untrimmed.ts"); + File.Move(partPath, untrimmedPartPath); + + var process = new Process + { + StartInfo = + { + FileName = downloadOptions.FfmpegPath, + Arguments = $"{sharedArgs} -ss {startOffset} -i {untrimmedPartPath} {partPath}", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardInput = false, + RedirectStandardOutput = true, + RedirectStandardError = true + } + }; + + firstPart = firstPart with + { + PartInfo = new M3U8.Stream.ExtPartInfo(firstPart.PartInfo.Duration - startOffset, firstPart.PartInfo.Live) + }; + + process.Start(); + await process.WaitForExitAsync(cancellationToken); + } + + if (downloadOptions.TrimEnding && endOffset != lastPart.PartInfo.Duration) + { + var partPath = Path.Combine(downloadFolder, lastPart.Path); + var untrimmedPartPath = Path.ChangeExtension(partPath, "untrimmed.ts"); + File.Move(partPath, untrimmedPartPath); + + var process = new Process + { + StartInfo = + { + FileName = downloadOptions.FfmpegPath, + Arguments = $"{sharedArgs} -t {endOffset} -i {untrimmedPartPath} {partPath}", + UseShellExecute = false, + CreateNoWindow = true, + RedirectStandardInput = false, + RedirectStandardOutput = true, + RedirectStandardError = true + } + }; + + lastPart = lastPart with + { + PartInfo = new M3U8.Stream.ExtPartInfo(endOffset, lastPart.PartInfo.Live) + }; + + process.Start(); + await process.WaitForExitAsync(cancellationToken); + } + + var newStreams = playlist.Streams.ToArray(); + newStreams[videoListCrop.Start.Value] = firstPart; + newStreams[videoListCrop.End.Value - 1] = lastPart; + + var newMetadata = playlist.FileMetadata with + { + TwitchTotalSeconds = playlist.FileMetadata.TwitchTotalSeconds - startOffset - (playlist.Streams[videoListCrop.End.Value - 1].PartInfo.Duration - endOffset) + }; + + return new M3U8(newMetadata, newStreams); + } + private void CheckAvailableStorageSpace(int bandwidth, TimeSpan videoLength) { var videoSizeInBytes = VideoSizeEstimator.EstimateVideoSize(bandwidth, @@ -348,24 +427,41 @@ private static bool VerifyVideoPart(string filePath) return true; } - private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string metadataPath, TimeSpan startOffset, TimeSpan seekDuration) + private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, TimeSpan videoLength) { var process = new Process { StartInfo = { FileName = downloadOptions.FfmpegPath, - Arguments = string.Format( - "-hide_banner -stats -y -avoid_negative_ts make_zero " + (downloadOptions.TrimBeginning ? "-ss {2} " : "") + "-i \"{0}\" -i \"{1}\" -map_metadata 1 -analyzeduration {3} -probesize {3} " + (downloadOptions.TrimEnding ? "-t {4} " : "") + "-c:v copy \"{5}\"", - Path.Combine(tempFolder, "output.ts"), metadataPath, startOffset.TotalSeconds.ToString(CultureInfo.InvariantCulture), int.MaxValue, seekDuration.TotalSeconds.ToString(CultureInfo.InvariantCulture), outputFile.FullName), UseShellExecute = false, CreateNoWindow = true, RedirectStandardInput = false, RedirectStandardOutput = true, - RedirectStandardError = true + RedirectStandardError = true, + WorkingDirectory = tempFolder } }; + var args = new List + { + "-stats", + "-y", + "-avoid_negative_ts", "make_zero", + "-analyzeduration", $"{int.MaxValue}", + "-probesize", $"{int.MaxValue}", + "-i", concatListPath, + "-i", metadataPath, + "-map_metadata", "1", + "-c", "copy", + outputFile.FullName + }; + + foreach (var arg in args) + { + process.StartInfo.ArgumentList.Add(arg); + } + var encodingTimeRegex = new Regex(@"(?<=time=)(\d\d):(\d\d):(\d\d)\.(\d\d)", RegexOptions.Compiled); var logQueue = new ConcurrentQueue(); @@ -376,7 +472,7 @@ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string me logQueue.Enqueue(e.Data); // We cannot use -report ffmpeg arg because it redirects stderr - HandleFfmpegOutput(e.Data, encodingTimeRegex, seekDuration); + HandleFfmpegOutput(e.Data, encodingTimeRegex, videoLength); }; process.Start(); @@ -413,7 +509,7 @@ private void HandleFfmpegOutput(string output, Regex encodingTimeRegex, TimeSpan _progress.ReportProgress(Math.Clamp(percent, 0, 100)); } - private async Task<(M3U8 playlist, Range cropRange, DateTimeOffset airDate)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken) + private async Task<(M3U8 playlist, DateTimeOffset airDate)> GetVideoPlaylist(string playlistUrl, CancellationToken cancellationToken) { var playlistString = await _httpClient.GetStringAsync(playlistUrl, cancellationToken); var playlist = M3U8.Parse(playlistString); @@ -425,22 +521,26 @@ private void HandleFfmpegOutput(string output, Regex encodingTimeRegex, TimeSpan airDate = vodAirDate; } - var videoListCrop = GetStreamListCrop(playlist.Streams, downloadOptions); - - return (playlist, videoListCrop, airDate); + return (playlist, airDate); } - private static Range GetStreamListCrop(IList streamList, VideoDownloadOptions downloadOptions) + private Range GetStreamListTrim(IList streamList, VideoInfo videoInfo, out TimeSpan videoLength, out decimal startOffset, out decimal endOffset) { + startOffset = 0; + endOffset = 0; + var startIndex = 0; if (downloadOptions.TrimBeginning) { var startTime = 0m; - var cropTotalSeconds = (decimal)downloadOptions.TrimBeginningTime.TotalSeconds; + var trimTotalSeconds = (decimal)downloadOptions.TrimBeginningTime.TotalSeconds; foreach (var videoPart in streamList) { - if (startTime + videoPart.PartInfo.Duration > cropTotalSeconds) + if (startTime + videoPart.PartInfo.Duration > trimTotalSeconds) + { + startOffset = trimTotalSeconds - startTime; break; + } startIndex++; startTime += videoPart.PartInfo.Duration; @@ -451,17 +551,25 @@ private static Range GetStreamListCrop(IList streamList, VideoDownl if (downloadOptions.TrimEnding) { var endTime = streamList.Sum(x => x.PartInfo.Duration); - var cropTotalSeconds = (decimal)downloadOptions.TrimEndingTime.TotalSeconds; + var trimTotalSeconds = (decimal)downloadOptions.TrimEndingTime.TotalSeconds; for (var i = streamList.Count - 1; i >= 0; i--) { - if (endTime - streamList[i].PartInfo.Duration < cropTotalSeconds) + var videoPart = streamList[i]; + if (endTime - videoPart.PartInfo.Duration < trimTotalSeconds) + { + endOffset = videoPart.PartInfo.Duration - (endTime - trimTotalSeconds); break; + } endIndex--; - endTime -= streamList[i].PartInfo.Duration; + endTime -= videoPart.PartInfo.Duration; } } + videoLength = + (downloadOptions.TrimEnding ? downloadOptions.TrimEndingTime : TimeSpan.FromSeconds(videoInfo.lengthSeconds)) + - (downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero); + return new Range(startIndex, endIndex); } @@ -485,42 +593,6 @@ private static Range GetStreamListCrop(IList streamList, VideoDownl return m3u8.GetStreamOfQuality(downloadOptions.Quality); } - private async Task CombineVideoParts(string downloadFolder, IEnumerable playlist, Range videoListCrop, CancellationToken cancellationToken) - { - DriveInfo outputDrive = DriveHelper.GetOutputDrive(downloadFolder); - string outputFile = Path.Combine(downloadFolder, "output.ts"); - - int partCount = videoListCrop.End.Value - videoListCrop.Start.Value; - int doneCount = 0; - - await using var outputStream = new FileStream(outputFile, FileMode.Create, FileAccess.Write, FileShare.Read); - foreach (var part in playlist.Take(videoListCrop)) - { - await DriveHelper.WaitForDrive(outputDrive, _progress, cancellationToken); - - string partFile = Path.Combine(downloadFolder, DownloadTools.RemoveQueryString(part.Path)); - if (File.Exists(partFile)) - { - await using (var fs = File.Open(partFile, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - await fs.CopyToAsync(outputStream, cancellationToken).ConfigureAwait(false); - } - - try - { - File.Delete(partFile); - } - catch { /* If we can't delete, oh well. It should get cleanup up later anyways */ } - } - - doneCount++; - int percent = (int)(doneCount / (double)partCount * 100); - _progress.ReportProgress(percent); - - cancellationToken.ThrowIfCancellationRequested(); - } - } - private static void Cleanup(string downloadFolder) { try From 503667a25bae58be996ccdb115241657fff5b272 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 02:29:33 -0400 Subject: [PATCH 02/11] Fix too long chapters causing some video players to hallucinate a longer video duration --- TwitchDownloaderCore/Tools/FfmpegMetadata.cs | 23 ++++++++++++++++---- TwitchDownloaderCore/VideoDownloader.cs | 2 +- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs index 023f288e..d5065c3f 100644 --- a/TwitchDownloaderCore/Tools/FfmpegMetadata.cs +++ b/TwitchDownloaderCore/Tools/FfmpegMetadata.cs @@ -14,7 +14,7 @@ public static class FfmpegMetadata private const string LINE_FEED = "\u000A"; public static async Task SerializeAsync(string filePath, string streamerName, string videoId, string videoTitle, DateTime videoCreation, int viewCount, string videoDescription = null, - TimeSpan startOffset = default, IEnumerable videoMomentEdges = null, CancellationToken cancellationToken = default) + TimeSpan startOffset = default, TimeSpan videoLength = default, IEnumerable videoMomentEdges = null, CancellationToken cancellationToken = default) { await using var fs = new FileStream(filePath, FileMode.Create, FileAccess.Write, FileShare.Read); await using var sw = new StreamWriter(fs) { NewLine = LINE_FEED }; @@ -22,7 +22,7 @@ public static async Task SerializeAsync(string filePath, string streamerName, st await SerializeGlobalMetadata(sw, streamerName, videoId, videoTitle, videoCreation, viewCount, videoDescription); await fs.FlushAsync(cancellationToken); - await SerializeChapters(sw, videoMomentEdges, startOffset); + await SerializeChapters(sw, videoMomentEdges, startOffset, videoLength); await fs.FlushAsync(cancellationToken); } @@ -44,14 +44,13 @@ private static async Task SerializeGlobalMetadata(StreamWriter sw, string stream await sw.WriteLineAsync(@$"Views: {viewCount}"); } - private static async Task SerializeChapters(StreamWriter sw, IEnumerable videoMomentEdges, TimeSpan startOffset) + private static async Task SerializeChapters(StreamWriter sw, IEnumerable videoMomentEdges, TimeSpan startOffset, TimeSpan videoLength) { if (videoMomentEdges is null) { return; } - // Note: FFmpeg automatically handles out of range chapters for us var startOffsetMillis = (int)startOffset.TotalMilliseconds; foreach (var momentEdge in videoMomentEdges) { @@ -64,6 +63,22 @@ private static async Task SerializeChapters(StreamWriter sw, IEnumerable TimeSpan.Zero) + { + var chapterStart = TimeSpan.FromMilliseconds(startMillis); + if (chapterStart >= videoLength) + { + continue; + } + + var chapterEnd = chapterStart + TimeSpan.FromMilliseconds(lengthMillis); + if (chapterEnd > videoLength) + { + lengthMillis = (int)(videoLength - chapterStart).TotalMilliseconds; + } + } + await sw.WriteLineAsync("[CHAPTER]"); await sw.WriteLineAsync("TIMEBASE=1/1000"); await sw.WriteLineAsync($"START={startMillis}"); diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 8d8ca8e8..e8731912 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -114,7 +114,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, downloadOptions.Id.ToString(), videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, videoInfo.description?.Replace(" \n", "\n").Replace("\n\n", "\n").TrimEnd(), downloadOptions.TrimBeginning ? downloadOptions.TrimBeginningTime : TimeSpan.Zero, - videoChapterResponse.data.video.moments.edges, cancellationToken); + videoLength, videoChapterResponse.data.video.moments.edges, cancellationToken); var concatListPath = Path.Combine(downloadFolder, "concat.txt"); await FfmpegConcatList.SerializeAsync(concatListPath, playlist, videoListCrop, cancellationToken); From f416212b5c5a68507ad1597f4ead40579fa4dc85 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 02:32:49 -0400 Subject: [PATCH 03/11] Fix race conditions --- TwitchDownloaderCore/ChatDownloader.cs | 8 +++++++- TwitchDownloaderCore/ChatRenderer.cs | 14 ++++++++++++-- TwitchDownloaderCore/ChatUpdater.cs | 8 +++++++- TwitchDownloaderCore/ClipDownloader.cs | 10 +++++++++- TwitchDownloaderCore/TsMerger.cs | 8 +++++++- TwitchDownloaderCore/VideoDownloader.cs | 10 +++++++++- 6 files changed, 51 insertions(+), 7 deletions(-) diff --git a/TwitchDownloaderCore/ChatDownloader.cs b/TwitchDownloaderCore/ChatDownloader.cs index f0d560db..40fb718f 100644 --- a/TwitchDownloaderCore/ChatDownloader.cs +++ b/TwitchDownloaderCore/ChatDownloader.cs @@ -263,10 +263,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; diff --git a/TwitchDownloaderCore/ChatRenderer.cs b/TwitchDownloaderCore/ChatRenderer.cs index d9373a07..293ceee2 100644 --- a/TwitchDownloaderCore/ChatRenderer.cs +++ b/TwitchDownloaderCore/ChatRenderer.cs @@ -85,10 +85,16 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } if (maskFileInfo is not null) @@ -96,7 +102,11 @@ public async Task RenderVideoAsync(CancellationToken cancellationToken) maskFileInfo.Refresh(); if (maskFileInfo.Exists && maskFileInfo.Length == 0) { - maskFileInfo.Delete(); + try + { + maskFileInfo.Delete(); + } + catch { } } } diff --git a/TwitchDownloaderCore/ChatUpdater.cs b/TwitchDownloaderCore/ChatUpdater.cs index 0eb70329..5370ea48 100644 --- a/TwitchDownloaderCore/ChatUpdater.cs +++ b/TwitchDownloaderCore/ChatUpdater.cs @@ -44,10 +44,16 @@ public async Task UpdateAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; diff --git a/TwitchDownloaderCore/ClipDownloader.cs b/TwitchDownloaderCore/ClipDownloader.cs index bf6a91f6..5d14ad49 100644 --- a/TwitchDownloaderCore/ClipDownloader.cs +++ b/TwitchDownloaderCore/ClipDownloader.cs @@ -43,10 +43,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; @@ -101,6 +107,8 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF } finally { + await Task.Delay(100, CancellationToken.None); + File.Delete(tempFile); } diff --git a/TwitchDownloaderCore/TsMerger.cs b/TwitchDownloaderCore/TsMerger.cs index b119eb81..b53295ff 100644 --- a/TwitchDownloaderCore/TsMerger.cs +++ b/TwitchDownloaderCore/TsMerger.cs @@ -38,10 +38,16 @@ public async Task MergeAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index e8731912..e7a587f3 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -51,10 +51,16 @@ public async Task DownloadAsync(CancellationToken cancellationToken) } catch { + await Task.Delay(100, CancellationToken.None); + outputFileInfo.Refresh(); if (outputFileInfo.Exists && outputFileInfo.Length == 0) { - outputFileInfo.Delete(); + try + { + outputFileInfo.Delete(); + } + catch { } } throw; @@ -144,6 +150,8 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d } finally { + await Task.Delay(100, CancellationToken.None); + if (_shouldClearCache) { Cleanup(downloadFolder); From 991df676d1cccdb7042a482077feecae8d9f79c1 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 02:34:10 -0400 Subject: [PATCH 04/11] Implement unused CancellationToken --- TwitchDownloaderCore/Tools/FfmpegConcatList.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TwitchDownloaderCore/Tools/FfmpegConcatList.cs b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs index d6f092e0..374c6dea 100644 --- a/TwitchDownloaderCore/Tools/FfmpegConcatList.cs +++ b/TwitchDownloaderCore/Tools/FfmpegConcatList.cs @@ -21,6 +21,8 @@ public static async Task SerializeAsync(string filePath, M3U8 playlist, Range vi foreach (var stream in playlist.Streams.Take(videoListCrop)) { + cancellationToken.ThrowIfCancellationRequested(); + await sw.WriteAsync("file '"); await sw.WriteAsync(stream.Path); await sw.WriteLineAsync('\''); From 5d99dba1656b29a2bd98eafb253c7d3355d1bcec Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 02:37:24 -0400 Subject: [PATCH 05/11] Log failure to clean up --- TwitchDownloaderCore/VideoDownloader.cs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index e7a587f3..139f57e8 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -601,7 +601,7 @@ private Range GetStreamListTrim(IList streamList, VideoInfo videoIn return m3u8.GetStreamOfQuality(downloadOptions.Quality); } - private static void Cleanup(string downloadFolder) + private void Cleanup(string downloadFolder) { try { @@ -610,7 +610,10 @@ private static void Cleanup(string downloadFolder) Directory.Delete(downloadFolder, true); } } - catch (IOException) { } // Directory is probably being used by another process + catch (IOException e) + { + _progress.LogWarning($"Failed to delete download cache: {e.Message}"); + } } } } \ No newline at end of file From 8fe17dba0a76ca911fc0b27a0a7d03a1aef01e14 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 03:05:45 -0400 Subject: [PATCH 06/11] Forgot to specify concat arg. May or may not be necessary --- TwitchDownloaderCore/VideoDownloader.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 139f57e8..4148f182 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -458,6 +458,7 @@ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string co "-avoid_negative_ts", "make_zero", "-analyzeduration", $"{int.MaxValue}", "-probesize", $"{int.MaxValue}", + "-f", "concat", "-i", concatListPath, "-i", metadataPath, "-map_metadata", "1", From 946e9b749a6c5b589146c38f544d3d74f7350373 Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:37:32 -0400 Subject: [PATCH 07/11] Fix video trimming --- TwitchDownloaderCore/VideoDownloader.cs | 113 +++++------------------- 1 file changed, 20 insertions(+), 93 deletions(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 4148f182..7150b3b4 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -75,7 +75,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF downloadOptions.TempFolder, $"{downloadOptions.Id}_{DateTimeOffset.UtcNow.Ticks}"); - _progress.SetStatus("Fetching Video Info [1/5]"); + _progress.SetStatus("Fetching Video Info [1/4]"); try { @@ -95,7 +95,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF var videoInfo = videoInfoResponse.data.video; var (playlist, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken); - var videoListCrop = GetStreamListTrim(playlist.Streams, videoInfo, out var videoLength, out var startOffset, out var endOffset); + var videoListCrop = GetStreamListTrim(playlist.Streams, videoInfo, out var videoLength, out var startOffset, out _); CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); @@ -103,19 +103,15 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF Directory.Delete(downloadFolder, true); TwitchHelper.CreateDirectory(downloadFolder); - _progress.SetTemplateStatus("Downloading {0}% [2/5]", 0); + _progress.SetTemplateStatus("Downloading {0}% [2/4]", 0); await DownloadVideoPartsAsync(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); - _progress.SetTemplateStatus("Verifying Parts {0}% [3/5]", 0); + _progress.SetTemplateStatus("Verifying Parts {0}% [3/4]", 0); await VerifyDownloadedParts(playlist.Streams, videoListCrop, baseUrl, downloadFolder, airDate, cancellationToken); - _progress.SetStatus("Applying Trim [4/5]"); - - playlist = await ApplyVideoTrim(downloadFolder, playlist, videoListCrop, startOffset, endOffset, cancellationToken); - - _progress.SetTemplateStatus("Finalizing Video {0}% [5/5]", 0); + _progress.SetTemplateStatus("Finalizing Video {0}% [4/4]", 0); string metadataPath = Path.Combine(downloadFolder, "metadata.txt"); await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, downloadOptions.Id.ToString(), videoInfo.title, videoInfo.createdAt, videoInfo.viewCount, @@ -131,7 +127,7 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d var ffmpegRetries = 0; do { - ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, videoLength), cancellationToken); + ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, startOffset, videoLength), cancellationToken); if (ffmpegExitCode != 0) { _progress.LogError($"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds..."); @@ -159,88 +155,6 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d } } - private async Task ApplyVideoTrim(string downloadFolder, M3U8 playlist, Range videoListCrop, decimal startOffset, decimal endOffset, CancellationToken cancellationToken) - { - if (!downloadOptions.TrimBeginning && !downloadOptions.TrimEnding) - { - return playlist; - } - - var firstPart = playlist.Streams[videoListCrop.Start.Value]; - var lastPart = playlist.Streams[videoListCrop.End.Value - 1]; - - var sharedArgs = $"-hide_banner -y -avoid_negative_ts make_zero -analyzeduration {int.MaxValue} -probesize {int.MaxValue}"; - - if (downloadOptions.TrimBeginning && startOffset != 0) - { - var partPath = Path.Combine(downloadFolder, firstPart.Path); - var untrimmedPartPath = Path.ChangeExtension(partPath, "untrimmed.ts"); - File.Move(partPath, untrimmedPartPath); - - var process = new Process - { - StartInfo = - { - FileName = downloadOptions.FfmpegPath, - Arguments = $"{sharedArgs} -ss {startOffset} -i {untrimmedPartPath} {partPath}", - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardInput = false, - RedirectStandardOutput = true, - RedirectStandardError = true - } - }; - - firstPart = firstPart with - { - PartInfo = new M3U8.Stream.ExtPartInfo(firstPart.PartInfo.Duration - startOffset, firstPart.PartInfo.Live) - }; - - process.Start(); - await process.WaitForExitAsync(cancellationToken); - } - - if (downloadOptions.TrimEnding && endOffset != lastPart.PartInfo.Duration) - { - var partPath = Path.Combine(downloadFolder, lastPart.Path); - var untrimmedPartPath = Path.ChangeExtension(partPath, "untrimmed.ts"); - File.Move(partPath, untrimmedPartPath); - - var process = new Process - { - StartInfo = - { - FileName = downloadOptions.FfmpegPath, - Arguments = $"{sharedArgs} -t {endOffset} -i {untrimmedPartPath} {partPath}", - UseShellExecute = false, - CreateNoWindow = true, - RedirectStandardInput = false, - RedirectStandardOutput = true, - RedirectStandardError = true - } - }; - - lastPart = lastPart with - { - PartInfo = new M3U8.Stream.ExtPartInfo(endOffset, lastPart.PartInfo.Live) - }; - - process.Start(); - await process.WaitForExitAsync(cancellationToken); - } - - var newStreams = playlist.Streams.ToArray(); - newStreams[videoListCrop.Start.Value] = firstPart; - newStreams[videoListCrop.End.Value - 1] = lastPart; - - var newMetadata = playlist.FileMetadata with - { - TwitchTotalSeconds = playlist.FileMetadata.TwitchTotalSeconds - startOffset - (playlist.Streams[videoListCrop.End.Value - 1].PartInfo.Duration - endOffset) - }; - - return new M3U8(newMetadata, newStreams); - } - private void CheckAvailableStorageSpace(int bandwidth, TimeSpan videoLength) { var videoSizeInBytes = VideoSizeEstimator.EstimateVideoSize(bandwidth, @@ -435,7 +349,7 @@ private static bool VerifyVideoPart(string filePath) return true; } - private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, TimeSpan videoLength) + private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, TimeSpan videoLength) { var process = new Process { @@ -466,6 +380,19 @@ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string co outputFile.FullName }; + // TODO: Make this optional - "Safe" and "Exact" trimming methods + if (downloadOptions.TrimEnding) + { + args.Insert(0, "-t"); + args.Insert(1, videoLength.TotalSeconds.ToString(CultureInfo.InvariantCulture)); + } + + if (downloadOptions.TrimBeginning) + { + args.Insert(0, "-ss"); + args.Insert(1, startOffset.ToString(CultureInfo.InvariantCulture)); + } + foreach (var arg in args) { process.StartInfo.ArgumentList.Add(arg); From 36edf0ca5e0135f866fefd83863180f5009f25ae Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 17:42:48 -0400 Subject: [PATCH 08/11] Make sure to dispose of FFmpeg --- TwitchDownloaderCore/VideoDownloader.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 7150b3b4..89a335ff 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -351,7 +351,7 @@ private static bool VerifyVideoPart(string filePath) private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, TimeSpan videoLength) { - var process = new Process + using var process = new Process { StartInfo = { From 2842ae64d3bcd2d5547f36bf1af5d5c70f09c2fc Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 18:22:28 -0400 Subject: [PATCH 09/11] Don't append trim args unless absolutely necessary --- TwitchDownloaderCore/VideoDownloader.cs | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 89a335ff..785a14c7 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -95,7 +95,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF var videoInfo = videoInfoResponse.data.video; var (playlist, airDate) = await GetVideoPlaylist(playlistUrl, cancellationToken); - var videoListCrop = GetStreamListTrim(playlist.Streams, videoInfo, out var videoLength, out var startOffset, out _); + var videoListCrop = GetStreamListTrim(playlist.Streams, videoInfo, out var videoLength, out var startOffset, out var endOffset); CheckAvailableStorageSpace(qualityPlaylist.StreamInfo.Bandwidth, videoLength); @@ -127,7 +127,7 @@ await FfmpegMetadata.SerializeAsync(metadataPath, videoInfo.owner.displayName, d var ffmpegRetries = 0; do { - ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, startOffset, videoLength), cancellationToken); + ffmpegExitCode = await Task.Run(() => RunFfmpegVideoCopy(downloadFolder, outputFileInfo, concatListPath, metadataPath, startOffset, endOffset, videoLength), cancellationToken); if (ffmpegExitCode != 0) { _progress.LogError($"Failed to finalize video (code {ffmpegExitCode}), retrying in 10 seconds..."); @@ -349,7 +349,7 @@ private static bool VerifyVideoPart(string filePath) return true; } - private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, TimeSpan videoLength) + private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string concatListPath, string metadataPath, decimal startOffset, decimal endOffset, TimeSpan videoLength) { using var process = new Process { @@ -381,13 +381,13 @@ private int RunFfmpegVideoCopy(string tempFolder, FileInfo outputFile, string co }; // TODO: Make this optional - "Safe" and "Exact" trimming methods - if (downloadOptions.TrimEnding) + if (endOffset > 0) { args.Insert(0, "-t"); args.Insert(1, videoLength.TotalSeconds.ToString(CultureInfo.InvariantCulture)); } - if (downloadOptions.TrimBeginning) + if (startOffset > 0) { args.Insert(0, "-ss"); args.Insert(1, startOffset.ToString(CultureInfo.InvariantCulture)); @@ -493,7 +493,10 @@ private Range GetStreamListTrim(IList streamList, VideoInfo videoIn var videoPart = streamList[i]; if (endTime - videoPart.PartInfo.Duration < trimTotalSeconds) { - endOffset = videoPart.PartInfo.Duration - (endTime - trimTotalSeconds); + if (endTime - trimTotalSeconds > 0) + { + endOffset = videoPart.PartInfo.Duration - (endTime - trimTotalSeconds); + } break; } From 7594e181473bdb8008154edb3ef9b47f6c74f14b Mon Sep 17 00:00:00 2001 From: ScrubN <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 19:55:33 -0400 Subject: [PATCH 10/11] Cleanup --- TwitchDownloaderCore/VideoDownloader.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/TwitchDownloaderCore/VideoDownloader.cs b/TwitchDownloaderCore/VideoDownloader.cs index 785a14c7..acc733db 100644 --- a/TwitchDownloaderCore/VideoDownloader.cs +++ b/TwitchDownloaderCore/VideoDownloader.cs @@ -493,10 +493,9 @@ private Range GetStreamListTrim(IList streamList, VideoInfo videoIn var videoPart = streamList[i]; if (endTime - videoPart.PartInfo.Duration < trimTotalSeconds) { - if (endTime - trimTotalSeconds > 0) - { - endOffset = videoPart.PartInfo.Duration - (endTime - trimTotalSeconds); - } + var offset = endTime - trimTotalSeconds; + if (offset > 0) endOffset = videoPart.PartInfo.Duration - offset; + break; } From 64ce33fa77375fe06eda8dcd3d1c822d03c9107b Mon Sep 17 00:00:00 2001 From: Scrub <72096833+ScrubN@users.noreply.github.com> Date: Wed, 19 Jun 2024 20:10:31 -0400 Subject: [PATCH 11/11] Fix --- TwitchDownloaderCore/Tools/M3U8.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/TwitchDownloaderCore/Tools/M3U8.cs b/TwitchDownloaderCore/Tools/M3U8.cs index f78c185e..15a35831 100644 --- a/TwitchDownloaderCore/Tools/M3U8.cs +++ b/TwitchDownloaderCore/Tools/M3U8.cs @@ -224,7 +224,6 @@ public enum PlaylistType // Twitch specific public uint TwitchLiveSequence { get; private set; } public decimal TwitchElapsedSeconds { get; private set; } - public decimal TwitchTotalSeconds { get; private set; } public decimal TwitchTotalSeconds { get; internal set; } // Other headers that we don't have dedicated properties for. Useful for debugging.