Skip to content

Commit

Permalink
Cleanup (#1152)
Browse files Browse the repository at this point in the history 
* Fix inconsistent capitalization

* Reduce repeat memory allocations due to excessive EnsureCapacity calls

* Manually report 100% progress

* Use adjusted FileInfo instead of mergeOptions.OutputFile

* Reduce allocations when aggregating & sorting comments

* Extract comment downloading & aggregating

* Use a Range for holding download region

* Clarify converted comment adding logic

* Fix potential for the last few seconds of chat messages to be not downloaded
The number of seconds that can be skipped is <= the number of chat connections

* Cleanup

* Make videoEnd exclusive to catch comments sent on last second of the broadcast

* progress will never be null

* Use constrained generic instead of IFormattable

* Support TimeSpanHFormat in FilenameService
  • Loading branch information
@ScrubN
ScrubN authored Jul 15, 2024
1 parent db0928a commit 68d60e0
Show file tree
Hide file tree
Showing 5 changed files with 98 additions and 73 deletions.
107 changes: 58 additions & 49 deletions TwitchDownloaderCore/ChatDownloader.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,12 +42,12 @@ public ChatDownloader(ChatDownloadOptions chatDownloadOptions, ITaskProgress pro
"TwitchDownloader");
}

private static async Task<List<Comment>> DownloadSection(double videoStart, double videoEnd, string videoId, IProgress<int> progress, ChatFormat format, CancellationToken cancellationToken)
private static async Task<List<Comment>> DownloadSection(Range downloadRange, string videoId, IProgress<int> progress, ChatFormat format, CancellationToken cancellationToken)
{
var comments = new List<Comment>();
//GQL only wants ints
videoStart = Math.Floor(videoStart);
double videoDuration = videoEnd - videoStart;
int videoStart = downloadRange.Start.Value;
int videoEnd = downloadRange.End.Value;
int videoDuration = videoEnd - videoStart;
double latestMessage = videoStart - 1;
bool isFirst = true;
string cursor = "";
Expand Down Expand Up @@ -104,25 +104,26 @@ private static async Task<List<Comment>> DownloadSection(double videoStart, doub
}

var convertedComments = ConvertComments(commentResponse[0].data.video, format);
comments.EnsureCapacity(Math.Min(0, comments.Capacity + convertedComments.Count));
foreach (var comment in convertedComments)
{
if (latestMessage < videoEnd && comment.content_offset_seconds > videoStart)
if (comment.content_offset_seconds >= videoStart && comment.content_offset_seconds < videoEnd)
{
comments.Add(comment);
}

latestMessage = comment.content_offset_seconds;
if (comment.content_offset_seconds > latestMessage)
{
latestMessage = comment.content_offset_seconds;
}
}

if (!commentResponse[0].data.video.comments.pageInfo.hasNextPage)
break;

cursor = commentResponse[0].data.video.comments.edges.Last().cursor;

if (progress != null)
{
int percent = (int)Math.Floor((latestMessage - videoStart) / videoDuration * 100);
progress.Report(percent);
}
var percent = (int)Math.Floor((latestMessage - videoStart) / videoDuration * 100);
progress.Report(percent);

if (isFirst)
{
Expand Down Expand Up @@ -276,43 +277,8 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF
DownloadType downloadType = downloadOptions.Id.All(char.IsDigit) ? DownloadType.Video : DownloadType.Clip;

var (chatRoot, connectionCount) = await InitChatRoot(downloadType);
var videoStart = chatRoot.video.start;
var videoEnd = chatRoot.video.end;
var videoId = chatRoot.video.id;
var videoDuration = videoEnd - videoStart;

var downloadTasks = new List<Task<List<Comment>>>(connectionCount);
var percentages = new int[connectionCount];

double chunk = videoDuration / connectionCount;
for (int i = 0; i < connectionCount; i++)
{
int tc = i;

var taskProgress = new Progress<int>(percent =>
{
percentages[tc] = Math.Clamp(percent, 0, 100);
var reportPercent = percentages.Sum() / connectionCount;
_progress.ReportProgress(reportPercent);
});

double start = videoStart + chunk * i;
downloadTasks.Add(DownloadSection(start, start + chunk, videoId, taskProgress, downloadOptions.DownloadFormat, cancellationToken));
}

_progress.SetTemplateStatus("Downloading {0}%", 0);
await Task.WhenAll(downloadTasks);

var sortedComments = new List<Comment>(downloadTasks.Count);
foreach (var commentTask in downloadTasks)
{
sortedComments.AddRange(commentTask.Result);
}

sortedComments.Sort(new CommentOffsetComparer());

chatRoot.comments = sortedComments.DistinctBy(x => x._id).ToList();
chatRoot.comments = await DownloadComments(cancellationToken, chatRoot.video, connectionCount);

if (downloadOptions.EmbedData && (downloadOptions.DownloadFormat is ChatFormat.Json or ChatFormat.Html))
{
Expand All @@ -326,7 +292,7 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF
await BackfillUserInfo(chatRoot);
}

_progress.SetStatus("Writing output file");
_progress.SetStatus("Writing Output File");
switch (downloadOptions.DownloadFormat)
{
case ChatFormat.Json:
Expand Down Expand Up @@ -440,6 +406,49 @@ private async Task DownloadAsyncImpl(FileInfo outputFileInfo, FileStream outputF
return (chatRoot, connectionCount);
}

private async Task<List<Comment>> DownloadComments(CancellationToken cancellationToken, Video video, int connectionCount)
{
_progress.SetTemplateStatus("Downloading {0}%", 0);

var videoStart = (int)Math.Floor(video.start);
var videoEnd = (int)Math.Ceiling(video.end) + 1; // Exclusive end
var videoDuration = videoEnd - videoStart;

var downloadTasks = new List<Task<List<Comment>>>(connectionCount);
var percentages = new int[connectionCount];

var chunkSize = (int)Math.Ceiling(videoDuration / (double)connectionCount);
for (var i = 0; i < connectionCount; i++)
{
var tc = i;

var taskProgress = new Progress<int>(percent =>
{
percentages[tc] = Math.Clamp(percent, 0, 100);
var reportPercent = percentages.Sum() / connectionCount;
_progress.ReportProgress(reportPercent);
});

var start = videoStart + chunkSize * i;
var end = Math.Min(videoEnd, start + chunkSize);
var downloadRange = new Range(start, end);
downloadTasks.Add(DownloadSection(downloadRange, video.id, taskProgress, downloadOptions.DownloadFormat, cancellationToken));
}

await Task.WhenAll(downloadTasks);

_progress.ReportProgress(100);

var commentList = downloadTasks
.SelectMany(task => task.Result)
.ToHashSet(new CommentIdEqualityComparer())
.ToList();

commentList.Sort(new CommentOffsetComparer());
return commentList;
}

private async Task EmbedImages(ChatRoot chatRoot, CancellationToken cancellationToken)
{
_progress.SetTemplateStatus("Downloading Embed Images {0}%", 0);
Expand Down
16 changes: 4 additions & 12 deletions TwitchDownloaderCore/ChatUpdater.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -431,26 +431,18 @@ private async Task ChatTrimEndingTask(CancellationToken cancellationToken)
ChatRoot newChatRoot = await ChatJson.DeserializeAsync(inputFile, getComments: true, onlyFirstAndLastComments: false, getEmbeds: false, cancellationToken);

// Append the new comment section
SortedSet<Comment> commentsSet = new SortedSet<Comment>(new CommentOffsetComparer());
foreach (var comment in newChatRoot.comments)
{
if (comment.content_offset_seconds < downloadOptions.TrimEndingTime && comment.content_offset_seconds >= downloadOptions.TrimBeginningTime)
{
commentsSet.Add(comment);
}
}
var commentsSet = newChatRoot.comments.ToHashSet(new CommentIdEqualityComparer());

lock (_trimChatRootLock)
{
commentsSet.EnsureCapacity(commentsSet.Count + chatRoot.comments.Count);
foreach (var comment in chatRoot.comments)
{
commentsSet.Add(comment);
}

List<Comment> comments = commentsSet.DistinctBy(x => x._id).ToList();
commentsSet.Clear();

chatRoot.comments = comments;
chatRoot.comments = commentsSet.ToList();
chatRoot.comments.Sort(new CommentOffsetComparer());
}
}

Expand Down
18 changes: 18 additions & 0 deletions TwitchDownloaderCore/Tools/CommentIdEqualityComparer.cs
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
using System.Collections.Generic;
using TwitchDownloaderCore.TwitchObjects;

namespace TwitchDownloaderCore.Tools
{
public class CommentIdEqualityComparer : IEqualityComparer<Comment>
{
public bool Equals(Comment x, Comment y)
{
if (x is null) return y is null;
if (y is null) return false;

return x._id.Equals(y._id);
}

public int GetHashCode(Comment obj) => obj._id.GetHashCode();
}
}
14 changes: 9 additions & 5 deletions TwitchDownloaderCore/Tools/FilenameService.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,27 +35,27 @@ public static string GetFilename(string template, [AllowNull] string title, [All
if (template.Contains("{trim_start_custom="))
{
var trimStartRegex = new Regex("{trim_start_custom=\"(.*?)\"}");
ReplaceCustomWithFormattable(stringBuilder, trimStartRegex, trimStart);
ReplaceCustomWithFormattable(stringBuilder, trimStartRegex, trimStart, TimeSpanHFormat.ReusableInstance);
}

if (template.Contains("{trim_end_custom="))
{
var trimEndRegex = new Regex("{trim_end_custom=\"(.*?)\"}");
ReplaceCustomWithFormattable(stringBuilder, trimEndRegex, trimEnd);
ReplaceCustomWithFormattable(stringBuilder, trimEndRegex, trimEnd, TimeSpanHFormat.ReusableInstance);
}

if (template.Contains("{length_custom="))
{
var lengthRegex = new Regex("{length_custom=\"(.*?)\"}");
ReplaceCustomWithFormattable(stringBuilder, lengthRegex, videoLength);
ReplaceCustomWithFormattable(stringBuilder, lengthRegex, videoLength, TimeSpanHFormat.ReusableInstance);
}

var fileName = stringBuilder.ToString();
var additionalSubfolders = GetTemplateSubfolders(ref fileName);
return Path.Combine(Path.Combine(additionalSubfolders), ReplaceInvalidFilenameChars(fileName));
}

private static void ReplaceCustomWithFormattable(StringBuilder sb, Regex regex, IFormattable formattable, IFormatProvider formatProvider = null)
private static void ReplaceCustomWithFormattable<TFormattable>(StringBuilder sb, Regex regex, TFormattable formattable, [AllowNull] IFormatProvider formatProvider = null) where TFormattable : IFormattable
{
do
{
Expand All @@ -66,8 +66,12 @@ private static void ReplaceCustomWithFormattable(StringBuilder sb, Regex regex,
break;

var formatString = match.Groups[1].Value;
var formattedString = formatProvider?.GetFormat(typeof(ICustomFormatter)) is ICustomFormatter customFormatter
? customFormatter.Format(formatString, formattable, formatProvider)
: formattable.ToString(formatString, formatProvider);

sb.Remove(match.Groups[0].Index, match.Groups[0].Length);
sb.Insert(match.Groups[0].Index, ReplaceInvalidFilenameChars(formattable.ToString(formatString, formatProvider)));
sb.Insert(match.Groups[0].Index, ReplaceInvalidFilenameChars(formattedString));
} while (true);
}

Expand Down
16 changes: 9 additions & 7 deletions TwitchDownloaderCore/TsMerger.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public async Task MergeAsync(CancellationToken cancellationToken)

try
{
await MergeAsyncImpl(outputFs, cancellationToken);
await MergeAsyncImpl(outputFileInfo, outputFs, cancellationToken);
}
catch
{
Expand All @@ -46,7 +46,7 @@ public async Task MergeAsync(CancellationToken cancellationToken)
}
}

private async Task MergeAsyncImpl(FileStream outputFs, CancellationToken cancellationToken)
private async Task MergeAsyncImpl(FileInfo outputFileInfo, FileStream outputFs, CancellationToken cancellationToken)
{
var isM3U8 = false;
var isFirst = true;
Expand Down Expand Up @@ -74,9 +74,7 @@ private async Task MergeAsyncImpl(FileStream outputFs, CancellationToken cancell

_progress.SetTemplateStatus("Combining Parts {0}% [2/2]", 0);

await CombineVideoParts(fileList, outputFs, cancellationToken);

_progress.ReportProgress(100);
await CombineVideoParts(fileList, outputFileInfo, outputFs, cancellationToken);
}

private async Task VerifyVideoParts(IReadOnlyCollection<string> fileList, CancellationToken cancellationToken)
Expand All @@ -100,6 +98,8 @@ private async Task VerifyVideoParts(IReadOnlyCollection<string> fileList, Cancel
cancellationToken.ThrowIfCancellationRequested();
}

_progress.ReportProgress(100);

if (failedParts.Count != 0)
{
if (failedParts.Count == fileList.Count)
Expand Down Expand Up @@ -131,9 +131,9 @@ private static async Task<bool> VerifyVideoPart(string filePath)
return true;
}

private async Task CombineVideoParts(IReadOnlyCollection<string> fileList, FileStream outputStream, CancellationToken cancellationToken)
private async Task CombineVideoParts(IReadOnlyCollection<string> fileList, FileInfo outputFileInfo, FileStream outputStream, CancellationToken cancellationToken)
{
DriveInfo outputDrive = DriveHelper.GetOutputDrive(mergeOptions.OutputFile);
DriveInfo outputDrive = DriveHelper.GetOutputDrive(outputFileInfo.FullName);

int partCount = fileList.Count;
int doneCount = 0;
Expand All @@ -153,6 +153,8 @@ private async Task CombineVideoParts(IReadOnlyCollection<string> fileList, FileS

cancellationToken.ThrowIfCancellationRequested();
}

_progress.ReportProgress(100);
}
}
}

0 comments on commit 68d60e0

Please sign in to comment.