using System.IO; using System.Net.Http; using CompatApiClient.Utils; using Microsoft.Extensions.Caching.Memory; using Microsoft.TeamFoundation.Build.WebApi; using SharpCompress.Readers; namespace CompatBot.Utils.Extensions; internal static class AzureDevOpsClientExtensions { private static readonly MemoryCache BuildInfoCache = new(new MemoryCacheOptions { ExpirationScanFrequency = TimeSpan.FromHours(1) }); private const string RepoId = "RPCS3/rpcs3"; private const string RepoType = "GitHub"; public record BuildInfo { public string? Commit { get; init; } public string? WindowsFilename { get; init; } public string? LinuxFilename { get; init; } public string? MacFilename { get; init; } public string? LinuxArmFilename { get; init; } public string? MacArmFilename { get; init; } public string? WindowsBuildDownloadLink { get; init; } public string? LinuxBuildDownloadLink { get; init; } public string? MacBuildDownloadLink { get; init; } public string? LinuxArmBuildDownloadLink { get; init; } public string? MacArmBuildDownloadLink { get; init; } public DateTime? StartTime { get; init; } public DateTime? FinishTime { get; init; } public BuildStatus? Status { get; init; } public BuildResult? Result { get; init; } } public record PipelineStats { public TimeSpan Percentile95 { get; init; } public TimeSpan Percentile90 { get; init; } public TimeSpan Percentile85 { get; init; } public TimeSpan Percentile80 { get; init; } public TimeSpan Mean { get; init; } public TimeSpan StdDev { get; init; } public int BuildCount { get; init; } public static readonly PipelineStats Defaults = new() { Percentile95 = TimeSpan.FromMinutes(60.82984169), Percentile90 = TimeSpan.FromMinutes(59.145449178333337), Percentile85 = TimeSpan.FromMinutes(57.569420649999998), Percentile80 = TimeSpan.FromMinutes(57.033788216666665), Mean = TimeSpan.FromMinutes(50.7788), StdDev = TimeSpan.FromMinutes(7.376), }; } public static async Task GetPipelineDurationAsync(this BuildHttpClient? azureDevOpsClient, CancellationToken cancellationToken) { const string cacheKey = "pipeline-duration"; if (BuildInfoCache.TryGetValue(cacheKey, out PipelineStats? result) && result is not null) return result; if (azureDevOpsClient is null) return PipelineStats.Defaults; var builds = await azureDevOpsClient.GetBuildsAsync( Config.AzureDevOpsProjectId, repositoryId: RepoId, repositoryType: RepoType, statusFilter: BuildStatus.Completed, resultFilter: BuildResult.Succeeded, minFinishTime: DateTime.UtcNow.AddDays(-7), cancellationToken: cancellationToken ).ConfigureAwait(false); var times = builds .Where(b => b is {StartTime: not null, FinishTime: not null}) .Select(b => (b.FinishTime - b.StartTime)!.Value) .OrderByDescending(t => t) .ToList(); if (times.Count <= 10) return PipelineStats.Defaults; result = new() { Percentile95 = times[(int)(times.Count * 0.05)], Percentile90 = times[(int)(times.Count * 0.10)], Percentile85 = times[(int)(times.Count * 0.16)], Percentile80 = times[(int)(times.Count * 0.20)], Mean = TimeSpan.FromTicks(times.Select(t => t.Ticks).Mean()), StdDev = TimeSpan.FromTicks((long)times.Select(t => t.Ticks).StdDev()), BuildCount = times.Count, }; BuildInfoCache.Set(cacheKey, result, TimeSpan.FromDays(1)); return result; } public static async ValueTask?> GetMasterBuildsAsync(this BuildHttpClient? azureDevOpsClient, string? oldestMergeCommit, string? newestMergeCommit, DateTime? oldestTimestamp, CancellationToken cancellationToken) { if (azureDevOpsClient == null || string.IsNullOrEmpty(oldestMergeCommit) || string.IsNullOrEmpty(newestMergeCommit)) return null; oldestMergeCommit = oldestMergeCommit.ToLower(); newestMergeCommit = newestMergeCommit.ToLower(); var builds = await azureDevOpsClient.GetBuildsAsync( Config.AzureDevOpsProjectId, repositoryId: RepoId, repositoryType: RepoType, reasonFilter: BuildReason.IndividualCI, minFinishTime: oldestTimestamp, cancellationToken: cancellationToken ).ConfigureAwait(false); builds = builds .Where(b => b.SourceBranch == "refs/heads/master" && b.Status is BuildStatus.Completed) .OrderByDescending(b => b.StartTime) .ToList(); builds = builds .SkipWhile(b => !newestMergeCommit.Equals(b.SourceVersion, StringComparison.InvariantCultureIgnoreCase)) .Skip(1) .TakeWhile(b => !oldestMergeCommit.Equals(b.SourceVersion, StringComparison.InvariantCultureIgnoreCase)) .ToList(); return await builds .ToAsyncEnumerable() .Select((b, ct) => azureDevOpsClient.GetArtifactsInfoAsync(b.SourceVersion, b, ct)) .ToListAsync(cancellationToken: cancellationToken) .ConfigureAwait(false); } public static async ValueTask GetMasterBuildInfoAsync(this BuildHttpClient? azureDevOpsClient, string? commit, DateTime? oldestTimestamp, CancellationToken cancellationToken) { if (azureDevOpsClient == null || string.IsNullOrEmpty(commit)) return null; commit = commit.ToLower(); if (BuildInfoCache.TryGetValue(commit, out BuildInfo? result) && result is not null) return result; var builds = await azureDevOpsClient.GetBuildsAsync( Config.AzureDevOpsProjectId, repositoryId: RepoId, repositoryType: RepoType, reasonFilter: BuildReason.IndividualCI, minFinishTime: oldestTimestamp, cancellationToken: cancellationToken ).ConfigureAwait(false); builds = builds .Where(b => b.SourceBranch == "refs/heads/master" && commit.Equals(b.SourceVersion, StringComparison.InvariantCultureIgnoreCase) && b.Status == BuildStatus.Completed ) .OrderByDescending(b => b.StartTime) .ToList(); var latestBuild = builds.FirstOrDefault(); if (latestBuild == null) return null; result = await azureDevOpsClient.GetArtifactsInfoAsync(commit, latestBuild, cancellationToken).ConfigureAwait(false); if (result.Status == BuildStatus.Completed && (result.Result == BuildResult.Succeeded || result.Result == BuildResult.PartiallySucceeded)) BuildInfoCache.Set(commit, result, TimeSpan.FromHours(1)); return result; } public static async ValueTask GetPrBuildInfoAsync(this BuildHttpClient? azureDevOpsClient, string? commit, DateTime? oldestTimestamp, int pr, CancellationToken cancellationToken) { if (azureDevOpsClient == null || string.IsNullOrEmpty(commit)) return null; commit = commit.ToLower(); if (BuildInfoCache.TryGetValue(commit, out BuildInfo? result) && result is not null) return result; var builds = await azureDevOpsClient.GetBuildsAsync( Config.AzureDevOpsProjectId, repositoryId: RepoId, repositoryType: RepoType, reasonFilter: BuildReason.PullRequest, minFinishTime: oldestTimestamp, cancellationToken: cancellationToken ).ConfigureAwait(false); var filterBranch = $"refs/pull/{pr}/merge"; builds = builds .Where(b => b.SourceBranch == filterBranch && b.TriggerInfo.TryGetValue("pr.sourceSha", out var trc) && trc.Equals(commit, StringComparison.InvariantCultureIgnoreCase)) .OrderByDescending(b => b.StartTime) .ToList(); var latestBuild = builds.FirstOrDefault(); if (latestBuild == null) return null; result = await azureDevOpsClient.GetArtifactsInfoAsync(commit, latestBuild, cancellationToken).ConfigureAwait(false); if (result is { Status: BuildStatus.Completed, Result: BuildResult.Succeeded or BuildResult.PartiallySucceeded }) BuildInfoCache.Set(commit, result, TimeSpan.FromHours(1)); return result; } public static async ValueTask GetArtifactsInfoAsync(this BuildHttpClient azureDevOpsClient, string commit, Build build, CancellationToken cancellationToken) { var result = new BuildInfo { Commit = commit, StartTime = build.StartTime, FinishTime = build.FinishTime, Status = build.Status, Result = build.Result, }; var artifacts = await azureDevOpsClient.GetArtifactsAsync(Config.AzureDevOpsProjectId, build.Id, cancellationToken: cancellationToken).ConfigureAwait(false); // windows build var windowsBuildArtifact = artifacts.FirstOrDefault(a => a.Name.Contains("Windows")); var windowsBuild = windowsBuildArtifact?.Resource; if (windowsBuild?.DownloadUrl is string winDownloadUrl) { result = result with {WindowsBuildDownloadLink = winDownloadUrl}; if (windowsBuild.DownloadUrl.Contains("format=zip", StringComparison.InvariantCultureIgnoreCase)) try { using var httpClient = HttpClientFactory.Create(); await using var stream = await httpClient.GetStreamAsync(winDownloadUrl, cancellationToken).ConfigureAwait(false); using var zipStream = ReaderFactory.Open(stream); while (zipStream.MoveToNextEntry() && !cancellationToken.IsCancellationRequested) { if (zipStream.Entry.Key?.EndsWith(".7z", StringComparison.InvariantCultureIgnoreCase) is true) { result = result with {WindowsFilename = Path.GetFileName(zipStream.Entry.Key)}; break; } } } catch (Exception e2) { Config.Log.Error(e2, "Failed to get windows build filename"); } } // linux build var linuxBuildArtifact = artifacts.FirstOrDefault(a => a.Name.EndsWith(".GCC") || a.Name.EndsWith("Linux") || a.Name.EndsWith("(gcc)") || a.Name.EndsWith("(clang)")); var linuxBuild = linuxBuildArtifact?.Resource; if (linuxBuild?.DownloadUrl is string linDownloadUrl) { result = result with {LinuxBuildDownloadLink = linDownloadUrl}; if (linuxBuild.DownloadUrl.Contains("format=zip", StringComparison.InvariantCultureIgnoreCase)) try { using var httpClient = HttpClientFactory.Create(); await using var stream = await httpClient.GetStreamAsync(linDownloadUrl, cancellationToken).ConfigureAwait(false); using var zipStream = ReaderFactory.Open(stream); while (zipStream.MoveToNextEntry() && !cancellationToken.IsCancellationRequested) { if (zipStream.Entry.Key?.EndsWith(".AppImage", StringComparison.InvariantCultureIgnoreCase) is true) { result = result with {LinuxFilename = Path.GetFileName(zipStream.Entry.Key)}; break; } } } catch (Exception e2) { Config.Log.Error(e2, "Failed to get linux build filename"); } } // mac build var macBuildArtifact = artifacts.FirstOrDefault(a => a.Name.Contains("Mac") && a.Name.Contains("Intel")); var macBuild = macBuildArtifact?.Resource; if (macBuild?.DownloadUrl is string macDownloadUrl) { result = result with { MacBuildDownloadLink = macDownloadUrl }; if (macBuild.DownloadUrl.Contains("format=zip", StringComparison.InvariantCultureIgnoreCase)) try { using var httpClient = HttpClientFactory.Create(); await using var stream = await httpClient.GetStreamAsync(macDownloadUrl, cancellationToken).ConfigureAwait(false); using var zipStream = ReaderFactory.Open(stream); while (zipStream.MoveToNextEntry() && !cancellationToken.IsCancellationRequested) { if (zipStream.Entry.Key?.EndsWith(".dmg", StringComparison.InvariantCultureIgnoreCase) is true || zipStream.Entry.Key?.EndsWith(".7z", StringComparison.InvariantCultureIgnoreCase) is true) { result = result with { MacFilename = Path.GetFileName(zipStream.Entry.Key) }; break; } } } catch (Exception e2) { Config.Log.Error(e2, "Failed to get mac build filename"); } } // mac arm build var macArmBuildArtifact = artifacts.FirstOrDefault(a => a.Name.Contains("Mac") && a.Name.Contains("Apple")); var macArmBuild = macArmBuildArtifact?.Resource; if (macArmBuild?.DownloadUrl is string macArmDownloadUrl) { result = result with { MacArmBuildDownloadLink = macArmDownloadUrl }; if (macArmBuild.DownloadUrl.Contains("format=zip", StringComparison.InvariantCultureIgnoreCase)) try { using var httpClient = HttpClientFactory.Create(); await using var stream = await httpClient.GetStreamAsync(macArmDownloadUrl, cancellationToken).ConfigureAwait(false); using var zipStream = ReaderFactory.Open(stream); while (zipStream.MoveToNextEntry() && !cancellationToken.IsCancellationRequested) { if (zipStream.Entry.Key?.EndsWith(".dmg", StringComparison.InvariantCultureIgnoreCase) is true || zipStream.Entry.Key?.EndsWith(".7z", StringComparison.InvariantCultureIgnoreCase) is true) { result = result with { MacArmFilename = Path.GetFileName(zipStream.Entry.Key) }; break; } } } catch (Exception e2) { Config.Log.Error(e2, "Failed to get mac arm build filename"); } } return result; } }