From 9fe21dcc20f4c8d4d98a7f885d858cb10b86332a Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sun, 1 Mar 2020 16:49:36 +0500 Subject: [PATCH 1/8] update deps --- Clients/AppveyorClient/AppveyorClient.csproj | 2 +- Clients/GithubClient/GithubClient.csproj | 2 +- Clients/PsnClient/PsnClient.csproj | 2 +- CompatBot/CompatBot.csproj | 24 ++++++++++---------- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/Clients/AppveyorClient/AppveyorClient.csproj b/Clients/AppveyorClient/AppveyorClient.csproj index 14dba063..bdbb74b3 100644 --- a/Clients/AppveyorClient/AppveyorClient.csproj +++ b/Clients/AppveyorClient/AppveyorClient.csproj @@ -6,7 +6,7 @@ - + diff --git a/Clients/GithubClient/GithubClient.csproj b/Clients/GithubClient/GithubClient.csproj index 14dba063..bdbb74b3 100644 --- a/Clients/GithubClient/GithubClient.csproj +++ b/Clients/GithubClient/GithubClient.csproj @@ -6,7 +6,7 @@ - + diff --git a/Clients/PsnClient/PsnClient.csproj b/Clients/PsnClient/PsnClient.csproj index bfb7131d..0e2577ad 100644 --- a/Clients/PsnClient/PsnClient.csproj +++ b/Clients/PsnClient/PsnClient.csproj @@ -14,7 +14,7 @@ - + diff --git a/CompatBot/CompatBot.csproj b/CompatBot/CompatBot.csproj index 1b5c880b..06e58143 100644 --- a/CompatBot/CompatBot.csproj +++ b/CompatBot/CompatBot.csproj @@ -24,28 +24,28 @@ - - - + + + - + - - + + all runtime; build; native; contentfiles; analyzers; buildtransitive - - - - - + + + + + - + From 424242ed3157226f2ccb0b4220ad157162618ce0 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sun, 1 Mar 2020 17:20:25 +0500 Subject: [PATCH 2/8] unify fatal error extraction --- CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs index 7a504cf8..183afe3e 100644 --- a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs +++ b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs @@ -193,8 +193,7 @@ namespace CompatBot.EventHandlers.LogParsing ["RSX: Your GPU does not support"] = new Regex(@"RSX: Your GPU does not support (?.+)\..+?\r?$", DefaultOptions), ["RSX: GPU/driver lacks support"] = new Regex(@"RSX: GPU/driver lacks support for (?.+)\..+?\r?$", DefaultOptions), ["RSX: Swapchain:"] = new Regex(@"RSX: Swapchain: present mode (?\d+?) in use.+?\r?$", DefaultOptions), - ["F "] = new Regex(@"F \d+:\d+:\d+\.\d+ ({(?.+?)} )?(?.*?(\:\W*\r?\n\(.*?)*)\r?$", DefaultOptions), - ["} SYS:"] = new Regex(@"F \d+:\d+:\d+\.\d+ (({(?.+)} )?SYS:\s*\r?\n)(?.*?)(\r?\n)(\r?\n|\xC2\xB7)", DefaultSingleLineOptions), + ["F "] = new Regex(@"F \d+:\d+:\d+\.\d+ (({(?[^}]+)} )?\w+:\s*(class [^\r\n]+ thrown: )?\r?\n?)(?.*?)(\r?\n)(\r?\n|\xC2\xB7)", DefaultSingleLineOptions), ["Failed to load RAP file:"] = new Regex(@"Failed to load RAP file: (?.*?\.rap).*\r?$", DefaultOptions), ["Rap file not found:"] = new Regex(@"Rap file not found: (?.*?)\r?$", DefaultOptions), ["Pad handler expected but none initialized"] = new Regex(@"(?Pad handler expected but none initialized).*?\r?$", DefaultOptions), From 9a47809f847092abad46262c45cfc9100f8694e3 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sun, 1 Mar 2020 18:25:57 +0500 Subject: [PATCH 3/8] log parser was skipping the very last line of the log --- CompatBot/EventHandlers/LogParsing/LogParser.PipeReader.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/CompatBot/EventHandlers/LogParsing/LogParser.PipeReader.cs b/CompatBot/EventHandlers/LogParsing/LogParser.PipeReader.cs index 1b77ab37..bfc06c7d 100644 --- a/CompatBot/EventHandlers/LogParsing/LogParser.PipeReader.cs +++ b/CompatBot/EventHandlers/LogParsing/LogParser.PipeReader.cs @@ -67,7 +67,11 @@ namespace CompatBot.EventHandlers.LogParsing state.Error = LogParseState.ErrorCode.SizeLimit; } else if (result.IsCompleted) + { + if (!buffer.End.Equals(currentSectionLines.Last.Value.End)) + await OnNewLineAsync(buffer.Slice(0), result.Buffer, currentSectionLines, state).ConfigureAwait(false); await FlushAllLinesAsync(result.Buffer, currentSectionLines, state).ConfigureAwait(false); + } var sectionStart = currentSectionLines.Count == 0 ? buffer : currentSectionLines.First.Value; totalReadBytes += result.Buffer.Slice(0, sectionStart.Start).Length; reader.AdvanceTo(sectionStart.Start); @@ -112,7 +116,6 @@ namespace CompatBot.EventHandlers.LogParsing if (currentProcessor.OnExtract != null) await TaskScheduler.AddAsync(state, Task.Run(() => currentProcessor.OnExtract(firstSectionLine, section, state))); sectionLines.RemoveFirst(); - } } } From f065a500f852f873c2c0f8694b787c736a012832 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sun, 1 Mar 2020 19:08:34 +0500 Subject: [PATCH 4/8] change note priority for controller binding --- .../LogParserResultFormatter.GeneralNotesSection.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs index 0adf9fbd..59d72741 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs @@ -461,7 +461,7 @@ namespace CompatBot.Utils.ResultFormatters } if (items["failed_pad"] is string failedPad) - notes.Add($"❌ Binding `{failedPad.Sanitize(replaceBackTicks: true)}` failed, check if device is connected."); + notes.Add($"⚠ Binding `{failedPad.Sanitize(replaceBackTicks: true)}` failed, check if device is connected."); if (DesIds.Contains(serial)) notes.Add("ℹ If you experience infinite load screen, clear game cache via `File` → `All games` → `Remove Disk Cache`"); From 23f706c98f18bf9cba57689e12bd3dd25526cf33 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 2 Mar 2020 13:29:15 +0500 Subject: [PATCH 5/8] fix license extractor for the new logs --- CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs index 183afe3e..32f0978c 100644 --- a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs +++ b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs @@ -195,7 +195,7 @@ namespace CompatBot.EventHandlers.LogParsing ["RSX: Swapchain:"] = new Regex(@"RSX: Swapchain: present mode (?\d+?) in use.+?\r?$", DefaultOptions), ["F "] = new Regex(@"F \d+:\d+:\d+\.\d+ (({(?[^}]+)} )?\w+:\s*(class [^\r\n]+ thrown: )?\r?\n?)(?.*?)(\r?\n)(\r?\n|\xC2\xB7)", DefaultSingleLineOptions), ["Failed to load RAP file:"] = new Regex(@"Failed to load RAP file: (?.*?\.rap).*\r?$", DefaultOptions), - ["Rap file not found:"] = new Regex(@"Rap file not found: (?.*?)\r?$", DefaultOptions), + ["Rap file not found:"] = new Regex(@"Rap file not found: (\xE2\x80\x9C)?(?.*?)(\xE2\x80\x9D)?\r?$", DefaultOptions), ["Pad handler expected but none initialized"] = new Regex(@"(?Pad handler expected but none initialized).*?\r?$", DefaultOptions), ["XAudio2Thread"] = new Regex(@"XAudio2Thread\s*: (?.+failed\s*\((?0x.+)\).*)\r?$", DefaultOptions), ["cellAudio Thread"] = new Regex(@"XAudio2Backend\s*: (?.+failed\s*\((?0x.+)\).*)\r?$", DefaultOptions), From 639bdd280c45478097fa24fc0ef88782103f2eac Mon Sep 17 00:00:00 2001 From: 13xforever Date: Sun, 1 Mar 2020 23:44:55 +0500 Subject: [PATCH 6/8] wip --- CompatBot/Utils/UniqueList.cs | 93 +++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 CompatBot/Utils/UniqueList.cs diff --git a/CompatBot/Utils/UniqueList.cs b/CompatBot/Utils/UniqueList.cs new file mode 100644 index 00000000..fd68033d --- /dev/null +++ b/CompatBot/Utils/UniqueList.cs @@ -0,0 +1,93 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace CompatBot.Utils +{ + public class UniqueList: IList + { + private readonly List list; + private readonly HashSet set; + + public UniqueList(IEqualityComparer comparer = null) + { + list = new List(); + set = new HashSet(comparer); + } + + public UniqueList(int count, IEqualityComparer comparer = null) + { + list = new List(count); + set = new HashSet(count, comparer); + } + + public UniqueList(IEnumerable collection, IEqualityComparer comparer = null) + { + if (collection is ICollection c) + { + list = new List(c.Count); + set = new HashSet(c.Count, comparer); + } + else + { + list = new List(); + set = new HashSet(comparer); + } + foreach (var item in collection) + if (set.Add(item)) + list.Add(item); + } + + public IEnumerator GetEnumerator() => list.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => ((IEnumerable)list).GetEnumerator(); + + public void Add(T item) + { + if (set.Add(item)) + list.Add(item); + } + + public void Clear() + { + list.Clear(); + set.Clear(); + } + + public bool Contains(T item) => set.Contains(item); + + public void CopyTo(T[] array, int arrayIndex) => list.CopyTo(array, arrayIndex); + + public bool Remove(T item) + { + list.Remove(item); + return set.Remove(item); + } + + public int Count => list.Count; + + public bool IsReadOnly => false; + + public int IndexOf(T item) => list.IndexOf(item); + + public void Insert(int index, T item) + { + if (set.Add(item)) + list.Insert(index, item); + else if (IndexOf(item) != index) + throw new ArgumentException("Collection already contains item at different index", nameof(item)); + } + + public void RemoveAt(int index) + { + var item = this[index]; + list.RemoveAt(index); + set.Remove(item); + } + + public T this[int index] { + get => list[index]; + set => throw new NotSupportedException(); + } + } +} From 0d5a838d2af8d6d6a87329a3119a9d063ec35b37 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 2 Mar 2020 16:06:33 +0500 Subject: [PATCH 7/8] introduce proper multivalue collections for log parser (WIP) --- .../LogParsing/LogParser.LogSections.cs | 6 ++ .../LogParser.StateMachineGenerator.cs | 7 +- .../LogParsing/POCOs/LogParseState.cs | 3 + .../EventHandlers/UnknownCommandHandler.cs | 3 +- CompatBot/Utils/NameUniqueObjectCollection.cs | 80 +++++++++++++++++++ ...rserResultFormatter.GeneralNotesSection.cs | 43 +++++----- ...serResultFormatter.WeirdSettingsSection.cs | 26 +++--- .../LogParserResultFormatter.cs | 30 ++++--- CompatBot/Utils/UniqueList.cs | 6 ++ 9 files changed, 151 insertions(+), 53 deletions(-) create mode 100644 CompatBot/Utils/NameUniqueObjectCollection.cs diff --git a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs index 32f0978c..478fe182 100644 --- a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs +++ b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs @@ -289,10 +289,15 @@ namespace CompatBot.EventHandlers.LogParsing void Copy(params string[] keys) { foreach (var key in keys) + { if (state.CompleteCollection?[key] is string value) state.WipCollection[key] = value; + if (state.CompleteMultiValueCollection?[key] is UniqueList collection) + state.WipMultiValueCollection[key] = collection; + } } state.WipCollection = new NameValueCollection(); + state.WipMultiValueCollection = new NameUniqueObjectCollection(); Copy( "build_and_specs", "fw_version_installed", "vulkan_gpu", "d3d_gpu", @@ -308,6 +313,7 @@ namespace CompatBot.EventHandlers.LogParsing private static void MarkAsComplete(LogParseState state) { state.CompleteCollection = state.WipCollection; + state.CompleteMultiValueCollection = state.WipMultiValueCollection; Config.Log.Trace("----- complete section"); } diff --git a/CompatBot/EventHandlers/LogParsing/LogParser.StateMachineGenerator.cs b/CompatBot/EventHandlers/LogParsing/LogParser.StateMachineGenerator.cs index 3d36c04a..bf5d7577 100644 --- a/CompatBot/EventHandlers/LogParsing/LogParser.StateMachineGenerator.cs +++ b/CompatBot/EventHandlers/LogParsing/LogParser.StateMachineGenerator.cs @@ -92,12 +92,7 @@ namespace CompatBot.EventHandlers.LogParsing lock (state) { if (MultiValueItems.Contains(group.Name)) - { - var currentValue = state.WipCollection[group.Name]; - if (!string.IsNullOrEmpty(currentValue)) - currentValue += Environment.NewLine; - state.WipCollection[group.Name] = currentValue + strValue; - } + state.WipMultiValueCollection[group.Name].Add(strValue); else state.WipCollection[group.Name] = strValue; if (CountValueItems.Contains(group.Name)) diff --git a/CompatBot/EventHandlers/LogParsing/POCOs/LogParseState.cs b/CompatBot/EventHandlers/LogParsing/POCOs/LogParseState.cs index 317c4247..49e6f3d6 100644 --- a/CompatBot/EventHandlers/LogParsing/POCOs/LogParseState.cs +++ b/CompatBot/EventHandlers/LogParsing/POCOs/LogParseState.cs @@ -2,13 +2,16 @@ using System.Collections.Generic; using System.Collections.Specialized; using CompatBot.Database; +using CompatBot.Utils; namespace CompatBot.EventHandlers.LogParsing.POCOs { public class LogParseState { public NameValueCollection CompleteCollection = null; + public NameUniqueObjectCollection CompleteMultiValueCollection = null; public NameValueCollection WipCollection = new NameValueCollection(); + public NameUniqueObjectCollection WipMultiValueCollection = new NameUniqueObjectCollection(); public readonly Dictionary ValueHitStats = new Dictionary(); public readonly Dictionary>> Syscalls = new Dictionary>>(); public int Id = 0; diff --git a/CompatBot/EventHandlers/UnknownCommandHandler.cs b/CompatBot/EventHandlers/UnknownCommandHandler.cs index 41de6930..263d0d85 100644 --- a/CompatBot/EventHandlers/UnknownCommandHandler.cs +++ b/CompatBot/EventHandlers/UnknownCommandHandler.cs @@ -40,11 +40,12 @@ namespace CompatBot.EventHandlers && (e.Context.Message.Content?.EndsWith("?") ?? false) && e.Context.CommandsNext.RegisteredCommands.TryGetValue("8ball", out var cmd)) { + var prefixLen = e.Context.Prefix.Length; // workaround for resharper bug var updatedContext = e.Context.CommandsNext.CreateContext( e.Context.Message, e.Context.Prefix, cmd, - e.Context.Message.Content[(e.Context.Prefix.Length)..].Trim() + e.Context.Message.Content[prefixLen..].Trim() ); try { await cmd.ExecuteAsync(updatedContext).ConfigureAwait(false); } catch { } return; diff --git a/CompatBot/Utils/NameUniqueObjectCollection.cs b/CompatBot/Utils/NameUniqueObjectCollection.cs new file mode 100644 index 00000000..2dad6fd8 --- /dev/null +++ b/CompatBot/Utils/NameUniqueObjectCollection.cs @@ -0,0 +1,80 @@ +using System; +using System.Collections; +using System.Collections.Generic; + +namespace CompatBot.Utils +{ + public class NameUniqueObjectCollection: IDictionary> + { + private readonly Dictionary> dict; + private readonly IEqualityComparer valueComparer; + + public NameUniqueObjectCollection(IEqualityComparer keyComparer = null, IEqualityComparer valueComparer = null) + { + dict = new Dictionary>(keyComparer); + this.valueComparer = valueComparer; + } + + public IEnumerator>> GetEnumerator() => dict.GetEnumerator(); + + IEnumerator IEnumerable.GetEnumerator() => ((IEnumerable)dict).GetEnumerator(); + + void ICollection>>.Add(KeyValuePair> item) => Add(item.Key, item.Value); + + public void Add(string key, UniqueList value) + { + value ??= new UniqueList(valueComparer); + if (dict.TryGetValue(key, out var c)) + c.AddRange(value); + else + dict.Add(key, value); + } + + public void Add(string key, TValue value) + { + if (dict.TryGetValue(key, out var c)) + c.Add(value); + else + { + dict[key] = c = new UniqueList(valueComparer); + c.Add(value); + } + } + + public void Clear() => dict.Clear(); + + bool ICollection>>.Contains(KeyValuePair> item) => ((IDictionary>)dict).Contains(item); + + void ICollection>>.CopyTo(KeyValuePair>[] array, int arrayIndex) => ((IDictionary>)dict).CopyTo(array, arrayIndex); + + bool ICollection>>.Remove(KeyValuePair> item) => ((IDictionary>)dict).Remove(item); + + public int Count => dict.Count; + public bool IsReadOnly => false; + + public bool ContainsKey(string key) => dict.ContainsKey(key); + + public bool Remove(string key) => dict.Remove(key); + + public bool TryGetValue(string key, out UniqueList value) + { + var result = dict.TryGetValue(key, out value); + if (!result) + dict[key] = value = new UniqueList(valueComparer); + return result; + } + + public UniqueList this[string key] + { + get + { + TryGetValue(key, out var value); + return value; + } + set => dict[key] = (value ?? new UniqueList(valueComparer)); + } + + public ICollection Keys => dict.Keys; + public ICollection> Values => dict.Values; + } +} \ No newline at end of file diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs index 59d72741..1222d61e 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs @@ -17,11 +17,13 @@ namespace CompatBot.Utils.ResultFormatters { internal static partial class LogParserResult { - private static async Task BuildNotesSectionAsync(DiscordEmbedBuilder builder, LogParseState state, NameValueCollection items, DiscordClient discordClient) + private static async Task BuildNotesSectionAsync(DiscordEmbedBuilder builder, LogParseState state, DiscordClient discordClient) { + var items = state.CompleteCollection; + var multiItems = state.CompleteMultiValueCollection; var notes = new List(); - var (irdChecked, brokenDump, longestPath) = await HasBrokenFilesAsync(items).ConfigureAwait(false); - brokenDump |= !string.IsNullOrEmpty(items["edat_block_offset"]); + var (irdChecked, brokenDump, longestPath) = await HasBrokenFilesAsync(state).ConfigureAwait(false); + brokenDump |= multiItems["edat_block_offset"].Any(); var elfBootPath = items["elf_boot_path"] ?? ""; var isEboot = !string.IsNullOrEmpty(elfBootPath) && elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); var isElf = !string.IsNullOrEmpty(elfBootPath) && !elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); @@ -74,12 +76,8 @@ namespace CompatBot.Utils.ResultFormatters notes.Add("❌ Failed to decrypt game content, license file might be corrupted"); if (items["failed_to_boot"] != null) notes.Add("❌ Failed to boot the game, the dump might be encrypted or corrupted"); - if (items["failed_to_verify"] is string verifyFails) - { - var types = verifyFails.Split(Environment.NewLine).Distinct().ToList(); - if (types.Contains("sce")) - notes.Add("❌ Failed to decrypt executables, PPU recompiler may crash or fail"); - } + if (multiItems["failed_to_verify"].Contains("sce")) + notes.Add("❌ Failed to decrypt executables, PPU recompiler may crash or fail"); if (brokenDump) notes.Add("❌ Some game files are missing or corrupted, please re-dump and validate."); if (items["fw_version_installed"] is string fw && !string.IsNullOrEmpty(fw)) @@ -491,8 +489,8 @@ namespace CompatBot.Utils.ResultFormatters if (state.Error == LogParseState.ErrorCode.SizeLimit) notes.Add("ℹ The log was too large, so only the last processed run is shown"); - BuildWeirdSettingsSection(builder, items, notes); - BuildMissingLicensesSection(builder, items, notes); + BuildWeirdSettingsSection(builder, state, notes); + BuildMissingLicensesSection(builder, serial, multiItems, notes); var notesContent = new StringBuilder(); foreach (var line in SortLines(notes, pirateEmoji)) @@ -500,12 +498,12 @@ namespace CompatBot.Utils.ResultFormatters PageSection(builder, notesContent.ToString().Trim(), "Notes"); } - private static void BuildMissingLicensesSection(DiscordEmbedBuilder builder, NameValueCollection items, List generalNotes) + private static void BuildMissingLicensesSection(DiscordEmbedBuilder builder, string serial, NameUniqueObjectCollection items, List generalNotes) { - if (items["rap_file"] is string rap) + if (items["rap_file"] is UniqueList raps && raps.Any()) { var limitTo = 5; - var licenseNames = rap.Split(Environment.NewLine) + var licenseNames = raps .Select(Path.GetFileName) .Distinct() .Except(KnownBogusLicenses) @@ -529,7 +527,7 @@ namespace CompatBot.Utils.ResultFormatters builder.AddField("Missing Licenses", content); - var gameRegion = items["serial"] is string serial && serial.Length > 3 ? new[] {serial[2]} : Enumerable.Empty(); + var gameRegion = serial?.Length > 3 ? new[] {serial[2]} : Enumerable.Empty(); var dlcRegions = licenseNames.Select(n => n[9]).Concat(gameRegion).Distinct().ToArray(); if (dlcRegions.Length > 1) generalNotes.Add($"🤔 That is a very interesting DLC collection from {dlcRegions.Length} different regions"); @@ -539,18 +537,21 @@ namespace CompatBot.Utils.ResultFormatters } } - private static async Task<(bool irdChecked, bool broken, int longestPath)> HasBrokenFilesAsync(NameValueCollection items) + private static async Task<(bool irdChecked, bool broken, int longestPath)> HasBrokenFilesAsync(LogParseState state) { + var items = state.CompleteCollection; + var multiItems = state.CompleteMultiValueCollection; var defaultLongestPath = "/PS3_GAME/USRDIR/".Length + (1+8+3)*2; // usually there's at least one more level for data files if (!(items["serial"] is string productCode)) return (false, false, defaultLongestPath); if (!productCode.StartsWith("B") && !productCode.StartsWith("M")) { - if (P5Ids.Contains(productCode) && items["broken_digital_filename"] is string brokenDigitalFiles) + if (P5Ids.Contains(productCode) + && multiItems["broken_digital_filename"] is UniqueList brokenDigitalFiles + && brokenDigitalFiles.Any()) { - var missingDigitalFiles = brokenDigitalFiles.Split(Environment.NewLine).Distinct().ToList(); - if (missingDigitalFiles.Contains("USRDIR/ps3.cpk") || missingDigitalFiles.Contains("USRDIR/data.cpk")) + if (brokenDigitalFiles.Contains("USRDIR/ps3.cpk") || brokenDigitalFiles.Contains("USRDIR/data.cpk")) return (false, true, defaultLongestPath); } return (false, false, defaultLongestPath); @@ -576,8 +577,8 @@ namespace CompatBot.Utils.ResultFormatters return (false, false, defaultLongestPath); var longestPath = knownFiles.Max(p => p.TrimEnd('.').Length); - if (string.IsNullOrEmpty(items["broken_directory"]) - && string.IsNullOrEmpty(items["broken_filename"])) + if (!multiItems["broken_directory"].Any() + && !multiItems["broken_filename"].Any()) return (false, false, longestPath); var missingDirs = items["broken_directory"]?.Split(Environment.NewLine).Distinct().ToList() ?? diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs index 08d8c7e7..abf419d6 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs @@ -6,14 +6,17 @@ using System.Linq; using System.Text; using System.Text.RegularExpressions; using CompatApiClient.Utils; +using CompatBot.EventHandlers.LogParsing.POCOs; using DSharpPlus.Entities; namespace CompatBot.Utils.ResultFormatters { internal static partial class LogParserResult { - private static void BuildWeirdSettingsSection(DiscordEmbedBuilder builder, NameValueCollection items, List generalNotes) + private static void BuildWeirdSettingsSection(DiscordEmbedBuilder builder, LogParseState state, List generalNotes) { + var items = state.CompleteCollection; + var multiItems = state.CompleteMultiValueCollection; var notes = new List(); var serial = items["serial"] ?? ""; int.TryParse(items["thread_count"], out var threadCount); @@ -136,18 +139,14 @@ namespace CompatBot.Utils.ResultFormatters if (KnownDisableVertexCacheIds.Contains(serial) && !vertexCacheDisabled) notes.Add("⚠ This game requires disabling `Vertex Cache` option"); - if (items["rsx_not_supported"] is string notSupported) + if (multiItems["rsx_not_supported"].Contains("alpha-to-one for multisampling")) { - var rsxCaveats = notSupported.Split(Environment.NewLine).Distinct().ToArray(); - if (rsxCaveats.Contains("alpha-to-one for multisampling")) - { - if (items["msaa"] is string msaa && msaa != "Disabled") - generalNotes.Add("ℹ The driver or GPU do not support all required features for proper MSAA implementation, which may result in minor visual artifacts"); - } + if (items["msaa"] is string msaa && msaa != "Disabled") + generalNotes.Add("ℹ The driver or GPU do not support all required features for proper MSAA implementation, which may result in minor visual artifacts"); } var isWireframeBugPossible = items["gpu_info"] is string gpuInfo - && Regex.IsMatch(gpuInfo, @"Radeon RX 5\d{3}", RegexOptions.IgnoreCase) // RX 590 is a thing 😔 - && !gpuInfo.Contains("RADV"); + && Regex.IsMatch(gpuInfo, @"Radeon RX 5\d{3}", RegexOptions.IgnoreCase) // RX 590 is a thing 😔 + && !gpuInfo.Contains("RADV"); if (items["msaa"] == "Disabled") { if (!isWireframeBugPossible) @@ -292,7 +291,7 @@ namespace CompatBot.Utils.ResultFormatters { CheckP5Settings(serial, items, notes, ppuPatches); CheckAsurasWrathSettings(serial, items, notes); - CheckJojoSettings(serial, items, notes, ppuPatches, ppuHashes, generalNotes); + CheckJojoSettings(serial, state, notes, ppuPatches, ppuHashes, generalNotes); CheckSimpsonsSettings(serial, generalNotes); CheckNierSettings(serial, items, notes, ppuPatches, ppuHashes, generalNotes); CheckDod3Settings(serial, items, notes, ppuPatches, ppuHashes, generalNotes); @@ -512,8 +511,9 @@ namespace CompatBot.Utils.ResultFormatters "18cf9a4e8196684ed9ee816f82649561fd1bf182", }; - private static void CheckJojoSettings(string serial, NameValueCollection items, List notes, Dictionary ppuPatches, HashSet ppuHashes, List generalNotes) + private static void CheckJojoSettings(string serial, LogParseState state, List notes, Dictionary ppuPatches, HashSet ppuHashes, List generalNotes) { + var items = state.CompleteCollection; if (AllStarBattleIds.Contains(serial) || serial == "BLJS10318" || serial == "NPJB00753") { if (items["audio_buffering"] == EnabledMark && items["audio_buffer_duration"] != "20") @@ -549,7 +549,7 @@ namespace CompatBot.Utils.ResultFormatters if (serial == "BLUS31405" && items["compat_database_path"] is string compatDbPath && compatDbPath.Contains("JoJo ASB Emulator v.04") - && !string.IsNullOrEmpty(items["rap_file"])) + && state.CompleteMultiValueCollection["rap_file"].Any()) generalNotes.Add("🤔 Very interesting version of the game you got there"); } } diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs index 9db35bf6..02ff822f 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs @@ -204,7 +204,11 @@ namespace CompatBot.Utils.ResultFormatters public static async Task AsEmbedAsync(this LogParseState state, DiscordClient client, DiscordMessage message, ISource source) { DiscordEmbedBuilder builder; - var collection = state.CompleteCollection ?? state.WipCollection; + state.CompleteCollection ??= state.WipCollection; + state.CompleteMultiValueCollection ??= state.WipMultiValueCollection; + var collection = state.CompleteCollection; + var multiValueCollection = state.CompleteMultiValueCollection; + if (collection?.Count > 0) { var ldrGameSerial = collection["ldr_game_serial"] ?? collection["ldr_path_serial"]; @@ -242,13 +246,13 @@ namespace CompatBot.Utils.ResultFormatters } else { - CleanupValues(collection); + CleanupValues(state); BuildInfoSection(builder, collection); var colA = BuildCpuSection(collection); var colB = BuildGpuSection(collection); BuildSettingsSections(builder, collection, colA, colB); BuildLibsSection(builder, collection); - await BuildNotesSectionAsync(builder, state, collection, client).ConfigureAwait(false); + await BuildNotesSectionAsync(builder, state, client).ConfigureAwait(false); } } else @@ -263,8 +267,10 @@ namespace CompatBot.Utils.ResultFormatters return builder; } - private static void CleanupValues(NameValueCollection items) + private static void CleanupValues(LogParseState state) { + var items = state.CompleteCollection; + var multiItems = state.CompleteMultiValueCollection; if (items["strict_rendering_mode"] == "true") items["resolution_scale"] = "Strict Mode"; if (items["spu_threads"] == "0") @@ -284,19 +290,19 @@ namespace CompatBot.Utils.ResultFormatters if (!string.IsNullOrEmpty(items["gpu_info"])) { items["gpu_info"] = items["gpu_info"].StripMarks(); - items["driver_version_info"] = GetVulkanDriverVersion(items["vulkan_initialized_device"], items["vulkan_found_device"]) ?? - GetVulkanDriverVersion(items["gpu_info"], items["vulkan_found_device"]) ?? + items["driver_version_info"] = GetVulkanDriverVersion(items["vulkan_initialized_device"], multiItems["vulkan_found_device"]) ?? + GetVulkanDriverVersion(items["gpu_info"], multiItems["vulkan_found_device"]) ?? GetOpenglDriverVersion(items["gpu_info"], items["driver_version_new"] ?? items["driver_version"]) ?? GetVulkanDriverVersionRaw(items["gpu_info"], items["vulkan_driver_version_raw"]); } if (items["driver_version_info"] != null) items["gpu_info"] += $" ({items["driver_version_info"]})"; - if (items["vulkan_compatible_device_name"] is string vulkanDevices) + if (multiItems["vulkan_compatible_device_name"] is UniqueList vulkanDevices && vulkanDevices.Any()) { - var devices = vulkanDevices.Split(Environment.NewLine) + var devices = vulkanDevices .Distinct() - .Select(n => new {name = n.StripMarks(), driverVersion = GetVulkanDriverVersion(n, items["vulkan_found_device"])}) + .Select(n => new {name = n.StripMarks(), driverVersion = GetVulkanDriverVersion(n, multiItems["vulkan_found_device"])}) .Reverse() .ToList(); if (string.IsNullOrEmpty(items["gpu_info"]) && devices.Count > 0) @@ -472,12 +478,12 @@ namespace CompatBot.Utils.ResultFormatters return version; } - private static string GetVulkanDriverVersion(string gpu, string foundDevices) + private static string GetVulkanDriverVersion(string gpu, UniqueList foundDevices) { - if (string.IsNullOrEmpty(gpu) || string.IsNullOrEmpty(foundDevices)) + if (string.IsNullOrEmpty(gpu) || !foundDevices.Any()) return null; - var info = (from line in foundDevices.Split(Environment.NewLine) + var info = (from line in foundDevices let m = VulkanDeviceInfo.Match(line) where m.Success select m diff --git a/CompatBot/Utils/UniqueList.cs b/CompatBot/Utils/UniqueList.cs index fd68033d..a8cff43b 100644 --- a/CompatBot/Utils/UniqueList.cs +++ b/CompatBot/Utils/UniqueList.cs @@ -48,6 +48,12 @@ namespace CompatBot.Utils list.Add(item); } + public void AddRange(IEnumerable collection) + { + foreach (var item in collection) + Add(item); + } + public void Clear() { list.Clear(); From 8e94a94a82b2e66520517b36e14d3fb5028cff18 Mon Sep 17 00:00:00 2001 From: 13xforever Date: Mon, 2 Mar 2020 16:57:04 +0500 Subject: [PATCH 8/8] new patch extraction --- .../LogParsing/LogParser.LogSections.cs | 17 ++++++-------- ...rserResultFormatter.GeneralNotesSection.cs | 16 +++++++------- ...serResultFormatter.WeirdSettingsSection.cs | 5 +++-- .../LogParserResultFormatter.cs | 22 +++++++------------ 4 files changed, 26 insertions(+), 34 deletions(-) diff --git a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs index 478fe182..4c116241 100644 --- a/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs +++ b/CompatBot/EventHandlers/LogParsing/LogParser.LogSections.cs @@ -200,10 +200,10 @@ namespace CompatBot.EventHandlers.LogParsing ["XAudio2Thread"] = new Regex(@"XAudio2Thread\s*: (?.+failed\s*\((?0x.+)\).*)\r?$", DefaultOptions), ["cellAudio Thread"] = new Regex(@"XAudio2Backend\s*: (?.+failed\s*\((?0x.+)\).*)\r?$", DefaultOptions), ["using a Null renderer instead"] = new Regex(@"Audio renderer (?.+) could not be initialized\r?$", DefaultOptions), - ["PPU executable hash:"] = new Regex(@"PPU executable hash: PPU-(?\w+) \(<-\s*(?\d+)\).*?\r?$", DefaultOptions), - ["OVL executable hash:"] = new Regex(@"OVL executable hash: OVL-(?\w+) \(<-\s*(?\d+)\).*?\r?$", DefaultOptions), - ["SPU executable hash:"] = new Regex(@"SPU executable hash: SPU-(?\w+) \(<-\s*(?\d+)\).*?\r?$", DefaultOptions), - ["Loaded SPU image:"] = new Regex(@"Loaded SPU image: SPU-(?\w+) \(<-\s*(?\d+)\).*?\r?$", DefaultOptions), + ["PPU executable hash:"] = new Regex(@"PPU executable hash: PPU-(?\w+ \(<-\s*\d+\)).*?\r?$", DefaultOptions), + ["OVL executable hash:"] = new Regex(@"OVL executable hash: OVL-(?\w+ \(<-\s*\d+\)).*?\r?$", DefaultOptions), + ["SPU executable hash:"] = new Regex(@"SPU executable hash: SPU-(?\w+ \(<-\s*\d+\)).*?\r?$", DefaultOptions), + ["Loaded SPU image:"] = new Regex(@"Loaded SPU image: SPU-(?\w+ \(<-\s*\d+\)).*?\r?$", DefaultOptions), ["'sys_fs_open' failed"] = new Regex(@"'sys_fs_open' failed (?!with 0x8001002c).+\xE2\x80\x9C(/dev_bdvd/(?.+)|/dev_hdd0/game/NP\w+/(?.+))\xE2\x80\x9D.*?\r?$", DefaultOptions), ["'sys_fs_opendir' failed"] = new Regex(@"'sys_fs_opendir' failed .+\xE2\x80\x9C/dev_bdvd/(?.+)\xE2\x80\x9D.*?\r?$", DefaultOptions), ["EDAT: "] = new Regex(@"EDAT: Block at offset (?0x[0-9a-f]+) has invalid hash!.*?\r?$", DefaultOptions), @@ -226,12 +226,9 @@ namespace CompatBot.EventHandlers.LogParsing "rap_file", "vulkan_found_device", "vulkan_compatible_device_name", - "ppu_hash", - "ppu_hash_patch", - "ovl_hash", - "ovl_hash_patch", - "spu_hash", - "spu_hash_patch", + "ppu_patch", + "ovl_patch", + "spu_patch", "broken_filename", "broken_digital_filename", "broken_directory", diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs index 1222d61e..e39d6a87 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.GeneralNotesSection.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Collections.Specialized; using System.IO; using System.Linq; using System.Text; @@ -22,7 +21,7 @@ namespace CompatBot.Utils.ResultFormatters var items = state.CompleteCollection; var multiItems = state.CompleteMultiValueCollection; var notes = new List(); - var (irdChecked, brokenDump, longestPath) = await HasBrokenFilesAsync(state).ConfigureAwait(false); + var (_, brokenDump, longestPath) = await HasBrokenFilesAsync(state).ConfigureAwait(false); brokenDump |= multiItems["edat_block_offset"].Any(); var elfBootPath = items["elf_boot_path"] ?? ""; var isEboot = !string.IsNullOrEmpty(elfBootPath) && elfBootPath.EndsWith("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase); @@ -348,9 +347,9 @@ namespace CompatBot.Utils.ResultFormatters if (!string.IsNullOrEmpty(items["patch_error_file"])) notes.Add($"⚠ Failed to load `patch.yml`, check syntax around line {items["patch_error_line"]} column {items["patch_error_column"]}"); - var ppuPatches = GetPatches(items["ppu_hash"], items["ppu_hash_patch"], true); - var ovlPatches = GetPatches(items["ovl_hash"], items["ovl_hash_patch"], true); - var allSpuPatches = GetPatches(items["spu_hash"], items["spu_hash_patch"], false); + var ppuPatches = GetPatches(multiItems["ppu_patch"], true); + var ovlPatches = GetPatches(multiItems["ovl_patch"], true); + var allSpuPatches = GetPatches(multiItems["spu_patch"], false); var spuPatches = new Dictionary(allSpuPatches.Where(kvp => kvp.Value != 0)); if (ppuPatches.Any() || spuPatches.Any() || ovlPatches.Any()) { @@ -474,9 +473,10 @@ namespace CompatBot.Utils.ResultFormatters msg += $" (update available: v{gameUpVer})"; notes.Add(msg); } - if (items["ppu_hash"] is string ppuHashes - && ppuHashes.Split(Environment.NewLine, 2, StringSplitOptions.RemoveEmptyEntries).FirstOrDefault() is string firstPpuHash - && !string.IsNullOrEmpty(firstPpuHash)) + if (multiItems["ppu_patch"].FirstOrDefault() is string firstPpuPatch + && ProgramHashPatch.Match(firstPpuPatch) is Match m + && m.Success + && m.Groups["hash"].Value is string firstPpuHash) { var exe = Path.GetFileName(items["elf_boot_path"] ?? ""); if (string.IsNullOrEmpty(exe) || exe.Equals("EBOOT.BIN", StringComparison.InvariantCultureIgnoreCase)) diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs index abf419d6..16e5f0d4 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.WeirdSettingsSection.cs @@ -233,8 +233,9 @@ namespace CompatBot.Utils.ResultFormatters notes.Add("⚠ `Resolution Scaling Threshold` below `16x16` may result in corrupted visuals and game crash"); } } - var ppuPatches = GetPatches(items["ppu_hash"], items["ppu_hash_patch"], true); - var ppuHashes = GetHashes(items["ppu_hash"]); + var allPpuHashes = GetPatches(multiItems["ppu_patch"], false); + var ppuPatches = allPpuHashes.Where(kvp => kvp.Value > 0).ToDictionary(kvp => kvp.Key, kvp => kvp.Value); + var ppuHashes = new HashSet(allPpuHashes.Keys, StringComparer.InvariantCultureIgnoreCase); if (items["write_color_buffers"] == DisabledMark && !string.IsNullOrEmpty(serial) && KnownWriteColorBuffersIds.Contains(serial)) diff --git a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs index 02ff822f..285ff533 100644 --- a/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs +++ b/CompatBot/Utils/ResultFormatters/LogParserResultFormatter.cs @@ -38,6 +38,7 @@ namespace CompatBot.Utils.ResultFormatters @"Operating system: (?[^,]+), (Name: (?[^,]+), Release: (?[^,]+), Version: (?[^\r\n]+)|Major: (?\d+), Minor: (?\d+), Build: (?\d+), Service Pack: (?[^,]+), Compatibility mode: (?[^,\r\n]+))\r?$", DefaultSingleLine); private static readonly Regex LinuxKernelVersion = new Regex(@"(?\d+\.\d+\.\d+(-\d+)?)", DefaultSingleLine); + private static readonly Regex ProgramHashPatch = new Regex(@"(?\w+) \(<-\s*(?\d+)\)", DefaultSingleLine); private static readonly char[] NewLineChars = {'\r', '\n'}; // rpcs3-v0.0.5-7105-064d0619_win64.7z @@ -704,25 +705,18 @@ namespace CompatBot.Utils.ResultFormatters .ToList(); } - private static Dictionary GetPatches(string hashList, string patchesList, in bool onlyApplied) + private static Dictionary GetPatches(UniqueList patchList, in bool onlyApplied) { - if (string.IsNullOrEmpty(hashList) || string.IsNullOrEmpty(patchesList)) + if (!patchList.Any()) return new Dictionary(0); - var hashes = hashList.Split(Environment.NewLine); - var patches = patchesList.Split(Environment.NewLine); - if (hashes.Length != patches.Length) + var result = new Dictionary(patchList.Count); + foreach (var patch in patchList) { - Config.Log.Warn($"Hashes count: {hashes.Length}, Patches count: {patches.Length}"); - return new Dictionary(0); - } - - var result = new Dictionary(); - for (var i = 0; i < hashes.Length; i++) - { - if (int.TryParse(patches[i], out var pCount)) + var match = ProgramHashPatch.Match(patch); + if (match.Success && int.TryParse(match.Groups["patch_count"].Value, out var pCount)) if (!onlyApplied || pCount > 0) - result[hashes[i]] = pCount; + result[match.Groups["hash"].Value] = pCount; } return result; }