From e802bc947f5110cde14928c0c3818e161db2be71 Mon Sep 17 00:00:00 2001 From: itismadness Date: Mon, 13 Oct 2025 23:02:58 +0000 Subject: [PATCH] Lower threshold for using chardet encoding --- src/Util.php | 45 ++++----- tests/logs/eac/details/ru_2.json | 19 ++++ tests/logs/eac/html/ru_2.log | 152 +++++++++++++++++++++++++++++ tests/logs/eac/originals/ru_2.log | 153 ++++++++++++++++++++++++++++++ 4 files changed, 347 insertions(+), 22 deletions(-) create mode 100644 tests/logs/eac/details/ru_2.json create mode 100644 tests/logs/eac/html/ru_2.log create mode 100644 tests/logs/eac/originals/ru_2.log diff --git a/src/Util.php b/src/Util.php index 47e2fe2..8264572 100644 --- a/src/Util.php +++ b/src/Util.php @@ -19,7 +19,6 @@ class Util } catch (\RuntimeException $exc) { $chardet = null; } - /** @var Chardet $chardet */ // Whipper uses UTF-8 so we don't need to bother checking, especially as it's // possible a log may be falsely detected as a different encoding by chardet @@ -38,29 +37,31 @@ class Util $log = substr($log, 3); } elseif ($chardet !== null) { $results = $chardet->analyze($logPath); - if ($results['charset'] !== 'utf-8' && $results['confidence'] > 0.7) { - // ideally we'd be able to use mb_convert_encoding as it has more widespread - // support vs iconv, but it does not support as many encodings as iconv, such - // as Windows-1250 (along with others) which is a common encoding for range logs - // coming from rutracker - // $log = mb_convert_encoding($log, 'UTF-8', $results['charset']); - $tmp = @iconv($results['charset'], 'UTF-8', $log); + if ($results['charset'] !== 'utf-8') { + if ($results['confidence'] > 0.6) { + // ideally we'd be able to use mb_convert_encoding as it has more widespread + // support vs iconv, but it does not support as many encodings as iconv, such + // as Windows-1250 (along with others) which is a common encoding for range logs + // coming from rutracker + // $log = mb_convert_encoding($log, 'UTF-8', $results['charset']); + $tmp = @iconv($results['charset'], 'UTF-8', $log); - // depending on your iconv version, some encodings may be represented - // with prefix of mac-* or mac (like maccentraleurope vs mac-centraleurope) - if ($tmp === false && substr($results['charset'], 0, 3) === 'mac') { - $tmp = @iconv('mac-' . substr($results['charset'], 3), 'UTF-8', $log); + // depending on your iconv version, some encodings may be represented + // with prefix of mac-* or mac (like maccentraleurope vs mac-centraleurope) + if ($tmp === false && substr($results['charset'], 0, 3) === 'mac') { + $tmp = @iconv('mac-' . substr($results['charset'], 3), 'UTF-8', $log); + } + $log = $tmp; + if ($log === false) { + throw new \RuntimeException('Could not properly decode log encoding'); + } + } elseif ($results['charset'] !== 'utf-8' && $results['confidence'] > 0.3) { + // If we've got a poor confidence on our decoding, we just use a generic + // ISO-8859-1 as that covers a decent range of things that people would + // inadvertently re-encode a log into. I seriously cannot express how + // much I hate how EAC does not use always UTF-8. + $log = iconv('ISO-8859-1', 'UTF-8', $log); } - $log = $tmp; - if ($log === false) { - throw new \RuntimeException('Could not properly decode log encoding'); - } - } elseif ($results['charset'] !== 'utf-8' && $results['confidence'] > 0.3) { - // If we've got a poor confidence on our decoding, we just use a generic - // ISO-8859-1 as that covers a decent range of things that people would - // inadvertently re-encode a log into. I seriously cannot express how - // much I hate how EAC does not use always UTF-8. - $log = iconv('ISO-8859-1', 'UTF-8', $log); } } diff --git a/tests/logs/eac/details/ru_2.json b/tests/logs/eac/details/ru_2.json new file mode 100644 index 0000000..c94bbd7 --- /dev/null +++ b/tests/logs/eac/details/ru_2.json @@ -0,0 +1,19 @@ +{ + "ripper": "EAC", + "version": "0.99 prebeta 1", + "language": "ru", + "combined": false, + "score": 55, + "checksum": "checksum_missing", + "details": [ + "[Notice] Translated log from \u0420\u0443\u0441\u0441\u043a\u0438\u0439 (Russian) to English.", + "\"Defeat audio cache\" should be yes (-10 points)", + "C2 pointers were used (-10 points)", + "Could not verify read offset (-1 point)", + "Combined read\/write offset cannot be verified (-4 points)", + "Could not verify null samples", + "Gap handling was not detected (-10 points)", + "Could not verify id3 tag setting", + "Test and copy was not used (-10 points)" + ] +} \ No newline at end of file diff --git a/tests/logs/eac/html/ru_2.log b/tests/logs/eac/html/ru_2.log new file mode 100644 index 0000000..be528a1 --- /dev/null +++ b/tests/logs/eac/html/ru_2.log @@ -0,0 +1,152 @@ +Exact Audio Copy V0.99 prebeta 1 from 25. May 2007 + +EAC extraction logfile from 10. March 2010, 12:34 + +Unknown Artist / Unknown Title + +Used Drive : _NEC DVD_RW ND-4571A Adapter: 0 ID: 0 + +Read mode : Secure +Utilize accurate stream : Yes +Defeat audio cache : No +Make use of C2 pointers : Yes + +Combined read/write offset correction : 0 +Overread into Lead-In and Lead-Out : No +Fill up missing offset samples with silence : Yes +Delete leading and trailing silent blocks : No +Used interface : Native Win32 interface for Win NT & 2000 +Gap handling : Not detected, thus appended to previous track + +Used output format : Internal WAV Routines +Sample format : 44.100 Hz; 16 Bit; Stereo + + +TOC of the extracted CD + + Track | Start | Length | Start sector | End sector + --------------------------------------------------------------------- + 1 | 0:00.00 | 6:54.17 | 0 | 31066 + 2 | 6:54.17 | 5:33.72 | 31067 | 56113 + 3 | 12:28.14 | 3:46.71 | 56114 | 73134 + 4 | 16:15.10 | 6:46.27 | 73135 | 103611 + 5 | 23:01.37 | 6:05.59 | 103612 | 131045 + 6 | 29:07.21 | 6:06.74 | 131046 | 158569 + 7 | 35:14.20 | 4:10.03 | 158570 | 177322 + 8 | 39:24.23 | 4:15.52 | 177323 | 196499 + 9 | 43:40.00 | 7:47.55 | 196500 | 231579 + 10 | 51:27.55 | 5:41.56 | 231580 | 257210 + 11 | 57:09.36 | 7:47.53 | 257211 | 292288 + 12 | 64:57.14 | 10:59.02 | 292289 | 341715 + + +Track 1 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track01.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC 37374B7A + Copy OK + +Track 2 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track02.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC D7016D6A + Copy OK + +Track 3 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track03.wav + + Peak level 99.9 % + Track quality 99.9 % + Copy CRC 522AB415 + Copy OK + +Track 4 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track04.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC 56E839AA + Copy OK + +Track 5 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track05.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC 5BB3F2A7 + Copy OK + +Track 6 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track06.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC C3D0197E + Copy OK + +Track 7 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track07.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC F9D68CEA + Copy OK + +Track 8 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track08.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC CD24B722 + Copy OK + +Track 9 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track09.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC DC1D7CA3 + Copy OK + +Track 10 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track10.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC E736E801 + Copy OK + +Track 11 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track11.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC DEDA10BB + Copy OK + +Track 12 + + Filename C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Copeâ„¢ Remixed [MAPACDA010]\Disc 1\Track12.wav + + Peak level 99.9 % + Track quality 100.0 % + Copy CRC 8E4C6F91 + Copy OK + +No errors occurred +End of status report \ No newline at end of file diff --git a/tests/logs/eac/originals/ru_2.log b/tests/logs/eac/originals/ru_2.log new file mode 100644 index 0000000..921a02e --- /dev/null +++ b/tests/logs/eac/originals/ru_2.log @@ -0,0 +1,153 @@ +Exact Audio Copy V0.99 prebeta 1 from 25. May 2007 + +Îò÷¸ò EAC îá èçâëå÷åíèè, âûïîëíåííîì 10. ìàðòà 2010, 12:34 + +Unknown Artist / Unknown Title + +Äèñêîâîä: _NEC DVD_RW ND-4571A Adapter: 0 ID: 0 + +Ðåæèì ÷òåíèÿ : Äîñòîâåðíîñòü +Èñïîëüçîâàíèå òî÷íîãî ïîòîêà : Äà +Îòêëþ÷åíèå êýøà àóäèî : Íåò +Èñïîëüçîâàíèå óêàçàòåëåé C2 : Äà + +Êîìáèíèðîâàííîå ñìåùåíèå ÷òåíèÿ/çàïèñè : 0 +Ñïîñîáíîñòü ÷èòàòü îáëàñòè Lead-in è Lead-out : Íåò +Çàïîëíåíèå ïðîïóùåííûõ ñýìïëîâ òèøèíîé : Äà +Óäàëåíèå áëîêîâ ñ òèøèíîé â íà÷àëå è êîíöå : Íåò +Èíòåðôåéñ : Âñòðîåííûé Win32-èíòåðôåéñ äëÿ Win NT/2000 +Îáðàáîòêà çàçîðîâ : Íå îáíàðóæåíî, äîáàâëåíî ê ïðåäûäóùåìó òðåêó + +Âûõîäíîé ôîðìàò : Âíóòðåííèå WAV-îïåðàöèè +Ôîðìàò ñýìïëîâ : 44.100 Ãö; 16 áèò; ñòåðåî + + +TOC èçâëå÷¸ííîãî CD + + Òðåê | Ñòàðò | Äëèòåëüíîñòü | Íà÷àëüíûé ñåêòîð | Êîíå÷íûé ñåêòîð + --------------------------------------------------------------------- + 1 | 0:00.00 | 6:54.17 | 0 | 31066 + 2 | 6:54.17 | 5:33.72 | 31067 | 56113 + 3 | 12:28.14 | 3:46.71 | 56114 | 73134 + 4 | 16:15.10 | 6:46.27 | 73135 | 103611 + 5 | 23:01.37 | 6:05.59 | 103612 | 131045 + 6 | 29:07.21 | 6:06.74 | 131046 | 158569 + 7 | 35:14.20 | 4:10.03 | 158570 | 177322 + 8 | 39:24.23 | 4:15.52 | 177323 | 196499 + 9 | 43:40.00 | 7:47.55 | 196500 | 231579 + 10 | 51:27.55 | 5:41.56 | 231580 | 257210 + 11 | 57:09.36 | 7:47.53 | 257211 | 292288 + 12 | 64:57.14 | 10:59.02 | 292289 | 341715 + + +Òðåê 1 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track01.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè 37374B7A + Êîïèðîâàíèå... OK + +Òðåê 2 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track02.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè D7016D6A + Êîïèðîâàíèå... OK + +Òðåê 3 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track03.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 99.9 % + CRC êîïèè 522AB415 + Êîïèðîâàíèå... OK + +Òðåê 4 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track04.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè 56E839AA + Êîïèðîâàíèå... OK + +Òðåê 5 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track05.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè 5BB3F2A7 + Êîïèðîâàíèå... OK + +Òðåê 6 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track06.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè C3D0197E + Êîïèðîâàíèå... OK + +Òðåê 7 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track07.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè F9D68CEA + Êîïèðîâàíèå... OK + +Òðåê 8 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track08.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè CD24B722 + Êîïèðîâàíèå... OK + +Òðåê 9 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track09.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè DC1D7CA3 + Êîïèðîâàíèå... OK + +Òðåê 10 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track10.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè E736E801 + Êîïèðîâàíèå... OK + +Òðåê 11 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track11.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè DEDA10BB + Êîïèðîâàíèå... OK + +Òðåê 12 + + Èìÿ ôàéëà C:\Users\BeatRokazZ\Desktop\[upload]\Freeland [2010] Cope™ Remixed [MAPACDA010]\Disc 1\Track12.wav + + Ïèêîâûé óðîâåíü 99.9 % + Êà÷åñòâî òðåêà 100.0 % + CRC êîïèè 8E4C6F91 + Êîïèðîâàíèå... OK + +Îøèáîê íå ïðîèçîøëî + +Êîíåö îò÷¸òà