PICARD-2716: Accept encodings other than utf-8 when opening CD extraction logs

# Conflicts:
#	requirements-macos-11.txt

picard/disc/dbpoweramplog.py

@@ -26,7 +26,7 @@ from picard.disc.utils import (
     TocEntry,
     calculate_mb_toc_numbers,
 )
-from picard.util import detect_unicode_encoding
+from picard.util import detect_file_encoding
 
 
 RE_TOC_ENTRY = re.compile(
@@ -50,6 +50,6 @@ def filter_toc_entries(lines):
 
 def toc_from_file(path):
     """Reads dBpoweramp log files, generates MusicBrainz disc TOC listing for use as discid."""
-    encoding = detect_unicode_encoding(path)
+    encoding = detect_file_encoding(path)
     with open(path, 'r', encoding=encoding) as f:
         return calculate_mb_toc_numbers(filter_toc_entries(f))

picard/disc/eaclog.py

@@ -31,7 +31,7 @@ from picard.disc.utils import (
     TocEntry,
     calculate_mb_toc_numbers,
 )
-from picard.util import detect_unicode_encoding
+from picard.util import detect_file_encoding
 
 
 RE_TOC_TABLE_HEADER = re.compile(r""" \s*
@@ -82,6 +82,6 @@ def toc_from_file(path):
 
     Warning: may work wrong for discs having data tracks. May generate wrong
     results on other non-standard cases."""
-    encoding = detect_unicode_encoding(path)
+    encoding = detect_file_encoding(path)
     with open(path, 'r', encoding=encoding) as f:
         return calculate_mb_toc_numbers(filter_toc_entries(f))

picard/util/__init__.py

@@ -40,7 +40,13 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 
-
+try:
+    from charset_normalizer import detect
+except ImportError:
+    try:
+        from chardet import detect
+    except ImportError:
+        detect = None
 from collections import (
     defaultdict,
     namedtuple,
@@ -1186,13 +1192,16 @@ ENCODING_BOMS = {
 }
 
 
-def detect_unicode_encoding(path):
+def detect_file_encoding(path, max_bytes_to_read=1024*256):
-    """Attempts to guess the unicode encoding of a file based on the BOM.
+    """Attempts to guess the unicode encoding of a file based on the BOM, and
+    depending on avalibility, using a charset detection method.
 
-    Assumes UTF-8 by default if there is no BOM.
+    Assumes UTF-8 by default if no other encoding is detected.
 
     Args:
         path: The path to the file
+        max_bytes_to_read: Maximum bytes to read from the file during encoding
+        detection.
 
     Returns: The encoding as a string, e.g. "utf-16-le" or "utf-8"
     """
@@ -1201,4 +1210,14 @@ def detect_unicode_encoding(path):
         for bom, encoding in ENCODING_BOMS.items():
             if first_bytes.startswith(bom):
                 return encoding
-        return 'utf-8'
+
+        if detect is None:
+            return 'utf-8'
+
+        f.seek(0)
+        result = detect(f.read(max_bytes_to_read))
+        if result['encoding'] is None:
+            log.warning("Couldn't detect encoding for file %r", path)
+            result['encoding'] = 'UTF-8'
+
+        return result['encoding'].lower()

requirements-macos-10.12.txt

@@ -8,3 +8,4 @@ pyobjc-framework-Cocoa==9.0.1
 PyQt5==5.13.1
 python-dateutil==2.8.2
 PyYAML==6.0.1
+charset-normalizer==3.3.2

requirements-macos-10.14.txt

@@ -8,3 +8,4 @@ pyobjc-framework-Cocoa==9.1.1
 PyQt5==5.15.10
 python-dateutil==2.8.2
 PyYAML==6.0.1
+charset-normalizer==3.3.2

requirements-win.txt

@@ -7,3 +7,4 @@ PyQt5==5.15.10
 python-dateutil==2.8.2
 pywin32==306
 PyYAML==6.0.1
+charset-normalizer==3.3.2

requirements.txt

@@ -10,3 +10,4 @@ PyQt5~=5.11
 python-dateutil~=2.7
 pywin32; sys_platform == 'win32'
 PyYAML>=5.1, <7
+charset-normalizer~=3.3.2

test/data/eac-windows1251.log

@@ -0,0 +1,143 @@
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20>1.6 <20><><EFBFBD><EFBFBD> 23. <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2020
+
+<EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 24. <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2022, 21:33
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>q<EFBFBD><EFBFBD> / <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>  : <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><>-218<31><38>   <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>: 1  <20><>: 1
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> : <20><><EFBFBD><EFBFBD><EFBFBD>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>                      : 6
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD>-<2D><> <20><><EFBFBD> <20><><EFBFBD><EFBFBD>-<2D><><EFBFBD>          : <20><>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> : <20><><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>   : <20><>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20><> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>       : <20><><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>                              : <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD>32 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD> <20><> & 2000
+<EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>                                : <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>, <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>              : <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>                : 768 <20><><EFBFBD><EFBFBD>/<2F>
+Q<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>                         : <20><><EFBFBD><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD> <20><>3 <20><><EFBFBD>                     : <20><>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>         : <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> (<28>86)\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> : -6 -<2D> -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1072;<3B><><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD>=#1090;<3B><><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD>=#1072;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD>=#1081;<3B><><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1090;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD>=#1075;<3B><><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1094;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD>=#1072;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1072;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1094;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" %<25><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%--<2D><><EFBFBD>-<2D><><EFBFBD><EFBFBD>-<2D><><EFBFBD><EFBFBD>=1051;<3B><><EFBFBD><EFBFBD><EFBFBD>=<3D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%"%<25><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>% -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1094;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1090;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" -<2D> "<22><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=#1085;<3B><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%" %<25><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%--<2D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>=<3D><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>%"%<25><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>% %<25><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>% -<2D> %<25><><EFBFBD><EFBFBD>%
+
+
+<EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><>
+
+     <20><><EFBFBD><EFBFBD><EFBFBD> |   <20><><EFBFBD><EFBFBD><EFBFBD>  |  <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  | <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> | <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+    ---------------------------------------------------------
+        1  |  0:00.00 |  5:32.14 |         0    |    24913
+        2  |  5:32.14 |  4:07.22 |     24914    |    43460
+        3  |  9:39.36 |  3:50.29 |     43461    |    60739
+        4  | 13:29.65 |  4:56.00 |     60740    |    82939
+        5  | 18:25.65 |  3:45.35 |     82940    |    99849
+        6  | 22:11.25 |  3:20.57 |     99850    |   114906
+        7  | 25:32.07 |  4:33.26 |    114907    |   135407
+        8  | 30:05.33 |  3:03.40 |    135408    |   149172
+
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  1
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\01 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 94.8 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 9.0 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> <20>924<32><34>20
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [3182EB6A]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  2
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\02 <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 98.1 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 12.0 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> <20>19<31>6<EFBFBD><36><EFBFBD>
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [79A374E7]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  3
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\03 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><> <20><> <20><><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 97.2 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 13.0 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> 300<30><30>15<31>
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [77DE3AFE]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  4
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\04 <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 98.8 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 14.0 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> 1657126<32>
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [4D824E2F]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  5
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\05 <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 92.8 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 14.9 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> 66<36><36><EFBFBD><EFBFBD>59
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [D0D39055]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  6
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\06 <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 96.5 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 15.6 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> 5<>7<EFBFBD>712<31>
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [CD69B7FC]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  7
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\07 <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 97.5 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 16.4 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> 694<39><34><EFBFBD>79
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [FE6DBCBF]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>  8
+
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>:\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD>\<5C><><EFBFBD>\<5C><><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD>\08 <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>.<2E><><EFBFBD>
+
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD> 98.4 %
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD> 17.0 <20>
+     <20><><EFBFBD><EFBFBD> <20><><EFBFBD> 629<32><39>273
+     <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> (<28><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> 2)  [7EB7F715]  (<28><> <20>2)
+     <20><><EFBFBD><EFBFBD> <20><>
+
+
+<EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+
+<EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+
+<EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+
+---- <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>2.1.6
+
+[CTDB TOCID: nj4ubZdFWgR4oA9uNKS.7OHP30U-] <20><><EFBFBD><EFBFBD><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>: <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+<EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> | <20><><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  1   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  2   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  3   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  4   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  5   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  6   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  7   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+  8   | (8/8) <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD>
+
+
+==#1051;<3B><> <20><><EFBFBD><EFBFBD><EFBFBD><EFBFBD><EFBFBD> <20>97<39>7<EFBFBD>5<EFBFBD>011<31><31>7520<32>7<EFBFBD><37>437<33>42<34>9<EFBFBD><39><EFBFBD><EFBFBD>5<EFBFBD>80929<32>8213<31><33><EFBFBD>90656<35>39<33>95<39>8<EFBFBD> ==

test/test_disc_eaclog.py

@@ -72,6 +72,9 @@ class TestTocFromFile(PicardTestCase):
         toc = toc_from_file(test_log)
         self.assertEqual((1, 8, 149323, 150, 25064, 43611, 60890, 83090, 100000, 115057, 135558), toc)
 
+    def test_toc_from_file_eac_windows1251(self):
+        self._test_toc_from_file('eac-windows1251.log')
+
     def test_toc_from_file_eac_utf8(self):
         self._test_toc_from_file('eac-utf8.log')
 

test/test_utils.py

@@ -44,7 +44,10 @@ from unittest.mock import (
     patch,
 )
 
-from test.picardtestcase import PicardTestCase
+from test.picardtestcase import (
+    PicardTestCase,
+    get_test_data_path,
+)
 
 from picard import util
 from picard.const import MUSICBRAINZ_SERVERS
@@ -57,7 +60,7 @@ from picard.util import (
     album_artist_from_path,
     any_exception_isinstance,
     build_qurl,
-    detect_unicode_encoding,
+    detect_file_encoding,
     encoded_queryargs,
     extract_year_from_date,
     find_best_match,
@@ -988,13 +991,13 @@ class IgnoreUpdatesContextTest(PicardTestCase):
 
 class DetectUnicodeEncodingTest(PicardTestCase):
 
-    def test_detect_encoding(self):
+    def test_detect_file_encoding_bom(self):
         boms = {
             b'\xff\xfe': 'utf-16-le',
             b'\xfe\xff': 'utf-16-be',
             b'\00\00\xff\xfe': 'utf-32-le',
             b'\00\00\xfe\xff': 'utf-32-be',
-            b'\xef\xbb\xbf': 'utf-8',
+            b'\xef\xbb\xbf': 'utf-8-sig',
             b'': 'utf-8',
             b'\00': 'utf-8',
         }
@@ -1003,7 +1006,17 @@ class DetectUnicodeEncodingTest(PicardTestCase):
                 f = NamedTemporaryFile(delete=False)
                 f.write(bom)
                 f.close()
-                self.assertEqual(expected_encoding, detect_unicode_encoding(f.name))
+                self.assertEqual(expected_encoding, detect_file_encoding(f.name))
             finally:
                 f.close()
                 os.remove(f.name)
+
+    def test_detect_file_encoding_eac_utf_16_le(self):
+        expected_encoding = 'utf-16-le'
+        file_path = get_test_data_path('eac-utf16le.log')
+        self.assertEqual(expected_encoding, detect_file_encoding(file_path))
+
+    def test_detect_file_encoding_eac_windows_1251(self):
+        expected_encoding = 'windows-1251'
+        file_path = get_test_data_path('eac-windows1251.log')
+        self.assertEqual(expected_encoding, detect_file_encoding(file_path))