diff --git a/picard/album.py b/picard/album.py index cca56275e..c4577f675 100644 --- a/picard/album.py +++ b/picard/album.py @@ -31,7 +31,8 @@ from picard.file import File from picard.track import Track from picard.script import ScriptParser from picard.ui.item import Item -from picard.util import format_time, mbid_validate, asciipunct +from picard.util import format_time, mbid_validate +from picard.util.textencoding import asciipunct from picard.cluster import Cluster from picard.collection import Collection, user_collections from picard.mbxml import ( diff --git a/picard/file.py b/picard/file.py index 8c5727914..20d16b68b 100644 --- a/picard/file.py +++ b/picard/file.py @@ -38,11 +38,13 @@ from picard.util import ( format_time, mimetype, pathcmp, - replace_non_ascii, replace_win32_incompat, sanitize_filename, thread, tracknum_from_filename, +) +from picard.util.textencoding import ( + replace_non_ascii, unaccent, ) from picard.util.filenaming import make_short_filename diff --git a/picard/track.py b/picard/track.py index 989978980..dd47825a5 100644 --- a/picard/track.py +++ b/picard/track.py @@ -22,7 +22,7 @@ from functools import partial from picard import config, log from picard.metadata import Metadata, run_track_metadata_processors from picard.dataobj import DataObject -from picard.util import asciipunct +from picard.util.textencoding import asciipunct from picard.mbxml import recording_to_metadata from picard.script import ScriptParser from picard.const import VARIOUS_ARTISTS_ID diff --git a/picard/util/__init__.py b/picard/util/__init__.py index f9545be73..1f7b61432 100644 --- a/picard/util/__init__.py +++ b/picard/util/__init__.py @@ -26,6 +26,7 @@ from time import time from PyQt4 import QtCore from encodings import rot_13 from string import Template +# Required for compatibility with lastfmplus which imports this from here rather than loading it direct. from functools import partial from collections import defaultdict @@ -42,31 +43,6 @@ class LockableDefaultDict(defaultdict): self.__lock.unlock() -def asciipunct(s): - mapping = { - u"…": u"...", - u"‘": u"'", - u"’": u"'", - u"‚": u"'", - u"“": u"\"", - u"”": u"\"", - u"„": u"\"", - u"′": u"'", - u"″": u"\"", - u"‹": u"<", - u"›": u">", - u"‐": u"-", - u"‒": u"-", - u"–": u"-", - u"−": u"-", - u"—": u"-", - u"―": u"--", - } - for orig, repl in mapping.iteritems(): - s = s.replace(orig, repl) - return s - - class LockableObject(QtCore.QObject): """Read/write lockable object.""" @@ -162,32 +138,6 @@ def sanitize_date(datestr): return ("", "%04d", "%04d-%02d", "%04d-%02d-%02d")[len(date)] % tuple(date) -_unaccent_dict = {u'Æ': u'AE', u'æ': u'ae', u'Œ': u'OE', u'œ': u'oe', u'ß': 'ss'} -_re_latin_letter = re.compile(r"^(LATIN [A-Z]+ LETTER [A-Z]+) WITH") -def unaccent(string): - """Remove accents ``string``.""" - result = [] - for char in string: - if char in _unaccent_dict: - char = _unaccent_dict[char] - else: - try: - name = unicodedata.name(char) - match = _re_latin_letter.search(name) - if match: - char = unicodedata.lookup(match.group(1)) - except: - pass - result.append(char) - return "".join(result) - - -_re_non_ascii = re.compile(r'[^\x00-\x7F]', re.UNICODE) -def replace_non_ascii(string, repl="_"): - """Replace non-ASCII characters from ``string`` by ``repl``.""" - return _re_non_ascii.sub(repl, asciipunct(string)) - - _re_win32_incompat = re.compile(r'["*:<>?|]', re.UNICODE) def replace_win32_incompat(string, repl=u"_"): """Replace win32 filename incompatible characters from ``string`` by diff --git a/picard/util/textencoding.py b/picard/util/textencoding.py new file mode 100644 index 000000000..c5c49524b --- /dev/null +++ b/picard/util/textencoding.py @@ -0,0 +1,442 @@ +# -*- coding: utf-8 -*- +# +# Picard, the next-generation MusicBrainz tagger +# Copyright (C) 2004 Robert Kaye +# Copyright (C) 2006 Lukáš Lalinský +# +# This program is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. + +# This modules provides functionality for simplifying unicode strings. + +# The unicode character set (of over 1m codepoints and 24,000 characters) includes: +# Normal ascii (latin) non-accented characters +# Combined latin characters e.g. ae in normal usage +# Compatibility combined latin characters (retained for compatibility with other character sets) +# These can look very similar to normal characters and can be confusing for searches, sort orders etc. +# Non-latin (e.g. japanese, greek, hebrew etc.) characters +# Both latin and non-latin characters can be accented. Accents can be either: +# Provided by separate nonspacing_mark characters which are visually overlaid (visually 1 character is actually 2); or +# Integrated accented characters (i.e. non-accented characters combined with a nonspace_mark into a single character) +# Again these can be confusing for searches, sort orders etc. +# Punctuation can also be confusing in unicode e.g. several types of single or double quote mark. + +# For latin script: +# Combined characters, accents and punctuation can be visually similar but look different to search engines, +# sort orders etc. and the number of ways to use similar looking characters can (does) result in inconsistent +# usage inside Music metadata. +# +# Simplifying # the unicode character sets by many-to-one mappings can improve consistency and reduce confusion, +# however sometimes the choice of specific characters can be a deliberate part of an album, song title or artist name +# (and should not therefore be changed without careful thought) and occasionally the choice of characters can be +# malicious (i.e. to defeat firewalls or spam filters or to appear to be something else). +# +# Finally, given the size of the unicode character set, fonts are unlikely to display all characters, +# making simplification a necessity. +# +# Simplification may also be needed to make tags conform to ISO-8859-1 (extended ascii) or to make tags or filenames +# into ascii, perhaps because the file system or player cannot support unicode. +# +# Non-latin scripts may also need to be converted to latin scripts through: +# Translation (e.g. hebrew word for mother is translated to "mother"); or +# Transliteration (e.g. the SOUND of the hebrew letter or word is spelt out in latin) +# These are non-trivial, and the software to do these is far from comprehensive. + +# This module provides utility functions to enable simplification of latin and punctuation unicode: +# 1. simplify compatibility characters; +# 2. split combined characters; +# 3. remove accents (entirely or if not in ISO-8859-1 as applicable); +# 4. replace remaining non-ascii or non-ISO-8859-1 characters with a default character +# This module also provides an extension infrastructure to allow translation and / or transliteration plugins to be added. + +import re +import unicodedata +import codecs +from functools import partial + +######################### LATIN SIMPLIFICATION ########################### +# The translation tables for punctuation and latin combined-characters are taken from +# http://unicode.org/repos/cldr/trunk/common/transforms/Latin-ASCII.xml +# Various bugs and mistakes in this have been ironed out during testing. + + +def _re_any(iterable): + return re.compile('([' + ''.join(iterable) + '])', re.UNICODE) + +_additional_compatibility = { + u"\u0276": u"Œ", # LATIN LETTER SMALL CAPITAL OE + u"\u1D00": u"A", # LATIN LETTER SMALL CAPITAL A + u"\u1D01": u"Æ", # LATIN LETTER SMALL CAPITAL AE + u"\u1D04": u"C", # LATIN LETTER SMALL CAPITAL C + u"\u1D05": u"D", # LATIN LETTER SMALL CAPITAL D + u"\u1D07": u"E", # LATIN LETTER SMALL CAPITAL E + u"\u1D0A": u"J", # LATIN LETTER SMALL CAPITAL J + u"\u1D0B": u"K", # LATIN LETTER SMALL CAPITAL K + u"\u1D0D": u"M", # LATIN LETTER SMALL CAPITAL M + u"\u1D0F": u"O", # LATIN LETTER SMALL CAPITAL O + u"\u1D18": u"P", # LATIN LETTER SMALL CAPITAL P + u"\u1D1B": u"T", # LATIN LETTER SMALL CAPITAL T + u"\u1D1C": u"U", # LATIN LETTER SMALL CAPITAL U + u"\u1D20": u"V", # LATIN LETTER SMALL CAPITAL V + u"\u1D21": u"W", # LATIN LETTER SMALL CAPITAL W + u"\u1D22": u"Z", # LATIN LETTER SMALL CAPITAL Z + u"\u3007": u"0", # IDEOGRAPHIC NUMBER ZERO + u"\u00A0": u" ", # NO-BREAK SPACE + u"\u3000": u" ", # IDEOGRAPHIC SPACE (from ‹character-fallback›) + u"\u2033": u"”", # DOUBLE PRIME +} +_re_additional_compatibility = _re_any(_additional_compatibility.keys()) + +def unicode_simplify_compatibility(string): + interim = _re_additional_compatibility.sub(lambda m: _additional_compatibility[m.group(0)], string) + return unicodedata.normalize("NFKC", interim) + + +_simplify_punctuation = { + u"\u013F": u"L", # LATIN CAPITAL LETTER L WITH MIDDLE DOT (compat) + u"\u0140": u"l", # LATIN SMALL LETTER L WITH MIDDLE DOT (compat) + u"\u2018": u"'", # LEFT SINGLE QUOTATION MARK (from ‹character-fallback›) + u"\u2019": u"'", # RIGHT SINGLE QUOTATION MARK (from ‹character-fallback›) + u"\u201A": u",", # SINGLE LOW-9 QUOTATION MARK (from ‹character-fallback›) + u"\u201B": u"'", # SINGLE HIGH-REVERSED-9 QUOTATION MARK (from ‹character-fallback›) + u"\u201C": u"\"", # LEFT DOUBLE QUOTATION MARK (from ‹character-fallback›) + u"\u201D": u"\"", # RIGHT DOUBLE QUOTATION MARK (from ‹character-fallback›) + u"\u201E": u",,", # DOUBLE LOW-9 QUOTATION MARK (from ‹character-fallback›) + u"\u201F": u"\"", # DOUBLE HIGH-REVERSED-9 QUOTATION MARK (from ‹character-fallback›) + u"\u2032": u"'", # PRIME + u"\u2033": u"\"", # DOUBLE PRIME + u"\u301D": u"\"", # REVERSED DOUBLE PRIME QUOTATION MARK + u"\u301E": u"\"", # DOUBLE PRIME QUOTATION MARK + u"\u00AB": u"<<", # LEFT-POINTING DOUBLE ANGLE QUOTATION MARK (from ‹character-fallback›) + u"\u00BB": u">>", # RIGHT-POINTING DOUBLE ANGLE QUOTATION MARK (from ‹character-fallback›) + u"\u2039": u"<", # SINGLE LEFT-POINTING ANGLE QUOTATION MARK + u"\u203A": u">", # SINGLE RIGHT-POINTING ANGLE QUOTATION MARK + u"\u00AD": u"-", # SOFT HYPHEN (from ‹character-fallback›) + u"\u2010": u"-", # HYPHEN (from ‹character-fallback›) + u"\u2011": u"-", # NON-BREAKING HYPHEN (from ‹character-fallback›) + u"\u2012": u"-", # FIGURE DASH (from ‹character-fallback›) + u"\u2013": u"-", # EN DASH (from ‹character-fallback›) + u"\u2014": u"-", # EM DASH (from ‹character-fallback›) + u"\u2015": u"-", # HORIZONTAL BAR (from ‹character-fallback›) + u"\uFE31": u"|", # PRESENTATION FORM FOR VERTICAL EM DASH (compat) + u"\uFE32": u"|", # PRESENTATION FORM FOR VERTICAL EN DASH (compat) + u"\uFE58": u"-", # SMALL EM DASH (compat) + u"\u2016": u"||", # DOUBLE VERTICAL LINE + u"\u2044": u"/", # FRACTION SLASH (from ‹character-fallback›) + u"\u2045": u"[", # LEFT SQUARE BRACKET WITH QUILL + u"\u2046": u"]", # RIGHT SQUARE BRACKET WITH QUILL + u"\u204E": u"*", # LOW ASTERISK + u"\u3001": u",", # IDEOGRAPHIC COMMA + u"\u3002": u".", # IDEOGRAPHIC FULL STOP + u"\u3008": u"<", # LEFT ANGLE BRACKET + u"\u3009": u">", # RIGHT ANGLE BRACKET + u"\u300A": u"<<", # LEFT DOUBLE ANGLE BRACKET + u"\u300B": u">>", # RIGHT DOUBLE ANGLE BRACKET + u"\u3014": u"[", # LEFT TORTOISE SHELL BRACKET + u"\u3015": u"]", # RIGHT TORTOISE SHELL BRACKET + u"\u3018": u"[", # LEFT WHITE TORTOISE SHELL BRACKET + u"\u3019": u"]", # RIGHT WHITE TORTOISE SHELL BRACKET + u"\u301A": u"[", # LEFT WHITE SQUARE BRACKET + u"\u301B": u"]", # RIGHT WHITE SQUARE BRACKET + u"\uFE11": u",", # PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC COMMA (compat) + u"\uFE12": u".", # PRESENTATION FORM FOR VERTICAL IDEOGRAPHIC FULL STOP (compat) + u"\uFE39": u"[", # PRESENTATION FORM FOR VERTICAL LEFT TORTOISE SHELL BRACKET (compat) + u"\uFE3A": u"]", # PRESENTATION FORM FOR VERTICAL RIGHT TORTOISE SHELL BRACKET (compat) + u"\uFE3D": u"<<", # PRESENTATION FORM FOR VERTICAL LEFT DOUBLE ANGLE BRACKET (compat) + u"\uFE3E": u">>", # PRESENTATION FORM FOR VERTICAL RIGHT DOUBLE ANGLE BRACKET (compat) + u"\uFE3F": u"<", # PRESENTATION FORM FOR VERTICAL LEFT ANGLE BRACKET (compat) + u"\uFE40": u">", # PRESENTATION FORM FOR VERTICAL RIGHT ANGLE BRACKET (compat) + u"\uFE51": u",", # SMALL IDEOGRAPHIC COMMA (compat) + u"\uFE5D": u"[", # SMALL LEFT TORTOISE SHELL BRACKET (compat) + u"\uFE5E": u"]", # SMALL RIGHT TORTOISE SHELL BRACKET (compat) + u"\uFF5F": u"((", # FULLWIDTH LEFT WHITE PARENTHESIS (compat)(from ‹character-fallback›) + u"\uFF60": u"))", # FULLWIDTH RIGHT WHITE PARENTHESIS (compat)(from ‹character-fallback›) + u"\uFF61": u".", # HALFWIDTH IDEOGRAPHIC FULL STOP (compat) + u"\uFF64": u",", # HALFWIDTH IDEOGRAPHIC COMMA (compat) + u"\u00D7": u"*", # MULTIPLICATION SIGN + u"\u00F7": u"/", # DIVISION SIGN + u"\u2212": u"-", # MINUS SIGN (from ‹character-fallback›) + u"\u2215": u"/", # DIVISION SLASH (from ‹character-fallback›) + u"\u2216": u"\\", # SET MINUS (from ‹character-fallback›) + u"\u2223": u"|", # DIVIDES (from ‹character-fallback›) + u"\u2225": u"||", # PARALLEL TO (from ‹character-fallback›) + u"\u226A": u"<<", # MUCH LESS-THAN + u"\u226B": u">>", # MUCH GREATER-THAN + u"\u2985": u"((", # LEFT WHITE PARENTHESIS + u"\u2986": u"))", # RIGHT WHITE PARENTHESIS + u"\u00B7": u".", # MIDDLE DOT +} +_re_simplify_punctuation = _re_any(_simplify_punctuation.keys()) + +def unicode_simplify_punctuation(string): + return _re_simplify_punctuation.sub(lambda m: _simplify_punctuation[m.group(0)], string) + + +_simplify_combinations = { + u"\u00C6": u"AE", # LATIN CAPITAL LETTER AE (from ‹character-fallback›) + u"\u00D0": u"D", # LATIN CAPITAL LETTER ETH + u"\u00D8": u"O", # LATIN CAPITAL LETTER O WITH STROKE + u"\u00DE": u"TH", # LATIN CAPITAL LETTER THORN + u"\u00DF": u"ss", # LATIN SMALL LETTER SHARP S (from ‹character-fallback›) + u"\u00E6": u"ae", # LATIN SMALL LETTER AE (from ‹character-fallback›) + u"\u00F0": u"d", # LATIN SMALL LETTER ETH + u"\u00F8": u"o", # LATIN SMALL LETTER O WITH STROKE + u"\u00FE": u"th", # LATIN SMALL LETTER THORN + u"\u0110": u"D", # LATIN CAPITAL LETTER D WITH STROKE + u"\u0111": u"d", # LATIN SMALL LETTER D WITH STROKE + u"\u0126": u"H", # LATIN CAPITAL LETTER H WITH STROKE + u"\u0127": u"h", # LATIN CAPITAL LETTER H WITH STROKE + u"\u0131": u"i", # LATIN SMALL LETTER DOTLESS I + u"\u0138": u"q", # LATIN SMALL LETTER KRA (collates with q in DUCET) + u"\u0141": u"L", # LATIN CAPITAL LETTER L WITH STROKE + u"\u0142": u"l", # LATIN SMALL LETTER L WITH STROKE + u"\u0149": u"'n", # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE (from ‹character-fallback›) + u"\u014A": u"N", # LATIN CAPITAL LETTER ENG + u"\u014B": u"n", # LATIN SMALL LETTER ENG + u"\u0152": u"OE", # LATIN CAPITAL LIGATURE OE (from ‹character-fallback›) + u"\u0153": u"oe", # LATIN SMALL LIGATURE OE (from ‹character-fallback›) + u"\u0166": u"T", # LATIN CAPITAL LETTER T WITH STROKE + u"\u0167": u"t", # LATIN SMALL LETTER T WITH STROKE + u"\u0180": u"b", # LATIN SMALL LETTER B WITH STROKE + u"\u0181": u"B", # LATIN CAPITAL LETTER B WITH HOOK + u"\u0182": u"B", # LATIN CAPITAL LETTER B WITH TOPBAR + u"\u0183": u"b", # LATIN SMALL LETTER B WITH TOPBAR + u"\u0187": u"C", # LATIN CAPITAL LETTER C WITH HOOK + u"\u0188": u"c", # LATIN SMALL LETTER C WITH HOOK + u"\u0189": u"D", # LATIN CAPITAL LETTER AFRICAN D + u"\u018A": u"D", # LATIN CAPITAL LETTER D WITH HOOK + u"\u018B": u"D", # LATIN CAPITAL LETTER D WITH TOPBAR + u"\u018C": u"d", # LATIN SMALL LETTER D WITH TOPBAR + u"\u0190": u"E", # LATIN CAPITAL LETTER OPEN E + u"\u0191": u"F", # LATIN CAPITAL LETTER F WITH HOOK + u"\u0192": u"f", # LATIN SMALL LETTER F WITH HOOK + u"\u0193": u"G", # LATIN CAPITAL LETTER G WITH HOOK + u"\u0195": u"hv", # LATIN SMALL LETTER HV + u"\u0196": u"I", # LATIN CAPITAL LETTER IOTA + u"\u0197": u"I", # LATIN CAPITAL LETTER I WITH STROKE + u"\u0198": u"K", # LATIN CAPITAL LETTER K WITH HOOK + u"\u0199": u"k", # LATIN SMALL LETTER K WITH HOOK + u"\u019A": u"l", # LATIN SMALL LETTER L WITH BAR + u"\u019D": u"N", # LATIN CAPITAL LETTER N WITH LEFT HOOK + u"\u019E": u"n", # LATIN SMALL LETTER N WITH LONG RIGHT LEG + u"\u01A2": u"OI", # LATIN CAPITAL LETTER OI + u"\u01A3": u"oi", # LATIN SMALL LETTER OI + u"\u01A4": u"P", # LATIN CAPITAL LETTER P WITH HOOK + u"\u01A5": u"p", # LATIN SMALL LETTER P WITH HOOK + u"\u01AB": u"t", # LATIN SMALL LETTER T WITH PALATAL HOOK + u"\u01AC": u"T", # LATIN CAPITAL LETTER T WITH HOOK + u"\u01AD": u"t", # LATIN SMALL LETTER T WITH HOOK + u"\u01AE": u"T", # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK + u"\u01B2": u"V", # LATIN CAPITAL LETTER V WITH HOOK + u"\u01B3": u"Y", # LATIN CAPITAL LETTER Y WITH HOOK + u"\u01B4": u"y", # LATIN SMALL LETTER Y WITH HOOK + u"\u01B5": u"Z", # LATIN CAPITAL LETTER Z WITH STROKE + u"\u01B6": u"z", # LATIN SMALL LETTER Z WITH STROKE + u"\u01C4": u"DZ", # LATIN CAPITAL LETTER DZ WITH CARON (compat) + u"\u01C5": u"Dz", # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON (compat) + u"\u01C6": u"dz", # LATIN SMALL LETTER DZ WITH CARON (compat) + u"\u01E4": u"G", # LATIN CAPITAL LETTER G WITH STROKE + u"\u01E5": u"g", # LATIN SMALL LETTER G WITH STROKE + u"\u0221": u"d", # LATIN SMALL LETTER D WITH CURL + u"\u0224": u"Z", # LATIN CAPITAL LETTER Z WITH HOOK + u"\u0225": u"z", # LATIN SMALL LETTER Z WITH HOOK + u"\u0234": u"l", # LATIN SMALL LETTER L WITH CURL + u"\u0235": u"n", # LATIN SMALL LETTER N WITH CURL + u"\u0236": u"t", # LATIN SMALL LETTER T WITH CURL + u"\u0237": u"j", # LATIN SMALL LETTER DOTLESS J + u"\u0238": u"db", # LATIN SMALL LETTER DB DIGRAPH + u"\u0239": u"qp", # LATIN SMALL LETTER QP DIGRAPH + u"\u023A": u"A", # LATIN CAPITAL LETTER A WITH STROKE + u"\u023B": u"C", # LATIN CAPITAL LETTER C WITH STROKE + u"\u023C": u"c", # LATIN SMALL LETTER C WITH STROKE + u"\u023D": u"L", # LATIN CAPITAL LETTER L WITH BAR + u"\u023E": u"T", # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE + u"\u023F": u"s", # LATIN SMALL LETTER S WITH SWASH TAIL + u"\u0240": u"z", # LATIN SMALL LETTER Z WITH SWASH TAIL + u"\u0243": u"B", # LATIN CAPITAL LETTER B WITH STROKE + u"\u0244": u"U", # LATIN CAPITAL LETTER U BAR + u"\u0246": u"E", # LATIN CAPITAL LETTER E WITH STROKE + u"\u0247": u"e", # LATIN SMALL LETTER E WITH STROKE + u"\u0248": u"J", # LATIN CAPITAL LETTER J WITH STROKE + u"\u0249": u"j", # LATIN SMALL LETTER J WITH STROKE + u"\u024C": u"R", # LATIN CAPITAL LETTER R WITH STROKE + u"\u024D": u"r", # LATIN SMALL LETTER R WITH STROKE + u"\u024E": u"Y", # LATIN CAPITAL LETTER Y WITH STROKE + u"\u024F": u"y", # LATIN SMALL LETTER Y WITH STROKE + u"\u0253": u"b", # LATIN SMALL LETTER B WITH HOOK + u"\u0255": u"c", # LATIN SMALL LETTER C WITH CURL + u"\u0256": u"d", # LATIN SMALL LETTER D WITH TAIL + u"\u0257": u"d", # LATIN SMALL LETTER D WITH HOOK + u"\u025B": u"e", # LATIN SMALL LETTER OPEN E + u"\u025F": u"j", # LATIN SMALL LETTER DOTLESS J WITH STROKE + u"\u0260": u"g", # LATIN SMALL LETTER G WITH HOOK + u"\u0261": u"g", # LATIN SMALL LETTER SCRIPT G + u"\u0262": u"G", # LATIN LETTER SMALL CAPITAL G + u"\u0266": u"h", # LATIN SMALL LETTER H WITH HOOK + u"\u0267": u"h", # LATIN SMALL LETTER HENG WITH HOOK + u"\u0268": u"i", # LATIN SMALL LETTER I WITH STROKE + u"\u026A": u"I", # LATIN LETTER SMALL CAPITAL I + u"\u026B": u"l", # LATIN SMALL LETTER L WITH MIDDLE TILDE + u"\u026C": u"l", # LATIN SMALL LETTER L WITH BELT + u"\u026D": u"l", # LATIN SMALL LETTER L WITH RETROFLEX HOOK + u"\u0271": u"m", # LATIN SMALL LETTER M WITH HOOK + u"\u0272": u"n", # LATIN SMALL LETTER N WITH LEFT HOOK + u"\u0273": u"n", # LATIN SMALL LETTER N WITH RETROFLEX HOOK + u"\u0274": u"N", # LATIN LETTER SMALL CAPITAL N + u"\u0276": u"OE", # LATIN LETTER SMALL CAPITAL OE + u"\u027C": u"r", # LATIN SMALL LETTER R WITH LONG LEG + u"\u027D": u"r", # LATIN SMALL LETTER R WITH TAIL + u"\u027E": u"r", # LATIN SMALL LETTER R WITH FISHHOOK + u"\u0280": u"R", # LATIN LETTER SMALL CAPITAL R + u"\u0282": u"s", # LATIN SMALL LETTER S WITH HOOK + u"\u0288": u"t", # LATIN SMALL LETTER T WITH RETROFLEX HOOK + u"\u0289": u"u", # LATIN SMALL LETTER U BAR + u"\u028B": u"v", # LATIN SMALL LETTER V WITH HOOK + u"\u028F": u"Y", # LATIN LETTER SMALL CAPITAL Y + u"\u0290": u"z", # LATIN SMALL LETTER Z WITH RETROFLEX HOOK + u"\u0291": u"z", # LATIN SMALL LETTER Z WITH CURL + u"\u0299": u"B", # LATIN LETTER SMALL CAPITAL B + u"\u029B": u"G", # LATIN LETTER SMALL CAPITAL G WITH HOOK + u"\u029C": u"H", # LATIN LETTER SMALL CAPITAL H + u"\u029D": u"j", # LATIN SMALL LETTER J WITH CROSSED-TAIL + u"\u029F": u"L", # LATIN LETTER SMALL CAPITAL L + u"\u02A0": u"q", # LATIN SMALL LETTER Q WITH HOOK + u"\u02A3": u"dz", # LATIN SMALL LETTER DZ DIGRAPH + u"\u02A5": u"dz", # LATIN SMALL LETTER DZ DIGRAPH WITH CURL + u"\u02A6": u"ts", # LATIN SMALL LETTER TS DIGRAPH + u"\u02AA": u"ls", # LATIN SMALL LETTER LS DIGRAPH + u"\u02AB": u"lz", # LATIN SMALL LETTER LZ DIGRAPH + u"\u1D01": u"AE", # LATIN LETTER SMALL CAPITAL AE + u"\u1D03": u"B", # LATIN LETTER SMALL CAPITAL BARRED B + u"\u1D06": u"D", # LATIN LETTER SMALL CAPITAL ETH + u"\u1D0C": u"L", # LATIN LETTER SMALL CAPITAL L WITH STROKE + u"\u1D6B": u"ue", # LATIN SMALL LETTER UE + u"\u1D6C": u"b", # LATIN SMALL LETTER B WITH MIDDLE TILDE + u"\u1D6D": u"d", # LATIN SMALL LETTER D WITH MIDDLE TILDE + u"\u1D6E": u"f", # LATIN SMALL LETTER F WITH MIDDLE TILDE + u"\u1D6F": u"m", # LATIN SMALL LETTER M WITH MIDDLE TILDE + u"\u1D70": u"n", # LATIN SMALL LETTER N WITH MIDDLE TILDE + u"\u1D71": u"p", # LATIN SMALL LETTER P WITH MIDDLE TILDE + u"\u1D72": u"r", # LATIN SMALL LETTER R WITH MIDDLE TILDE + u"\u1D73": u"r", # LATIN SMALL LETTER R WITH FISHHOOK AND MIDDLE TILDE + u"\u1D74": u"s", # LATIN SMALL LETTER S WITH MIDDLE TILDE + u"\u1D75": u"t", # LATIN SMALL LETTER T WITH MIDDLE TILDE + u"\u1D76": u"z", # LATIN SMALL LETTER Z WITH MIDDLE TILDE + u"\u1D7A": u"th", # LATIN SMALL LETTER TH WITH STRIKETHROUGH + u"\u1D7B": u"I", # LATIN SMALL CAPITAL LETTER I WITH STROKE + u"\u1D7D": u"p", # LATIN SMALL LETTER P WITH STROKE + u"\u1D7E": u"U", # LATIN SMALL CAPITAL LETTER U WITH STROKE + u"\u1D80": u"b", # LATIN SMALL LETTER B WITH PALATAL HOOK + u"\u1D81": u"d", # LATIN SMALL LETTER D WITH PALATAL HOOK + u"\u1D82": u"f", # LATIN SMALL LETTER F WITH PALATAL HOOK + u"\u1D83": u"g", # LATIN SMALL LETTER G WITH PALATAL HOOK + u"\u1D84": u"k", # LATIN SMALL LETTER K WITH PALATAL HOOK + u"\u1D85": u"l", # LATIN SMALL LETTER L WITH PALATAL HOOK + u"\u1D86": u"m", # LATIN SMALL LETTER M WITH PALATAL HOOK + u"\u1D87": u"n", # LATIN SMALL LETTER N WITH PALATAL HOOK + u"\u1D88": u"p", # LATIN SMALL LETTER P WITH PALATAL HOOK + u"\u1D89": u"r", # LATIN SMALL LETTER R WITH PALATAL HOOK + u"\u1D8A": u"s", # LATIN SMALL LETTER S WITH PALATAL HOOK + u"\u1D8C": u"v", # LATIN SMALL LETTER V WITH PALATAL HOOK + u"\u1D8D": u"x", # LATIN SMALL LETTER X WITH PALATAL HOOK + u"\u1D8E": u"z", # LATIN SMALL LETTER Z WITH PALATAL HOOK + u"\u1D8F": u"a", # LATIN SMALL LETTER A WITH RETROFLEX HOOK + u"\u1D91": u"d", # LATIN SMALL LETTER D WITH HOOK AND TAIL + u"\u1D92": u"e", # LATIN SMALL LETTER E WITH RETROFLEX HOOK + u"\u1D93": u"e", # LATIN SMALL LETTER OPEN E WITH RETROFLEX HOOK + u"\u1D96": u"i", # LATIN SMALL LETTER I WITH RETROFLEX HOOK + u"\u1D99": u"u", # LATIN SMALL LETTER U WITH RETROFLEX HOOK + u"\u1E9A": u"a", # LATIN SMALL LETTER A WITH RIGHT HALF RING + u"\u1E9C": u"s", # LATIN SMALL LETTER LONG S WITH DIAGONAL STROKE + u"\u1E9D": u"s", # LATIN SMALL LETTER LONG S WITH HIGH STROKE + u"\u1E9E": u"SS", # LATIN CAPITAL LETTER SHARP S + u"\u1EFA": u"LL", # LATIN CAPITAL LETTER MIDDLE-WELSH LL + u"\u1EFB": u"ll", # LATIN SMALL LETTER MIDDLE-WELSH LL + u"\u1EFC": u"V", # LATIN CAPITAL LETTER MIDDLE-WELSH V + u"\u1EFD": u"v", # LATIN SMALL LETTER MIDDLE-WELSH V + u"\u1EFE": u"Y", # LATIN CAPITAL LETTER Y WITH LOOP + u"\u1EFF": u"y", # LATIN SMALL LETTER Y WITH LOOP + u"\u00A9": u"(C)", # COPYRIGHT SIGN (from ‹character-fallback›) + u"\u00AE": u"(R)", # REGISTERED SIGN (from ‹character-fallback›) + u"\u20A0": u"CE", # EURO-CURRENCY SIGN (from ‹character-fallback›) + u"\u20A2": u"Cr", # CRUZEIRO SIGN (from ‹character-fallback›) + u"\u20A3": u"Fr.", # FRENCH FRANC SIGN (from ‹character-fallback›) + u"\u20A4": u"L.", # LIRA SIGN (from ‹character-fallback›) + u"\u20A7": u"Pts", # PESETA SIGN (from ‹character-fallback›) + u"\u20BA": u"TL", # TURKISH LIRA SIGN (from ‹character-fallback›) + u"\u20B9": u"Rs", # INDIAN RUPEE SIGN (from ‹character-fallback›) + u"\u211E": u"Rx", # PRESCRIPTION TAKE (from ‹character-fallback›) + u"\u33A7": u"m/s", # SQUARE M OVER S (compat) (from ‹character-fallback›) + u"\u33AE": u"rad/s", # SQUARE RAD OVER S (compat) (from ‹character-fallback›) + u"\u33C6": u"C/kg", # SQUARE C OVER KG (compat) (from ‹character-fallback›) + u"\u33DE": u"V/m", # SQUARE V OVER M (compat) (from ‹character-fallback›) + u"\u33DF": u"A/m", # SQUARE A OVER M (compat) (from ‹character-fallback›) + u"\u00BC": u" 1/4", # VULGAR FRACTION ONE QUARTER (from ‹character-fallback›) + u"\u00BD": u" 1/2", # VULGAR FRACTION ONE HALF (from ‹character-fallback›) + u"\u00BE": u" 3/4", # VULGAR FRACTION THREE QUARTERS (from ‹character-fallback›) + u"\u2153": u" 1/3", # VULGAR FRACTION ONE THIRD (from ‹character-fallback›) + u"\u2154": u" 2/3", # VULGAR FRACTION TWO THIRDS (from ‹character-fallback›) + u"\u2155": u" 1/5", # VULGAR FRACTION ONE FIFTH (from ‹character-fallback›) + u"\u2156": u" 2/5", # VULGAR FRACTION TWO FIFTHS (from ‹character-fallback›) + u"\u2157": u" 3/5", # VULGAR FRACTION THREE FIFTHS (from ‹character-fallback›) + u"\u2158": u" 4/5", # VULGAR FRACTION FOUR FIFTHS (from ‹character-fallback›) + u"\u2159": u" 1/6", # VULGAR FRACTION ONE SIXTH (from ‹character-fallback›) + u"\u215A": u" 5/6", # VULGAR FRACTION FIVE SIXTHS (from ‹character-fallback›) + u"\u215B": u" 1/8", # VULGAR FRACTION ONE EIGHTH (from ‹character-fallback›) + u"\u215C": u" 3/8", # VULGAR FRACTION THREE EIGHTHS (from ‹character-fallback›) + u"\u215D": u" 5/8", # VULGAR FRACTION FIVE EIGHTHS (from ‹character-fallback›) + u"\u215E": u" 7/8", # VULGAR FRACTION SEVEN EIGHTHS (from ‹character-fallback›) + u"\u215F": u" 1/", # FRACTION NUMERATOR ONE (from ‹character-fallback›) + u"\u1E9F": u"dd", # LATIN SMALL LETTER DELTA + u"\u0184": u"H", # LATIN CAPITAL LETTER TONE SIX + u"\u0185": u"h", # LATIN SMALL LETTER TONE SIX + u"\u00D8": u"OE", # LATIN CAPITAL LETTER O WITH STROKE + u"\u00F8": u"oe", # LATIN SMALL LETTER O WITH STROKE +} +_re_simplify_combinations = _re_any(_simplify_combinations) + +def unicode_simplify_combinations(string): + return _re_simplify_combinations.sub(lambda m: _simplify_combinations[m.group(0)], string) + + +def unicode_simplify_accents(string): + result = ''.join(c for c in unicodedata.normalize('NFKD', string) if not unicodedata.combining(c)) + # result = ''.join(c for c in unicodedata.normalize('NFKD', string) if unicodedata.category(c) != 'Mn') + # result = _re_nonspacing_marks.sub(u"",unicodedata.normalize('NFKD', string)) + assert result == unicodedata.normalize('NFC',result) + return result + + +def asciipunct(string): + interim = unicode_simplify_compatibility(string) + return unicode_simplify_punctuation(interim) + + +def unaccent(string): + """Remove accents ``string``.""" + return unicode_simplify_accents(string) + + +def replace_non_ascii(string, repl="_"): + """Replace non-ASCII characters from ``string`` by ``repl``.""" + interim = unicode_simplify_combinations(string) + interim = unicode_simplify_accents(interim) + interim = unicode_simplify_punctuation(interim) + interim = unicode_simplify_compatibility(interim) + + def error_repl(e, repl=u"_"): + return(repl, e.start+1) + codecs.register_error('repl', partial(error_repl,repl=unicode(repl))) + + return interim.encode('ascii','repl') diff --git a/test/test_textencoding.py b/test/test_textencoding.py new file mode 100644 index 000000000..7e5a6cfab --- /dev/null +++ b/test/test_textencoding.py @@ -0,0 +1,233 @@ +# -*- coding: utf-8 -*- + +import os.path +import unittest +from picard import util +#from picard.util import textencoding + +# Set the value to true below to show the coverage of Latin characters +show_latin2ascii_coverage = False + +compatibility_from = ( + u"\u0132\u0133\u017F\u01C7\u01C8\u01C9\u01CA\u01CB\u01CC\u01F1" # IJijſLJLjljNJNjnjDZ + u"\u01F2\u01F3\uFB00\uFB01\uFB02\uFB03\uFB04\uFB05\uFB06\uFF21" # DzdzfffiflffifflſtstＡ + u"\uFF22\uFF23\uFF24\uFF25\uFF26\uFF27\uFF28\uFF29\uFF2A\uFF2B" # ＢＣＤＥＦＧＨＩＪＫ + u"\uFF2C\uFF2D\uFF2E\uFF2F\uFF30\uFF31\uFF32\uFF33\uFF34\uFF35" # ＬＭＮＯＰＱＲＳＴＵ + u"\uFF36\uFF37\uFF38\uFF39\uFF3A\uFF41\uFF42\uFF43\uFF44\uFF45" # ＶＷＸＹＺａｂｃｄｅ + u"\uFF46\uFF47\uFF48\uFF49\uFF4A\uFF4B\uFF4C\uFF4D\uFF4E\uFF4F" # ｆｇｈｉｊｋｌｍｎｏ + u"\uFF50\uFF51\uFF52\uFF53\uFF54\uFF55\uFF56\uFF57\uFF58\uFF59" # ｐｑｒｓｔｕｖｗｘｙ + u"\uFF5A\u2100\u2101\u2102\u2105\u2106\u210A\u210B\u210C\u210D" # ｚ℀℁ℂ℅℆ℊℋℌℍ + u"\u210E\u2110\u2111\u2112\u2113\u2115\u2116\u2119\u211A\u211B" # ℎℐℑℒℓℕ№ℙℚℛ + u"\u211C\u211D\u2121\u2124\u2128\u212C\u212D\u212F\u2130\u2131" # ℜℝ℡ℤℨℬℭℯℰℱ + u"\u2133\u2134\u2139\u213B\u2145\u2146\u2147\u2148\u2149\u3371" # ℳℴℹ℻ⅅⅆⅇⅈⅉ㍱ + u"\u3372\u3373\u3374\u3375\u3376\u3377\u337A\u3380\u3381\u3383" # ㍲㍳㍴㍵㍶㍷㍺㎀㎁㎃ + u"\u3384\u3385\u3386\u3387\u3388\u3389\u338A\u338B\u338E\u338F" # ㎄㎅㎆㎇㎈㎉㎊㎋㎎㎏ + u"\u3390\u3391\u3392\u3393\u3394\u3399\u339A\u339C\u339D\u339E" # ㎐㎑㎒㎓㎔㎙㎚㎜㎝㎞ + u"\u33A9\u33AA\u33AB\u33AC\u33AD\u33B0\u33B1\u33B3\u33B4\u33B5" # ㎩㎪㎫㎬㎭㎰㎱㎳㎴㎵ + u"\u33B7\u33B8\u33B9\u33BA\u33BB\u33BD\u33BE\u33BF\u33C2\u33C3" # ㎷㎸㎹㎺㎻㎽㎾㎿㏂㏃ + u"\u33C4\u33C5\u33C7\u33C8\u33C9\u33CA\u33CB\u33CC\u33CD\u33CE" # ㏄㏅㏇㏈㏉㏊㏋㏌㏍㏎ + u"\u33CF\u33D0\u33D1\u33D2\u33D3\u33D4\u33D5\u33D6\u33D7\u33D8" # ㏏㏐㏑㏒㏓㏔㏕㏖㏗㏘ + u"\u33D9\u33DA\u33DB\u33DC\u33DD\u249C\u249D\u249E\u249F\u24A0" # ㏙㏚㏛㏜㏝⒜⒝⒞⒟⒠ + u"\u24A1\u24A2\u24A3\u24A4\u24A5\u24A6\u24A7\u24A8\u24A9\u24AA" # ⒡⒢⒣⒤⒥⒦⒧⒨⒩⒪ + u"\u24AB\u24AC\u24AD\u24AE\u24AF\u24B0\u24B1\u24B2\u24B3\u24B4" # ⒫⒬⒭⒮⒯⒰⒱⒲⒳⒴ + u"\u24B5\u2160\u2161\u2162\u2163\u2164\u2165\u2166\u2167\u2168" # ⒵ⅠⅡⅢⅣⅤⅥⅦⅧⅨ + u"\u2169\u216A\u216B\u216C\u216D\u216E\u216F\u2170\u2171\u2172" # ⅩⅪⅫⅬⅭⅮⅯⅰⅱⅲ + u"\u2173\u2174\u2175\u2176\u2177\u2178\u2179\u217A\u217B\u217C" # ⅳⅴⅵⅶⅷⅸⅹⅺⅻⅼ + u"\u217D\u217E\u217F\u2474\u2475\u2476\u2477\u2478\u2479\u247A" # ⅽⅾⅿ⑴⑵⑶⑷⑸⑹⑺ + u"\u247B\u247C\u247D\u247E\u247F\u2480\u2481\u2482\u2483\u2484" # ⑻⑼⑽⑾⑿⒀⒁⒂⒃⒄ + u"\u2485\u2486\u2487\u2488\u2489\u248A\u248B\u248C\u248D\u248E" # ⒅⒆⒇⒈⒉⒊⒋⒌⒍⒎ + u"\u248F\u2490\u2491\u2492\u2493\u2494\u2495\u2496\u2497\u2498" # ⒏⒐⒑⒒⒓⒔⒕⒖⒗⒘ + u"\u2499\u249A\u249B\uFF10\uFF11\uFF12\uFF13\uFF14\uFF15\uFF16" # ⒙⒚⒛０１２３４５６ + u"\uFF17\uFF18\uFF19\u2002\u2003\u2004\u2005\u2006\u2007\u2008" # ７８９\u2002\u2003\u2004\u2005\u2006\u2007\u2008 + u"\u2009\u200A\u205F\uFF02\uFF07\uFE63\uFF0D\u2024\u2025\u2026" # \u2009\u200A\u205F＂＇﹣－․‥… + u"\u203C\u2047\u2048\u2049\uFE10\uFE13\uFE14\uFE15\uFE16\uFE19" # ‼⁇⁈⁉︐︓︔︕︖︙ + u"\uFE30\uFE35\uFE36\uFE37\uFE38\uFE47\uFE48\uFE50\uFE52\uFE54" # ︰︵︶︷︸﹇﹈﹐﹒﹔ + u"\uFE55\uFE56\uFE57\uFE59\uFE5A\uFE5B\uFE5C\uFE5F\uFE60\uFE61" # ﹕﹖﹗﹙﹚﹛﹜﹟﹠﹡ + u"\uFE62\uFE64\uFE65\uFE66\uFE68\uFE69\uFE6A\uFE6B\uFF01\uFF03" # ﹢﹤﹥﹦﹨﹩﹪﹫！＃ + u"\uFF04\uFF05\uFF06\uFF08\uFF09\uFF0A\uFF0B\uFF0C\uFF0E\uFF0F" # ＄％＆（）＊＋，．／ + u"\uFF1A\uFF1B\uFF1C\uFF1D\uFF1E\uFF1F\uFF20\uFF3B\uFF3C\uFF3D" # ：；＜＝＞？＠［＼］ + u"\uFF3E\uFF3F\uFF40\uFF5B\uFF5C\uFF5D\uFF5E\u2A74\u2A75\u2A76" # ＾＿｀｛｜｝～⩴⩵⩶ + ) +compatibility_to = ( + u"IJijsLJLjljNJNjnjDZDzdzfffiflffifflststABCDEFGHIJKLMNOPQRSTU" + u"VWXYZabcdefghijklmnopqrstuvwxyza/ca/sCc/oc/ugHHH" + u"hIILlNNoPQRRRTELZZBCeEFMoiFAXDdeijhPadaAUbaroVpcdmIUpAnAmA" + u"kAKBMBGBcalkcalpFnFmgkgHzkHzMHzGHzTHzfmnmmmcmkmPakPaMPaGParadpsnsmspVnVmVkVMVpWnWmWkWMWa.m.Bq" + u"cccdCo.dBGyhaHPinKKKMktlmlnloglxmbmilmolPHp.m.PPMPRsrSvWb(a)(b)(c)(d)(e)(f)(g)(h)(i)(j)(k)(l)(m)(n)(o)" + u"(p)(q)(r)(s)(t)(u)(v)(w)(x)(y)(z)IIIIIIIVVVIVIIVIIIIXXXIXIILCDMiiiiiiivvviviiviiiixxxixiil" + u"cdm(1)(2)(3)(4)(5)(6)(7)(8)(9)(10)(11)(12)(13)(14)(15)(16)(17)(18)(19)(20)1.2.3.4.5.6.7.8.9.10.11.12.13.14.15.16.17." + u"18.19.20.0123456789 \"'--......!!???!!?,:;!?..." + u"..(){}[],.;:?!(){}#&*+<>=\\$%@!#$%&()*+,./" + u":;<=>?@[\\]^_`{|}~::======" + ) +compatibility_from += ( + u"\u1D00\u1D04\u1D05\u1D07\u1D0A\u1D0B\u1D0D\u1D0F\u1D18\u1D1B" # ᴀᴄᴅᴇᴊᴋᴍᴏᴘᴛ + u"\u1D1C\u1D20\u1D21\u1D22\u3007\u00A0\u3000" # ᴜᴠᴡᴢ〇\u00A0\u3000 + ) +compatibility_to += u"ACDEJKMOPTUVWZ0 " +punctuation_from = ( + u"\u2018\u2019\u201A\u201B\u201C\u201D\u201E\u201F\u2032\u301D" # ‘’‚‛“”„‟′〝 + u"\u301E\u00AB\u00BB\u2039\u203A\u00AD\u2010\u2012\u2013\u2014" # 〞«»‹›\u00AD‐‒–— + u"\u2015\u2016\u2044\u2045\u2046\u204E\u3001\u3002\u3008\u3009" # ―‖⁄⁅⁆⁎、。〈〉 + u"\u300A\u300B\u3014\u3015\u3018\u3019\u301A\u301B\u00D7\u00F7" # 《》〔〕〘〙〚〛×÷ + u"\u2212\u2215\u2216\u2223\u2225\u226A\u226B\u2985\u2986\u00B7" # −∕∖∣∥≪≫⦅⦆· + ) +punctuation_to = u"'','\"\",,\"'\"\"<<>><>------||/[]*,.<><<>>[][][]*/-/\\|||<<>>(())." +combinations_from = ( + u"\u00C6\u00D0\u00D8\u00DE\u00DF\u00E6\u00F0\u00F8\u00FE\u0110" # ÆÐØÞßæðøþĐ + u"\u0111\u0126\u0127\u0131\u0138\u0141\u0142\u014A\u014B\u0152" # đĦħıĸŁłŊŋŒ + u"\u0153\u0166\u0167\u0180\u0181\u0182\u0183\u0187\u0188\u0189" # œŦŧƀƁƂƃƇƈƉ + u"\u018A\u018B\u018C\u0190\u0191\u0192\u0193\u0195\u0196\u0197" # ƊƋƌƐƑƒƓƕƖƗ + u"\u0198\u0199\u019A\u019D\u019E\u01A2\u01A3\u01A4\u01A5\u01AB" # ƘƙƚƝƞƢƣƤƥƫ + u"\u01AC\u01AD\u01AE\u01B2\u01B3\u01B4\u01B5\u01B6\u01E4\u01E5" # ƬƭƮƲƳƴƵƶǤǥ + u"\u0221\u0224\u0225\u0234\u0235\u0236\u0237\u0238\u0239\u023A" # ȡȤȥȴȵȶȷȸȹȺ + u"\u023B\u023C\u023D\u023E\u023F\u0240\u0243\u0244\u0246\u0247" # ȻȼȽȾȿɀɃɄɆɇ + u"\u0248\u0249\u024C\u024D\u024E\u024F\u0253\u0255\u0256\u0257" # ɈɉɌɍɎɏɓɕɖɗ + u"\u025B\u025F\u0260\u0261\u0262\u0266\u0267\u0268\u026A\u026B" # ɛɟɠɡɢɦɧɨɪɫ + u"\u026C\u026D\u0271\u0272\u0273\u0274\u027C\u027D\u027E\u0280" # ɬɭɱɲɳɴɼɽɾʀ + u"\u0282\u0288\u0289\u028B\u028F\u0290\u0291\u0299\u029B\u029C" # ʂʈʉʋʏʐʑʙʛʜ + u"\u029D\u029F\u02A0\u02A3\u02A5\u02A6\u02AA\u02AB\u1D03\u1D06" # ʝʟʠʣʥʦʪʫᴃᴆ + u"\u1D0C\u1D6B\u1D6C\u1D6D\u1D6E\u1D6F\u1D70\u1D71\u1D72\u1D73" # ᴌᵫᵬᵭᵮᵯᵰᵱᵲᵳ + u"\u1D74\u1D75\u1D76\u1D7A\u1D7B\u1D7D\u1D7E\u1D80\u1D81\u1D82" # ᵴᵵᵶᵺᵻᵽᵾᶀᶁᶂ + u"\u1D83\u1D84\u1D85\u1D86\u1D87\u1D88\u1D89\u1D8A\u1D8C\u1D8D" # ᶃᶄᶅᶆᶇᶈᶉᶊᶌᶍ + u"\u1D8E\u1D8F\u1D91\u1D92\u1D93\u1D96\u1D99\u1E9C\u1E9D\u1E9E" # ᶎᶏᶑᶒᶓᶖᶙẜẝẞ + u"\u1EFA\u1EFB\u1EFC\u1EFD\u1EFE\u1EFF\u00A9\u00AE\u20A0\u20A2" # ỺỻỼỽỾỿ©®₠₢ + u"\u20A3\u20A4\u20A7\u20BA\u20B9\u211E" # ₣₤₧₺₹℞ + ) +combinations_to = ( + u"AEDOETHssaedoethDdHhiqLlNnOEoeTtbBBbCcDDDdEFfGhvII" + U"KklNnOIoiPptTtTVYyZzGgdZzlntjdbqpACcLTszBUEe" + u"JjRrYybcddejggGhhiIlllmnnNrrrRstuvYzzBGH" + u"jLqdzdztslslzBDLuebdfmnprrstzthIpUbdfgklmnprsvx" + u"zadeeiussSSLLllVvYy(C)(R)CECrFr.L.PtsTLRsRx" + ) +ascii = u" !\"#$%&'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\]^_`abcdefghijklmnopqrstuvwxyz{|}~" + + +class CompatibilityTest(unittest.TestCase): + + def test_correct(self): + self.maxDiff = None + self.assertEqual(util.textencoding.unicode_simplify_compatibility(compatibility_from), compatibility_to) + self.assertEqual(util.textencoding.unicode_simplify_compatibility(punctuation_from), punctuation_from) + self.assertEqual(util.textencoding.unicode_simplify_compatibility(combinations_from), combinations_from) + self.assertEqual(util.textencoding.unicode_simplify_compatibility(ascii), ascii) + + def test_incorrect(self): + pass + + +class PunctuationTest(unittest.TestCase): + + def test_correct(self): + self.maxDiff = None + self.assertEqual(util.textencoding.unicode_simplify_punctuation(compatibility_from), compatibility_from) + self.assertEqual(util.textencoding.unicode_simplify_punctuation(punctuation_from), punctuation_to) + self.assertEqual(util.textencoding.unicode_simplify_punctuation(combinations_from), combinations_from) + self.assertEqual(util.textencoding.unicode_simplify_punctuation(ascii), ascii) + + def test_incorrect(self): + pass + + +class CombinationsTest(unittest.TestCase): + + def test_correct(self): + self.maxDiff = None + self.assertEqual(util.textencoding.unicode_simplify_combinations(combinations_from), combinations_to) + self.assertEqual(util.textencoding.unicode_simplify_combinations(compatibility_from), compatibility_from) + self.assertEqual(util.textencoding.unicode_simplify_combinations(punctuation_from), punctuation_from) + self.assertEqual(util.textencoding.unicode_simplify_combinations(ascii), ascii) + + def test_incorrect(self): + pass + + +class AsciiPunctTest(unittest.TestCase): + + def test_correct(self): + self.assertEqual(util.textencoding.asciipunct(u"‘Test’"), u"'Test'") # Quotations + self.assertEqual(util.textencoding.asciipunct(u"“Test”"), u"\"Test\"") # Quotations + self.assertEqual(util.textencoding.asciipunct(u"1′6″"), u"1'6\"") # Quotations + self.assertEqual(util.textencoding.asciipunct(u"…"), u"...") # Ellipses + + def test_incorrect(self): + pass + + +class UnaccentTest(unittest.TestCase): + + def test_correct(self): + self.assertEqual(util.textencoding.unaccent(u"Lukáš"), u"Lukas") + self.assertEqual(util.textencoding.unaccent(u"Björk"), u"Bjork") + self.assertEqual(util.textencoding.unaccent(u"小室哲哉"), u"小室哲哉") + + def test_incorrect(self): + self.assertNotEqual(util.textencoding.unaccent(u"Björk"), u"Björk") + self.assertNotEqual(util.textencoding.unaccent(u"小室哲哉"), u"Tetsuya Komuro") + self.assertNotEqual(util.textencoding.unaccent(u"Trentemøller"), u"Trentemoller") + self.assertNotEqual(util.textencoding.unaccent(u"Ænima"), u"AEnima") + self.assertNotEqual(util.textencoding.unaccent(u"ænima"), u"aenima") + + +class ReplaceNonAsciiTest(unittest.TestCase): + + def test_correct(self): + self.assertEqual(util.textencoding.replace_non_ascii(u"Lukáš"), u"Lukas") + self.assertEqual(util.textencoding.replace_non_ascii(u"Björk"), u"Bjork") + self.assertEqual(util.textencoding.replace_non_ascii(u"Trentemøller"), u"Trentemoeller") + self.assertEqual(util.textencoding.replace_non_ascii(u"Ænima"), u"AEnima") + self.assertEqual(util.textencoding.replace_non_ascii(u"ænima"), u"aenima") + self.assertEqual(util.textencoding.replace_non_ascii(u"小室哲哉"), u"____") + self.assertEqual(util.textencoding.replace_non_ascii(u"ᴀᴄᴇ"), u"ACE") # Latin Letter Small + self.assertEqual(util.textencoding.replace_non_ascii(u"Ａｂｃ"), u"Abc") # Fullwidth Latin + self.assertEqual(util.textencoding.replace_non_ascii(u"500㎏,2㎓"), u"500kg,2GHz") # Technical + self.assertEqual(util.textencoding.replace_non_ascii(u"⒜⒝⒞"), u"(a)(b)(c)") # Parenthesised Latin + self.assertEqual(util.textencoding.replace_non_ascii(u"ⅯⅯⅩⅣ"), u"MMXIV") # Roman numerals + self.assertEqual(util.textencoding.replace_non_ascii(u"ⅿⅿⅹⅳ"), u"mmxiv") # Roman numerals small + self.assertEqual(util.textencoding.replace_non_ascii(u"⑴⑵⑶"), u"(1)(2)(3)") # Parenthesised numbers + self.assertEqual(util.textencoding.replace_non_ascii(u"⒈ ⒉ ⒊"), u"1. 2. 3.") # Digit full stop + self.assertEqual(util.textencoding.replace_non_ascii(u"１２３"), u"123") # Fullwidth digits + + def test_incorrect(self): + self.assertNotEqual(util.textencoding.replace_non_ascii(u"Lukáš"), u"Lukáš") + self.assertNotEqual(util.textencoding.replace_non_ascii(u"Lukáš"), u"Luk____") + +if show_latin2ascii_coverage: + # The following code set blocks are taken from: + # http://en.wikipedia.org/wiki/Latin_script_in_Unicode + latin_1 = u"ÀÁÂÃÄÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖ×ØÙÚÛÜÝÞßàáâãäåæçèéêëìíîïðñòóôõö÷øùúûüýþÿ" + latin_a = u"ĀāĂ㥹ĆćĈĉĊċČčĎďĐđĒēĔĕĖėĘęĚěĜĝĞğĠġĢģĤĥĦħĨĩĪīĬĭĮįİıIJijĴĵĶķĸĹĺĻļĽľĿ" \ + u"ŀŁłŃńŅņŇňʼnŊŋŌōŎŏŐőŒœŔŕŖŗŘřŚśŜŝŞşŠšŢţŤťŦŧŨũŪūŬŭŮůŰűŲųŴŵŶŷŸŹźŻżŽž" + latin_b = u"ƀƁƂƃƄƅƆƇƈƉƊƋƌƍƎƏƐƑƒƓƔƕƖƗƘƙƚƛƜƝƞƟƠơƢƣƤƥƦƧƨƩƪƫƬƭƮƯưƱƲƳƴƵƶƷƸƹƺƻƼƽƾƿ" \ + u"ǀǁǂǃDŽDždžLJLjljNJNjnjǍǎǏǐǑǒǓǔǕǖǗǘǙǚǛǜǝǞǟǠǡǢǣǤǥǦǧǨǩǪǫǬǭǮǯǰDZDzdzǴǵǶǷǸǹǺǻǼǽǾǿ" \ + u"ȀȁȂȃȄȅȆȇȈȉȊȋȌȍȎȏȐȑȒȓȔȕȖȗȘșȚțȜȝȞȟȠȡȢȣȤȥȦȧȨȩȪȫȬȭȮȯȰȱȲȳȴȵȶȷȸȹȺȻȼȽȾȿ" \ + u"ɀɁɂɃɄɅɆɇɈɉɊɋɌɍɎɏ" + ipa_ext = u"ɐɑɒɓɔɕɖɗɘəɚɛɜɝɞɟɠɡɢɣɤɥɦɧɨɩɪɫɬɭɮɯɰɱɲɳɴɵɶɷɸɹɺɻɼɽɾɿʀʁʂʃʄʅʆʇʈʉʊʋʌʍʎʏ" \ + u"ʐʑʒʓʔʕʖʗʘʙʚʛʜʝʞʟʠʡʢʣʤʥʦʧʨʩʪʫʬʭʮʯ" + phonetic = u"ᴀᴁᴂᴃᴄᴅᴆᴇᴈᴉᴊᴋᴌᴍᴎᴏᴐᴑᴒᴓᴔᴕᴖᴗᴘᴙᴚᴛᴜᴝᴞᴟᴠᴡᴢᴣᴤᴥᴦᴧᴨᴩᴪᴫᴬᴭᴮᴯᴰᴱᴲᴳᴴᴵᴶᴷᴸᴹᴺᴻᴼᴽᴾᴿ" \ + u"ᵀᵁᵂᵃᵄᵅᵆᵇᵈᵉᵊᵋᵌᵍᵎᵏᵐᵑᵒᵓᵔᵕᵖᵗᵘᵙᵚᵛᵜᵝᵞᵟᵠᵡᵢᵣᵤᵥᵦᵧᵨᵩᵪᵫᵬᵭᵮᵯᵰᵱᵲᵳᵴᵵᵶᵷᵸᵹᵺᵻᵼᵽᵾᵿ" \ + u"ᶀᶁᶂᶃᶄᶅᶆᶇᶈᶉᶊᶋᶌᶍᶎᶏᶐᶑᶒᶓᶔᶕᶖᶗᶘᶙᶚᶛᶜᶝᶞᶟᶠᶡᶢᶣᶤᶥᶦᶧᶨᶩᶪᶫᶬᶭᶮᶯᶰᶱᶲᶳᶴᶵᶶᶷᶸᶹᶺᶻᶼᶽᶾᶿ" + latin_ext_add = u"ḀḁḂḃḄḅḆḇḈḉḊḋḌḍḎḏḐḑḒḓḔḕḖḗḘḙḚḛḜḝḞḟḠḡḢḣḤḤḦḧḨḩḪḫḬḭḮḯḰḱḲḳḴḵḶḷḸḹḺḻḼḽḾḿ" \ + u"ṀṁṂṃṄṅṆṇṈṉṊṋṌṍṎṏṐṑṒṓṔṕṖṗṘṙṚṛṜṝṞṟṠṡṢṣṤṥṦṧṨṩṪṫṬṭṮṯṰṱṲṳṴṵṶṷṸṹṺṻṼṽṾṿ" \ + u"ẀẁẂẃẄẅẆẇẈẉẊẋẌẍẎẏẐẑẒẓẔẕẖẗẘẙẚẛẜẝẞẟẠạẢảẤấẦầẨẩẪẫẬậẮắẰằẲẳẴẵẶặẸẹẺẻẼẽẾế" \ + u"ỀềỂểỄễỆệỈỉỊịỌọỎỏỐốỒồỔổỖỗỘộỚớỜờỞởỠỡỢợỤụỦủỨứỪừỬửỮữỰựỲỳỴỵỶỷỸỹỺỻỼỽỾỿ" + letter_like = u"℀℁ℂ℃℄℅℆ℇ℈℉ℊℋℌℍℎℏℐℑℒℓ℔ℕ№℗℘ℙℚℛℜℝ℞℟℠℡™℣ℤ℥Ω℧ℨ℩KÅℬℭ℮ℯℰℱℲℳℴℵℶℷℸℹ℺℻ℼℽℾℿ" \ + u"⅀⅁⅂⅃⅄ⅅⅆⅇⅈⅉ⅊⅋⅌⅍ⅎ⅏" + enclosed = u"⒐⒑⒒⒓⒔⒕⒖⒗⒘⒙⒚⒛⒜⒝⒞⒟⒠⒡⒢⒣⒤⒥⒦⒧⒨⒩⒪⒫⒬⒭⒮⒯⒰⒱⒲⒳⒴⒵ⒶⒷⒸⒹⒺⒻⒼⒽⒾⒿⓀⓁⓂⓃⓄⓅⓆⓇⓈⓉⓊⓋⓌⓍⓎⓏ" \ + u"ⓐⓑⓒⓓⓔⓕⓖⓗⓘⓙⓚⓛⓜⓝⓞⓟⓠⓡⓢⓣⓤⓥⓦⓧⓨⓩ⓪⓫⓬⓭⓮⓯" + + print "The following lines show the coverage of Latin characters conversion to ascii." + print "Underscores are characters which currently do not have an ASCII representation." + print + print "latin-1: ",util.textencoding.replace_non_ascii(latin_1) + print "latin-1: ",util.textencoding.replace_non_ascii(latin_1) + print "latin-a: ",util.textencoding.replace_non_ascii(latin_a) + print "latin-b: ",util.textencoding.replace_non_ascii(latin_b) + print "ipa-ext: ",util.textencoding.replace_non_ascii(ipa_ext) + print "phonetic: ",util.textencoding.replace_non_ascii(phonetic) + print "latin-ext-add: ",util.textencoding.replace_non_ascii(latin_ext_add) + print "letter-like: ",util.textencoding.replace_non_ascii(letter_like) + print "enclosed: ",util.textencoding.replace_non_ascii(enclosed) + print + diff --git a/test/test_utils.py b/test/test_utils.py index 2b788f67c..c8e7965a0 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -5,34 +5,6 @@ import unittest from picard import util -class UnaccentTest(unittest.TestCase): - - def test_correct(self): - self.assertEqual(util.unaccent(u"Lukáš"), u"Lukas") - self.assertEqual(util.unaccent(u"Björk"), u"Bjork") - self.assertEqual(util.unaccent(u"Trentemøller"), u"Trentemoller") - self.assertEqual(util.unaccent(u"小室哲哉"), u"小室哲哉") - self.assertEqual(util.unaccent(u"Ænima"), u"AEnima") - self.assertEqual(util.unaccent(u"ænima"), u"aenima") - - def test_incorrect(self): - self.assertNotEqual(util.unaccent(u"Björk"), u"Björk") - self.assertNotEqual(util.unaccent(u"小室哲哉"), u"Tetsuya Komuro") - - -class ReplaceNonAsciiTest(unittest.TestCase): - - def test_correct(self): - self.assertEqual(util.replace_non_ascii(u"Lukáš"), u"Luk__") - self.assertEqual(util.replace_non_ascii(u"Björk"), u"Bj_rk") - self.assertEqual(util.replace_non_ascii(u"Trentemøller"), u"Trentem_ller") - self.assertEqual(util.replace_non_ascii(u"小室哲哉"), u"____") - - def test_incorrect(self): - self.assertNotEqual(util.replace_non_ascii(u"Lukáš"), u"Lukáš") - self.assertNotEqual(util.replace_non_ascii(u"Lukáš"), u"Luk____") - - class ReplaceWin32IncompatTest(unittest.TestCase): def test_correct(self):