#!/usr/bin/python3
# vim:fileencoding=utf-8:sw=4:et

# generate-chinese-variants
#
# Copyright (c) 2013-2018 Mike FABIAN <mfabian@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.

from typing import Any
import re
import logging
import sys

# Unihan_Variants.txt contains the following 2 lines:
#
# U+50DE  kSimplifiedVariant      U+4F2A
# U+50DE  kTraditionalVariant     U+507D U+50DE
#
# This seems to be currently the only case in Unihan_Variants.txt where
# a character which has entries for kTraditionalVariant and
# the same character is listed again among the traditional variants
# is *not* simplified Chinese.
#
# U+50DE 僞 is traditional Chinese.
# U+507D 偽 is also traditional Chinese.
# U+4F2A 伪 is simplified Chinese
#
# This does not cause a problem with the current parsing code
# of Unihan_Variants.txt because the line
#
# U+50DE  kSimplifiedVariant      U+4F2A
#
# is read first and thus the character is already inserted in the
# “VARIANTS_TABLE_ORIG” dictionary as traditional Chinese, which is correct.
# If a character is already in the dictionary and more lines for the
# same character are read from Unihan_Variants.txt, these extra lines
# are ignored.
#
# But maybe for some corner cases more tweaking of the code is
# necessary. One can also add overrides manually to the
# initial content of “VARIANTS_TABLE_ORIG”.

VARIANTS_TABLE_ORIG = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
    #
    # overrides can be added manually here. For example the following
    # line marks the 〇 character as used in both
    # simplified and traditional Chinese:
    u'〇': 3 # simplified *and* traditional Chinese
    }

# keep the lines from Unihan_Variants.txt which were used for debugging
VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED = {}

def read_unihan_variants(unihan_variants_file) -> None:
    '''
    Read the Unihan_Variants.txt file downloaded  from Unicode.org.
    '''
    for line in unihan_variants_file:
        line = line.strip()
        if not re.match('^#', line):
            if re.search('(kTraditionalVariant|kSimplifiedVariant)', line):
                match = re.match(r'^U\+([0-9A-F]{4,5})', line)
                if match:
                    char = chr(int(match.group(1), 16))
                    category = 0 # should never  stay at this value
                    if re.match(re.escape(match.group(0))
                                + r'.*'
                                + re.escape(match.group(0)), line):
                        # is both simplified and traditional
                        category = 1 | 1 << 1
                    elif re.search('kTraditionalVariant', line):
                        category = 1 # simplified only
                    elif re.search('kSimplifiedVariant', line):
                        category = 1 << 1 # traditional only
                    logging.debug(
                        'char=%s category=%d line=%s',
                        char, category, line)
                    if not char in VARIANTS_TABLE_ORIG:
                        VARIANTS_TABLE_ORIG[char] = category
                    if (not char
                            in VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED):
                        VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED[
                            char] = line

def detect_chinese_category_old(phrase: str) -> int:
    '''
    Old function using encoding conversion to guess whether
    a text is simplified Chinese, traditional Chinese, both,
    or unknown. Does not work well, is included here for reference
    and for comparing with the results of the new, improved function
    using the data from the Unihan database.
    '''
    # this is the bitmask we will use,
    # from low to high, 1st bit is simplified Chinese,
    # 2nd bit is traditional Chinese,
    # 3rd bit means out of gbk
    category = 0
    # make sure that we got a unicode string
    tmp_phrase = ''.join(re.findall(u'['
                                    + u'\u4E00-\u9FCB'
                                    + u'\u3400-\u4DB5'
                                    + u'\uF900-\uFaFF'
                                    + u'\U00020000-\U0002A6D6'
                                    + u'\U0002A700-\U0002B734'
                                    + u'\U0002B740-\U0002B81D'
                                    + u'\U0002F800-\U0002FA1D'
                                    + u']+',
                                    phrase))
    # first whether in gb2312
    try:
        tmp_phrase.encode('gb2312')
        category |= 1
    except:
        if u'〇' in tmp_phrase:
            # we add '〇' into SC as well
            category |= 1
    # second check big5-hkscs
    try:
        tmp_phrase.encode('big5hkscs')
        category |= 1 << 1
    except:
        # then check whether in gbk,
        if category & 1:
            # already know in SC
            pass
        else:
            # need to check
            try:
                tmp_phrase.encode('gbk')
                category |= 1
            except:
                # not in gbk
                pass
    # then set for 3rd bit, if not in SC and TC
    if not category & (1 | 1 << 1):
        category |= (1 << 2)
    return category

def write_variants_script(script_file) -> None:
    '''
    Write the generated Python script.
    '''
    script_file.write('''#!/usr/bin/python
# vim:fileencoding=utf-8:sw=4:et

# auto-generated by “generate-chinese-variants.py”, do not edit here!
#
# Copyright (c) 2013 Mike FABIAN <mfabian@redhat.com>
#
# This library is free software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public
# License as published by the Free Software Foundation; either
# version 3.0 of the License, or (at your option) any later version.
#
# This library is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
# Lesser General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
''')

    script_file.write('''
import sys
''')

    script_file.write('''
VARIANTS_TABLE = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
''')

    for phrase in sorted(VARIANTS_TABLE_ORIG):
        script_file.write(
            "    u'" + phrase + "': "
            + "%s" %VARIANTS_TABLE_ORIG[phrase] + ",\n")

    script_file.write('''    }
''')

    script_file.write('''
def detect_chinese_category(phrase: str) -> int:
    \'\'\'
    New function using Unihan data to guess whether a text is
    simplified Chinese, traditional Chinese, both, or something rare
    like a mixture of exclusively simplified with exclusively traditional
    characters.

    Meaning of the bits in the category value returned by this function:
    1 = 1 << 0       simplified Chinese
    2 = 1 << 1       traditional Chinese
    3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    4 = 1 << 2       mixture of simplified and traditional Chinese
    \'\'\'
    # make sure that we got a unicode string
    if phrase in VARIANTS_TABLE:
        # the complete phrase is in VARIANTS_TABLE, just return the
        # value found:
        return VARIANTS_TABLE[phrase]
    category = 0xFF
    for char in phrase:
        if char in VARIANTS_TABLE:
            category &= VARIANTS_TABLE[char]
        else:
            # If it is not listed in VARIANTS_TABLE, assume it is
            # both simplified and traditional Chinese.
            # It could be something non-Chinese as well then, but
            # if it is non-Chinese, it should also be allowed to
            # occur in any Chinese text and thus classified as
            # both simplified *and* traditional Chinese (the emoji
            # table for example uses many non-Chinese characters)
            category &= (1 | 1 << 1)
    if category == 0:
        # If category is 0 after binary & of the categories of all the
        # characters in the phrase, it means that the phrase contained
        # exclusively simplified *and* exclusively traditional
        # characters at the same time.  For example if the phrase is
        # “乌烏” then “乌” gets category 1 (simplified Chinese)
        # and “烏” gets category 2 (traditional Chinese), the result
        # of the binary & is thus 0. In that case, classify it as
        # category 4 which is for weird, excentric, rare stuff. If the
        # user selects one of the modes “all characters but
        # simplified Chinese first” or “all characters but
        # traditional Chinese first”, phrases with category 4 will be
        # shown but filtered to be shown only at the end of the
        # candidate list.
        category = 1 << 2
    return category
''')

TEST_DATA = {
    # Meaning of the bits in the values:
    # 1 = 1 << 0       simplified Chinese
    # 2 = 1 << 1       traditional Chinese
    # 3 = (1 | 1 << 1) used both in simplified *and* traditional Chinese
    # 4 = 1 << 2       mixture of simplified and traditional Chinese
    u'乌': 1,
    u'烏': 2,
    u'晞': 3,
    u'䖷': 3,
    u'乌烏': 4,
    u'a☺α乌': 1,
    u'a☺α烏': 2,
    u'台': 3,
    u'同': 3,
    u'表': 3, # U+8868
    u'面': 3, # U+9762
    # Characters below this comments probably have buggy entries
    # in Unihan_Variants.txt:
    u'覆': 3, # U+8986
    u'杰': 3, # U+6770
    u'系': 3, # U+7CFB
    u'乾': 3, # U+4E7E
    u'著': 3, # U+8457 Patch by Heiher <r@hev.cc>
    u'只': 3, # U+53EA, see: https://github.com/kaio/ibus-table/issues/74
    # Problems reported in https://github.com/ibus/ibus/issues/2323
    u'着': 3, # U+7740, used in HK
    u'枱': 3, # U+67B1, used in HK (correct already, no SC variant entry in Unihan_Variants.txt)
    u'云': 3, # U+4E91, used in HK and TW
    u'裡': 3, # U+88E1, (Untypable in S) used in all places same meaning as 裏
    u'復': 3, # U+5FA9, (Untypable in S) used in all places same meaning in S, diff in T
    u'采': 3, # U+91C7, (Untypable in T) used in Hong Kong, not sure about TW
    # http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi has 采, i.e. probably
    # it is used in TW
    u'吓': 3, # U+5413, (Untypable in T) used in Cantonese.
    u'尸': 3, # U+5C38, (Untypable in T) idk where it is used, but Cangjie has that as a radical.
    u'揾': 3, # U+63FE, used in HK
    # (TW seems to use only 搵, see http://dict.revised.moe.edu.tw/cgi-bin/cbdic/gsweb.cgi)
    u'栗': 3, # U+6817 https://github.com/mike-fabian/ibus-table/issues/95
    u'了': 3, # U+4E86 https://github.com/mike-fabian/ibus-table/issues/96
    u'伙': 3, # U+4F19 https://github.com/mike-fabian/ibus-table/issues/96
    u'借': 3, # U+501F https://github.com/mike-fabian/ibus-table/issues/96
    u'冬': 3, # U+51AC https://github.com/mike-fabian/ibus-table/issues/96
    u'千': 3, # U+5343 https://github.com/mike-fabian/ibus-table/issues/96
    u'卜': 3, # U+535C https://github.com/mike-fabian/ibus-table/issues/96
    u'卷': 3, # U+5377 https://github.com/mike-fabian/ibus-table/issues/96
    u'吁': 3, # U+5401 https://github.com/mike-fabian/ibus-table/issues/96
    u'合': 3, # U+5408 https://github.com/mike-fabian/ibus-table/issues/96
    u'回': 3, # U+56DE https://github.com/mike-fabian/ibus-table/issues/96
    u'夥': 3, # U+5925 https://github.com/mike-fabian/ibus-table/issues/96
    u'姜': 3, # U+59DC https://github.com/mike-fabian/ibus-table/issues/96
    u'家': 3, # U+5BB6 https://github.com/mike-fabian/ibus-table/issues/96
    u'才': 3, # U+624D https://github.com/mike-fabian/ibus-table/issues/96
    u'折': 3, # U+6298 https://github.com/mike-fabian/ibus-table/issues/96
    u'摺': 3, # U+647A https://github.com/mike-fabian/ibus-table/issues/96
    u'旋': 3, # U+65CB https://github.com/mike-fabian/ibus-table/issues/96
    u'朱': 3, # U+6731 https://github.com/mike-fabian/ibus-table/issues/96
    u'灶': 3, # U+7076 https://github.com/mike-fabian/ibus-table/issues/96
    u'秋': 3, # U+79CB https://github.com/mike-fabian/ibus-table/issues/96
    u'蒙': 3, # U+8499 https://github.com/mike-fabian/ibus-table/issues/96
    u'蔑': 3, # U+8511 https://github.com/mike-fabian/ibus-table/issues/96
    u'霉': 3, # U+9709 https://github.com/mike-fabian/ibus-table/issues/96
    u'沄': 3, # U+6C84 https://github.com/mike-fabian/ibus-table/issues/97
    # https://dict.revised.moe.edu.tw/search.jsp?md=1&word=%E6%B2%84&qMd=0&qCol=1
    u'干': 3, # U+5E72 https://github.com/mike-fabian/ibus-table/issues/97
    # See https://github.com/mike-fabian/ibus-table/issues/100 especially:
    # https://github.com/mike-fabian/ibus-table/issues/100#issuecomment-1020358521
    # These characters were classified as “Simplified only” but they are in
    # this dictionary: https://dict.revised.moe.edu.tw/
    u'时': 3, # U+65F6 https://github.com/mike-fabian/ibus-table/issues/100
    u'旷': 3, # U+65F7 ...
    u'晒': 3, # U+6652 ...
    u'幂': 3, # U+5E42 ...
    u'胆': 3, # U+80C6 ...
    u'册': 3, # U+518C ...
    u'脚': 3, # U+811A ...
    u'胜': 3, # U+80DC ...
    u'脉': 3, # U+8109 ...
    u'膑': 3, # U+8191 ...
    u'网': 3, # U+7F51 ...
    u'删': 3, # U+5220 ...
    u'腼': 3, # U+817C ...
    u'脍': 3, # U+810D ...
    u'腭': 3, # U+816D ...
    u'腊': 3, # U+814A ...
    u'眦': 3, # U+7726 ...
    u'肮': 3, # U+80AE ...
    u'谷': 3, # U+8C37 ...
    u'兑': 3, # U+5151 ...
    u'单': 3, # U+5355 ...
    u'栅': 3, # U+6805 ...
    u'松': 3, # U+677E ...
    u'梦': 3, # U+68A6 ...
    u'权': 3, # U+6743 ...
    u'楼': 3, # U+697C ...
    u'栀': 3, # U+6800 ...
    u'机': 3, # U+673A ...
    u'栖': 3, # U+6816 ...
    u'杆': 3, # U+6746 ...
    u'标': 3, # U+6807 ...
    u'构': 3, # U+6784 ...
    u'柜': 3, # U+67DC ...
    u'朴': 3, # U+6734 ...
    u'温': 3, # U+6E29 ...
    u'泪': 3, # U+6CEA ...
    u'对': 3, # U+5BF9 ...
    u'双': 3, # U+53CC ...
    u'叠': 3, # U+53E0 ...
    u'滩': 3, # U+6EE9 ...
    u'洼': 3, # U+6D3C ...
    u'没': 3, # U+6CA1 ...
    u'沪': 3, # U+6CAA ...
    u'戏': 3, # U+620F ...
    u'浅': 3, # U+6D45 ...
    u'沪': 3, # U+6CAA ...
    u'滨': 3, # U+6EE8 ...
    u'劝': 3, # U+529D ...
    u'沈': 3, # U+6C88 ...
    u'渊': 3, # U+6E0A ...
    u'洒': 3, # U+6D12 ...
    u'㳽': 3, # U+3CFD ...
    u'欢': 3, # U+6B22 ...
    u'难': 3, # U+96BE ...
    u'涂': 3, # U+6D82 ...
    u'涛': 3, # U+6D9B ...
    u'汹': 3, # U+6C79 ...
    u'滦': 3, # U+6EE6 ...
    u'湾': 3, # U+6E7E ...
    u'滚': 3, # U+6EDA ...
    u'漓': 3, # U+6F13 ...
    u'尝': 3, # U+5C1D ...
    u'党': 3, # U+515A ...
    u'誉': 3, # U+8A89 ...
    u'粮': 3, # U+7CAE ...
    u'糇': 3, # U+7CC7 ...
    u'娄': 3, # U+5A04 ...
    u'炉': 3, # U+7089 ...
    u'烛': 3, # U+70DB ...
    u'灯': 3, # U+706F ...
    u'烩': 3, # U+70E9 ...
    u'当': 3, # U+5F53 ...
    u'烬': 3, # U+70EC ...
    u'数': 3, # U+6570 ...
    u'烟': 3, # U+70DF ...
    u'声': 3, # U+58F0 ...
    u'壳': 3, # U+58F3 ...
    u'块': 3, # U+5757 ...
    u'坂': 3, # U+5742 ...
    u'却': 3, # U+5374 ...
    u'坏': 3, # U+574F ...
    u'坛': 3, # U+575B ...
    u'赶': 3, # U+8D76 ...
    u'趋': 3, # U+8D8B ...
    u'制': 3, # U+5236 ...
    u'秃': 3, # U+79C3 ...
    u'种': 3, # U+79CD ...
    u'秆': 3, # U+79C6 ...
    u'称': 3, # U+79F0 ...
    u'稳': 3, # U+7A33 ...
    u'篓': 3, # U+7BD3 ...
    u'丢': 3, # U+4E22 ...
    u'笔': 3, # U+7B14 ...
    u'躯': 3, # U+8EAF ...
    u'么': 3, # U+4E48 ...
    u'乔': 3, # U+4E54 ...
    u'筑': 3, # U+7B51 ...
    u'惩': 3, # U+60E9 ...
    u'几': 3, # U+51E0 ...
    u'凤': 3, # U+51E4 ...
    u'风': 3, # U+98CE ...
    u'御': 3, # U+5FA1 ...
    u'刮': 3, # U+522E ...
    u'乱': 3, # U+4E71 ...
    u'辞': 3, # U+8F9E ...
    u'笋': 3, # U+7B0B ...
    u'衅': 3, # U+8845 ...
    u'毡': 3, # U+6BE1 ...
    u'鼹': 3, # U+9F39 ...
    u'乐': 3, # U+4E50 ...
    u'箩': 3, # U+7BA9 ...
    u'篱': 3, # U+7BF1 ...
    u'术': 3, # U+672F ...
    u'应': 3, # U+5E94 ...
    u'祢': 3, # U+7962 ...
    u'祷': 3, # U+7977 ...
    u'礼': 3, # U+793C ...
    u'庄': 3, # U+5E84 ...
    u'咸': 3, # U+54B8 ...
    u'庐': 3, # U+5E90 ...
    u'义': 3, # U+4E49 ...
    u'参': 3, # U+53C2 ...
    u'划': 3, # U+5212 ...
    u'庙': 3, # U+5E99 ...
    u'减': 3, # U+51CF ...
    u'冲': 3, # U+51B2 ...
    u'准': 3, # U+51C6 ...
    u'凑': 3, # U+51D1 ...
    u'况': 3, # U+51B5 ...
    u'凉': 3, # U+51C9 ...
    u'户': 3, # U+6237 ...
    u'献': 3, # U+732E ...
    u'穷': 3, # U+7A77 ...
    u'帘': 3, # U+5E18 ...
    u'窃': 3, # U+7A83 ...
    u'灾': 3, # U+707E ...
    u'宝': 3, # U+5B9D ...
    u'宁': 3, # U+5B81 ...
    u'宾': 3, # U+5BBE ...
    u'胡': 3, # U+80E1 ...
    u'克': 3, # U+514B ...
    u'实': 3, # U+5B9E ...
    u'郁': 3, # U+90C1 ...
    u'刹': 3, # U+5239 ...
    u'瘘': 3, # U+7618 ...
    u'犹': 3, # U+72B9 ...
    u'猪': 3, # U+732A ...
    u'独': 3, # U+72EC ...
    u'猕': 3, # U+7315 ...
    u'狯': 3, # U+72EF ...
    u'猫': 3, # U+732B ...
    u'猬': 3, # U+732C ...
    u'夸': 3, # U+5938 ...
    u'疴': 3, # U+75B4 ...
    u'症': 3, # U+75C7 ...
    u'疱': 3, # U+75B1 ...
    u'办': 3, # U+529E ...
    u'刾': 3, # U+523E ...
    u'痒': 3, # U+75D2 ...
    u'袜': 3, # U+889C ...
    u'隶': 3, # U+96B6 ...
    u'袄': 3, # U+8884 ...
    u'蝼': 3, # U+877C ...
    u'蚁': 3, # U+8681 ...
    u'虾': 3, # U+867E ...
    u'虬': 3, # U+866C ...
    u'师': 3, # U+5E08 ...
    u'归': 3, # U+5F52 ...
    u'壮': 3, # U+58EE ...
    u'虫': 3, # U+866B ...
    u'鼗': 3, # U+9F17 ...
    u'儿': 3, # U+513F ...
    u'电': 3, # U+7535 ...
    u'补': 3, # U+8865 ...
    u'厩': 3, # U+53A9 ...
    u'霡': 3, # U+9721 ...
    u'晋': 3, # U+664B ...
    u'于': 3, # U+4E8E ...
    u'珐': 3, # U+73D0 ...
    u'压': 3, # U+538B ...
    u'环': 3, # U+73AF ...
    u'琼': 3, # U+743C ...
    u'厂': 3, # U+5382 ...
    u'动': 3, # U+52A8 ...
    u'蚕': 3, # U+8695 ...
    u'历': 3, # U+5386 ...
    u'无': 3, # U+65E0 ...
    u'厉': 3, # U+5389 ...
    u'厨': 3, # U+53A8 ...
    u'厦': 3, # U+53A6 ...
    u'亏': 3, # U+4E8F ...
    u'殡': 3, # U+6BA1 ...
    u'两': 3, # U+4E24 ...
    u'碍': 3, # U+788D ...
    u'确': 3, # U+786E ...
    u'万': 3, # U+4E07 ...
    u'励': 3, # U+52B1 ...
    u'开': 3, # U+5F00 ...
    u'厮': 3, # U+53AE ...
    u'画': 3, # U+753B ...
    u'厘': 3, # U+5398 ...
    u'堕': 3, # U+5815 ...
    u'弹': 3, # U+5F39 ...
    u'孙': 3, # U+5B59 ...
    u'尔': 3, # U+5C14 ...
    u'丑': 3, # U+4E11 ...
    u'际': 3, # U+9645 ...
    u'隐': 3, # U+9690 ...
    u'随': 3, # U+968F ...
    u'强': 3, # U+5F3A ...
    u'争': 3, # U+4E89 ...
    u'皱': 3, # U+76B1 ...
    u'龟': 3, # U+9F9F ...
    u'复': 3, # U+590D ...
    u'内': 3, # U+5185 ...
    u'佣': 3, # U+4F63 ...
    u'叙': 3, # U+53D9 ...
    u'体': 3, # U+4F53 ...
    u'仅': 3, # U+4EC5 ...
    u'籴': 3, # U+7C74 ...
    u'偻': 3, # U+507B ...
    u'凭': 3, # U+51ED ...
    u'隽': 3, # U+96BD ...
    u'仪': 3, # U+4EEA ...
    u'优': 3, # U+4F18 ...
    u'矫': 3, # U+77EB ...
    u'侠': 3, # U+4FA0 ...
    u'个': 3, # U+4E2A ...
    u'舍': 3, # U+820D ...
    u'会': 3, # U+4F1A ...
    u'气': 3, # U+6C14 ...
    u'从': 3, # U+4ECE ...
    u'伤': 3, # U+4F24 ...
    u'价': 3, # U+4EF7 ...
    u'侩': 3, # U+4FA9 ...
    u'众': 3, # U+4F17 ...
    u'偬': 3, # U+506C ...
    u'俦': 3, # U+4FE6 ...
    u'仆': 3, # U+4EC6 ...
    u'愠': 3, # U+6120 ...
    u'惧': 3, # U+60E7 ...
    u'匀': 3, # U+5300 ...
    u'恒': 3, # U+6052 ...
    u'怀': 3, # U+6000 ...
    u'怜': 3, # U+601C ...
    u'够': 3, # U+591F ...
    u'恼': 3, # U+607C ...
    u'担': 3, # U+62C5 ...
    u'静': 3, # U+9759 ...
    u'摊': 3, # U+644A ...
    u'撑': 3, # U+6491 ...
    u'挡': 3, # U+6321 ...
    u'挂': 3, # U+6302 ...
    u'热': 3, # U+70ED ...
    u'势': 3, # U+52BF ...
    u'蛰': 3, # U+86F0 ...
    u'丰': 3, # U+4E30 ...
    u'艳': 3, # U+8273 ...
    u'摈': 3, # U+6448 ...
    u'寿': 3, # U+5BFF ...
    u'执': 3, # U+6267 ...
    u'抛': 3, # U+629B ...
    u'挟': 3, # U+631F ...
    u'帮': 3, # U+5E2E ...
    u'麦': 3, # U+9EA6 ...
    u'挽': 3, # U+633D ...
    u'携': 3, # U+643A ...
    u'据': 3, # U+636E ...
    u'报': 3, # U+62A5 ...
    u'掷': 3, # U+63B7 ...
    u'摆': 3, # U+6446 ...
    u'扑': 3, # U+6251 ...
    u'喽': 3, # U+55BD ...
    u'响': 3, # U+54CD ...
    u'听': 3, # U+542C ...
    u'咏': 3, # U+548F ...
    u'叶': 3, # U+53F6 ...
    u'咤': 3, # U+54A4 ...
    u'黾': 3, # U+9EFE ...
    u'踪': 3, # U+8E2A ...
    u'吴': 3, # U+5434 ...
    u'蹰': 3, # U+8E70 ...
    u'踊': 3, # U+8E0A ...
    u'号': 3, # U+53F7 ...
    u'嘱': 3, # U+5631 ...
    u'别': 3, # U+522B ...
    u'啰': 3, # U+5570 ...
    u'屡': 3, # U+5C61 ...
    u'属': 3, # U+5C5E ...
    u'职': 3, # U+804C ...
    u'联': 3, # U+8054 ...
    u'耻': 3, # U+803B ...
    u'区': 3, # U+533A ...
    u'届': 3, # U+5C4A ...
    u'昼': 3, # U+663C ...
    u'医': 3, # U+533B ...
    u'尽': 3, # U+5C3D ...
    u'剧': 3, # U+5267 ...
    u'荣': 3, # U+8363 ...
    u'劳': 3, # U+52B3 ...
    u'范': 3, # U+8303 ...
    u'盖': 3, # U+76D6 ...
    u'芦': 3, # U+82A6 ...
    u'蘖': 3, # U+8616 ...
    u'荆': 3, # U+8346 ...
    u'荐': 3, # U+8350 ...
    u'郑': 3, # U+90D1 ...
    u'苏': 3, # U+82CF ...
    u'黄': 3, # U+9EC4 ...
    u'苹': 3, # U+82F9 ...
    u'芸': 3, # U+82B8 ...
    u'莅': 3, # U+8385 ...
    u'葱': 3, # U+8471 ...
    u'并': 3, # U+5E76 ...
    u'荆': 3, # U+8346 ...
    u'萝': 3, # U+841D ...
    u'蔂': 3, # U+8502 ...
    u'岁': 3, # U+5C81 ...
    u'粜': 3, # U+7C9C ...
    u'姗': 3, # U+59D7 ...
    u'断': 3, # U+65AD ...
    u'娇': 3, # U+5A07 ...
    u'姹': 3, # U+59F9 ...
    u'奸': 3, # U+5978 ...
    u'娱': 3, # U+5A31 ...
    u'困': 3, # U+56F0 ...
    u'里': 3, # U+91CC ...
    u'图': 3, # U+56FE ...
    u'罢': 3, # U+7F62 ...
    u'罗': 3, # U+7F57 ...
    u'园': 3, # U+56ED ...
    u'累': 3, # U+7D2F ...
    u'兑': 3, # U+5151 ...
    u'栀': 3, # U+6800 ...
    u'构': 3, # U+6784 ...
    u'双': 3, # U+53CC ...
    u'洒': 3, # U+6D12 ...
    u'滚': 3, # U+6EDA ...
    u'娄': 3, # U+5A04 ...
    u'壳': 3, # U+58F3 ...
    u'术': 3, # U+672F ...
    u'庐': 3, # U+5E90 ...
    u'义': 3, # U+4E49 ...
    u'郁': 3, # U+90C1 ...
    u'刹': 3, # U+5239 ...
    u'疴': 3, # U+75B4 ...
    u'压': 3, # U+538B ...
    u'历': 3, # U+5386 ...
    u'亏': 3, # U+4E8F ...
    u'随': 3, # U+968F ...
    u'内': 3, # U+5185 ...
    u'仪': 3, # U+4EEA ...
    u'气': 3, # U+6C14 ...
    u'掷': 3, # U+63B7 ...
    u'别': 3, # U+522B ...
    u'荣': 3, # U+8363 ...
    u'郑': 3, # U+90D1 ...
    u'并': 3, # U+5E76 ...
    u'萝': 3, # U+841D ...
    u'困': 3, # U+56F0 ...
    u'堕': 3, # U+5815 ...
    u'头': 3, # U+5934 ...
    u'欤': 3, # U+6B24 ...
    u'逊': 3, # U+900A ...
    u'点': 3, # U+70B9 ...
    u'韵': 3, # U+97F5 ...
    u'銮': 3, # U+92AE ...
    u'栾': 3, # U+683E ...
    u'衮': 3, # U+886E ...
    u'蛮': 3, # U+86EE ...
    u'弯': 3, # U+5F2F ...
    u'孪': 3, # U+5B6A ...
    u'递': 3, # U+9012 ...
    u'脔': 3, # U+8114 ...
    u'恋': 3, # U+604B ...
    u'挛': 3, # U+631B ...
    u'峦': 3, # U+5CE6 ...
    u'娈': 3, # U+5A08 ...
    u'过': 3, # U+8FC7 ...
    u'广': 3, # U+5E7F ...
    u'选': 3, # U+9009 ...
    u'彦': 3, # U+5F66 ...
    u'迁': 3, # U+8FC1 ...
    u'适': 3, # U+9002 ...
    u'弃': 3, # U+5F03 ...
    u'斗': 3, # U+6597 ...
    u'头': 3, # U+5934 ...
    u'刘': 3, # U+5218 ...
    u'斋': 3, # U+658B ...
    u'边': 3, # U+8FB9 ...
    u'还': 3, # U+8FD8 ...
    u'远': 3, # U+8FDC ...
    u'欤': 3, # U+6B24 ...
    u'逊': 3, # U+900A ...
    u'迩': 3, # U+8FE9 ...
    u'点': 3, # U+70B9 ...
    u'战': 3, # U+6218 ...
    u'逻': 3, # U+903B ...
    u'禀': 3, # U+7980 ...
    u'这': 3, # U+8FD9 ...
    u'迹': 3, # U+8FF9 Last of https://github.com/mike-fabian/ibus-table/issues/100
    }

def test_detection(generated_script) -> int:
    '''
    Test whether the generated script does the detection correctly.

    Returns the number of errors found.
    '''
    logging.info('Testing detection ...')
    error_count = 0
    for phrase in TEST_DATA:
        if (generated_script.detect_chinese_category(phrase)
                != TEST_DATA[phrase]):
            print('phrase', phrase, repr(phrase),
                  'detected as',
                  generated_script.detect_chinese_category(phrase),
                  'should have been', TEST_DATA[phrase],
                  'FAIL.')
            error_count += 1
        else:
            logging.info('phrase=%s %s detected as %d PASS.',
                         phrase,
                         repr(phrase),
                         TEST_DATA[phrase])
    return error_count

def compare_old_new_detection(phrase, generated_script) -> None:
    '''
    Only for debugging.

    Compares results of the Chinese category detection using the
    old and the new function.
    '''
    if (detect_chinese_category_old(phrase)
            != generated_script.detect_chinese_category(phrase)):
        logging.debug(
            '%s %s old=%d new=%d',
            phrase.encode('utf-8'),
            repr(phrase),
            detect_chinese_category_old(phrase),
            generated_script.detect_chinese_category(phrase))
        if phrase in VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED:
            logging.debug(
                VARIANTS_TABLE_ORIG_UNIHAN_VARIANTS_ENTRY_USED[phrase])

def parse_args() -> Any:
    '''Parse the command line arguments'''
    import argparse
    parser = argparse.ArgumentParser(
        description=(
            'Generate a script containing a table and a function '
            + 'to check whether a string of Chinese characters '
            + 'is simplified or traditional'))
    parser.add_argument('-i', '--inputfilename',
                        nargs='?',
                        type=str,
                        default='./Unihan_Variants.txt',
                        help='input file, default is ./Unihan_Variants.txt')
    parser.add_argument('-o', '--outputfilename',
                        nargs='?',
                        type=str,
                        default='./chinese_variants.py',
                        help='output file, default is ./chinese_variants.py')
    parser.add_argument('-d', '--debug',
                        action='store_true',
                        help='print debugging output')
    return parser.parse_args()

def main() -> None:
    '''Main program'''
    args = parse_args()
    log_level = logging.INFO
    if args.debug:
        log_level = logging.DEBUG
    logging.basicConfig(format="%(levelname)s: %(message)s", level=log_level)
    with open(args.inputfilename, 'r') as inputfile:
        logging.info("input file=%s", inputfile)
        read_unihan_variants(inputfile)
    with open(args.outputfilename, 'w') as outputfile:
        logging.info("output file=%s", outputfile)
        write_variants_script(outputfile)

    import imp
    generated_script = imp.load_source('dummy', args.outputfilename)

    logging.info('Testing detection ...')
    error_count = test_detection(generated_script)
    if error_count:
        logging.info('FAIL: %s tests failed, exiting ...', error_count)
        exit(1)
    else:
        logging.info('PASS: All tests passed.')

    for phrase in generated_script.VARIANTS_TABLE: # type: ignore
        compare_old_new_detection(phrase, generated_script)

if __name__ == '__main__':
    main()
