From 26f563565c0f254b1fd5f64023ed586ec55abbc7 Mon Sep 17 00:00:00 2001 From: Martin Jansa Date: Thu, 12 Dec 2019 07:22:55 -0800 Subject: [PATCH 1/3] *.py: use python3 explicitly and migrate with 2to3 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * python2 is EOL, use python3 https://python3statement.org/ * fixes build on hosts without python (2) installed rm -f android.db; \ ../../../../git/data/db/android/create_db.py ../../../../git/data/db/android/rawdict_utf16_65105_freq.txt | work/qemux86-webos-linux/pyzy/0.1.0+gitAUTOINC+d774746656-r0/recipe-sysroot-native/usr/bin/sqlite3 android.db || \ ( rm -f android.db ; exit 1 ) /usr/bin/env: ‘python’: No such file or directory hosttools/mkdir -p 'work/qemux86-webos-linux/pyzy/0.1.0+gitAUTOINC+d774746656-r0/image/usr/share/pyzy/db' hosttools/install -c -m 644 ../../../../git/data/db/android/android.db 'work/qemux86-webos-linux/pyzy/0.1.0+gitAUTOINC+d774746656-r0/image/usr/share/pyzy/db' hosttools/install: cannot stat '../../../../git/data/db/android/android.db': No such file or directory Makefile:395: recipe for target 'install-main_dbDATA' failed make[4]: *** [install-main_dbDATA] Error 1 Signed-off-by: Martin Jansa --- data/db/android/create_db.py | 28 ++++++++++++++-------------- scripts/addheader.py | 8 ++++---- scripts/update-simptrad-table.py | 24 ++++++++++++------------ tools/googlecode_upload.py | 24 ++++++++++++------------ 4 files changed, 42 insertions(+), 42 deletions(-) diff --git a/data/db/android/create_db.py b/data/db/android/create_db.py index 4fff1d0..2da5bff 100755 --- a/data/db/android/create_db.py +++ b/data/db/android/create_db.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 from pydict import * from id import * from valid_hanzi import * @@ -9,7 +9,7 @@ def get_sheng_yun(pinyin): return None, None if pinyin == "ng": return "", "en" - for i in xrange(2, 0, -1): + for i in range(2, 0, -1): t = pinyin[:i] if t in SHENGMU_DICT: return t, pinyin[len(t):] @@ -17,13 +17,13 @@ def get_sheng_yun(pinyin): def read_phrases(filename): buf = file(filename).read() - buf = unicode(buf, "utf16") + buf = str(buf, "utf16") buf = buf.strip() - for l in buf.split(u'\n'): - hanzi, freq, flag, pinyin = l.split(u' ', 3) + for l in buf.split('\n'): + hanzi, freq, flag, pinyin = l.split(' ', 3) freq = float(freq) pinyin = pinyin.split() - if any(map(lambda c: c not in valid_hanzi, hanzi)): + if any([c not in valid_hanzi for c in hanzi]): continue yield hanzi, freq, pinyin @@ -33,9 +33,9 @@ def create_db(filename): # con.execute ("PRAGMA synchronous = NORMAL;") # con.execute ("PRAGMA temp_store = MEMORY;") # con.execute ("PRAGMA default_cache_size = 5000;") - print "PRAGMA synchronous = NORMAL;" - print "PRAGMA temp_store = MEMORY;" - print "PRAGMA default_cache_size = 5000;" + print("PRAGMA synchronous = NORMAL;") + print("PRAGMA temp_store = MEMORY;") + print("PRAGMA default_cache_size = 5000;") sql = "CREATE TABLE py_phrase_%d (phrase TEXT, freq INTEGER, %s);" @@ -44,7 +44,7 @@ def create_db(filename): for j in range(0, i + 1): column.append ("s%d INTEGER" % j) column.append ("y%d INTEGER" % j) - print sql % (i, ",".join(column)) + print(sql % (i, ",".join(column))) # con.execute(sql % (i, column)) # con.commit() @@ -60,7 +60,7 @@ def create_db(filename): records_new.append((hanzi, i, pinyin)) records_new.reverse() - print "BEGIN;" + print("BEGIN;") insert_sql = "INSERT INTO py_phrase_%d VALUES (%s);" for hanzi, freq, pinyin in records_new: columns = [] @@ -72,9 +72,9 @@ def create_db(filename): values = "'%s', %d, %s" % (hanzi.encode("utf8"), freq, ",".join(map(str,columns))) sql = insert_sql % (len(hanzi) - 1, values) - print sql - print "COMMIT;" - print "VACUUM;" + print(sql) + print("COMMIT;") + print("VACUUM;") def main(): create_db(sys.argv[1]) diff --git a/scripts/addheader.py b/scripts/addheader.py index 09e628e..7874105 100644 --- a/scripts/addheader.py +++ b/scripts/addheader.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys def add_header(name, header): @@ -6,13 +6,13 @@ def add_header(name, header): lines = fin.readlines() with file(name, "w") as fout: for l in header: - print >> fout, l, + print(l, end=' ', file=fout) if lines[0].startswith("/*") and lines[0].endswith("*/\n"): pass else: - print >> fout, lines[0], + print(lines[0], end=' ', file=fout) for l in lines[1:]: - print >> fout, l, + print(l, end=' ', file=fout) def main(): with file("header") as f: diff --git a/scripts/update-simptrad-table.py b/scripts/update-simptrad-table.py index 64d3d45..39c5e60 100755 --- a/scripts/update-simptrad-table.py +++ b/scripts/update-simptrad-table.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 import sys sys.path.append(".") @@ -6,7 +6,7 @@ from valid_hanzi import * def convert(s, d, n): - out = u"" + out = "" end = len(s) begin = 0 while begin < end: @@ -20,9 +20,9 @@ def convert(s, d, n): return out def filter_more(records, n): - han = filter(lambda (k, v): len(k) <= n, records) + han = [k_v1 for k_v1 in records if len(k_v1[0]) <= n] hand = dict(han) - hanm = filter(lambda (k, v): convert(k, hand, n) != v, records) + hanm = [k_v2 for k_v2 in records if convert(k_v2[0], hand, n) != k_v2[1]] return hanm + han def filter_func(args): @@ -46,24 +46,24 @@ def filter_func(args): return True def get_records(): - records = zh2Hant.items() + records = list(zh2Hant.items()) - records = filter(filter_func, records) + records = list(filter(filter_func, records)) - maxlen = max(map(lambda (k,v): len(k), records)) + maxlen = max([len(k_v[0]) for k_v in records]) for i in range(1, maxlen - 1): records = filter_more(records, i) - records = map(lambda (k, v): (k.encode("utf8"), v.encode("utf8")), records) + records = [(k_v3[0].encode("utf8"), k_v3[1].encode("utf8")) for k_v3 in records] records.sort() return maxlen, records def main(): - print "static const char *simp_to_trad[][2] = {" + print("static const char *simp_to_trad[][2] = {") maxlen, records = get_records() for s, ts in records: - print ' { "%s", "%s" },' % (s, ts) - print "};" - print '#define SIMP_TO_TRAD_MAX_LEN (%d)' % maxlen + print(' { "%s", "%s" },' % (s, ts)) + print("};") + print('#define SIMP_TO_TRAD_MAX_LEN (%d)' % maxlen) if __name__ == "__main__": main() diff --git a/tools/googlecode_upload.py b/tools/googlecode_upload.py index d2d5f97..e72b21e 100755 --- a/tools/googlecode_upload.py +++ b/tools/googlecode_upload.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2006, 2007 Google Inc. All Rights Reserved. # Author: danderson@google.com (David Anderson) @@ -48,7 +48,7 @@ __author__ = 'danderson@google.com (David Anderson)' -import httplib +import http.client import os.path import optparse import getpass @@ -95,7 +95,7 @@ def upload(file, project_name, user_name, password, summary, labels=None): 'Content-Type': content_type, } - server = httplib.HTTPSConnection(upload_host) + server = http.client.HTTPSConnection(upload_host) server.request('POST', upload_uri, body, headers) resp = server.getresponse() server.close() @@ -177,17 +177,17 @@ def upload_find_auth(file_path, project_name, summary, labels=None, user_name = sys.stdin.readline().rstrip() if password is None: # Read password if not loaded from svn config, or on subsequent tries. - print 'Please enter your googlecode.com password.' - print '** Note that this is NOT your Gmail account password! **' - print 'It is the password you use to access Subversion repositories,' - print 'and can be found here: http://code.google.com/hosting/settings' + print('Please enter your googlecode.com password.') + print('** Note that this is NOT your Gmail account password! **') + print('It is the password you use to access Subversion repositories,') + print('and can be found here: http://code.google.com/hosting/settings') password = getpass.getpass() status, reason, url = upload(file_path, project_name, user_name, password, summary, labels) # Returns 403 Forbidden instead of 401 Unauthorized for bad # credentials as of 2007-07-17. - if status in [httplib.FORBIDDEN, httplib.UNAUTHORIZED]: + if status in [http.client.FORBIDDEN, http.client.UNAUTHORIZED]: # Rest for another try. user_name = password = None tries = tries - 1 @@ -235,12 +235,12 @@ def main(): options.summary, labels, options.user, options.password) if url: - print 'The file was uploaded successfully.' - print 'URL: %s' % url + print('The file was uploaded successfully.') + print('URL: %s' % url) return 0 else: - print 'An error occurred. Your file was not uploaded.' - print 'Google Code upload server said: %s (%s)' % (reason, status) + print('An error occurred. Your file was not uploaded.') + print('Google Code upload server said: %s (%s)' % (reason, status)) return 1 From 87758d9ba6fcec58d95b13f2de46d0ef1f6a8f42 Mon Sep 17 00:00:00 2001 From: Gunnar Hjalmarsson Date: Mon, 4 May 2020 19:55:45 +0200 Subject: [PATCH 2/3] More tweaks of data/db/android/create_db.py --- data/db/android/create_db.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/data/db/android/create_db.py b/data/db/android/create_db.py index 2da5bff..d6a1a27 100755 --- a/data/db/android/create_db.py +++ b/data/db/android/create_db.py @@ -3,6 +3,7 @@ from id import * from valid_hanzi import * import sys +from functools import cmp_to_key def get_sheng_yun(pinyin): if pinyin == None: @@ -16,7 +17,7 @@ def get_sheng_yun(pinyin): return "", pinyin def read_phrases(filename): - buf = file(filename).read() + buf = open(filename, 'r+b').read() buf = str(buf, "utf16") buf = buf.strip() for l in buf.split('\n'): @@ -49,7 +50,7 @@ def create_db(filename): # con.commit() records = list(read_phrases(filename)) - records.sort(lambda a, b: 1 if a[1] > b[1] else -1) + records.sort (key = cmp_to_key (lambda a, b: 1 if a[1] > b[1] else -1)) records_new = [] i = 0 max_freq = 0.0 From 7326d8f8f9bef18684a7d8243d422666d3560540 Mon Sep 17 00:00:00 2001 From: Pugalendhi Ganesan Date: Wed, 29 Jul 2020 18:04:48 +0530 Subject: [PATCH 3/3] Add "utf-16" encoding when opening the phrase file Signed-off-by: Martin Jansa --- data/db/android/create_db.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/data/db/android/create_db.py b/data/db/android/create_db.py index d6a1a27..f648670 100755 --- a/data/db/android/create_db.py +++ b/data/db/android/create_db.py @@ -17,8 +17,7 @@ def get_sheng_yun(pinyin): return "", pinyin def read_phrases(filename): - buf = open(filename, 'r+b').read() - buf = str(buf, "utf16") + buf = open(filename,encoding='utf-16').read() buf = buf.strip() for l in buf.split('\n'): hanzi, freq, flag, pinyin = l.split(' ', 3) @@ -70,8 +69,7 @@ def create_db(filename): s, y = pinyin_id[s], pinyin_id[y] columns.append(s) columns.append(y) - values = "'%s', %d, %s" % (hanzi.encode("utf8"), freq, ",".join(map(str,columns))) - + values = "'%s', %d, %s" % (hanzi, freq, ",".join(map(str,columns))) sql = insert_sql % (len(hanzi) - 1, values) print(sql) print("COMMIT;")