#!/usr/bin/env python3 r"""LonaCraft text checker: diffs Text//*.txt against Text/ENG. Usage: python tools/check_text.py [--lang CHT] [--quiet] (run from anywhere; the mod folder is the parent of tools/) --lang X check only Text/X (ENG is always read as the reference) --quiet leave out the detail lines, keep the tables and the SUMMARY Per language folder (every folder under Text/ except ENG) and per file: missing keys in ENG that the language file lacks (or the whole file) extra keys in the language file that ENG lacks same keys whose text is identical to ENG (= untranslated) colon body lines containing ASCII ':' outside comments (the game reads "a: b" as speaker "a", Data/Scripts/Frames/121_Dialog_Control_System.rb:137) CR / BOM carriage returns / byte order marks (files must be LF, UTF-8 no BOM) ENG itself is checked for colon, CR and BOM too. Control codes, per key present in both ENG and the language (second table per language, one column per check, the number is the keys that differ from ENG): lines number of body lines nl count of \n (not counting \narr) narr count of \narr brace count of \} optB number of \optB[...] lists entries number of entries in each \optB list (split on ',') expr the #{...} expressions, identical text (order does not matter) color the \c[..] / \C[..] codes, identical text fmt count of %s and of %d token count of the words NUM, HP_NOW, HP_MAX optchar an \optB entry holding '[' or ']', an empty entry, or no closing ']' (a ',' inside an entry shows up as a wrong entry count; ENG is checked for optchar too) Parsing follows the game's Text#parse: blank-line separated blocks, the first line is the key, lines starting with '#' are skipped. Exit code 1 when any key is missing or any control-code mismatch is found, else 0. """ import os import re import sys MOD = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) TEXT = os.path.join(MOD, "Text") BASE = "ENG" def read_file(path): """(blocks, colon_lines, cr_count, has_bom); blocks = {key: body text}.""" raw = open(path, "rb").read() bom = raw.startswith(b"\xef\xbb\xbf") cr = raw.count(b"\r") text = raw.decode("utf-8", errors="replace") if bom: text = text[1:] lines = text.replace("\r", "").split("\n") blocks = {} colons = [] i = 0 while i < len(lines): line = lines[i] i += 1 if line == "" or line[0] == "#": continue key = line body = [] while i < len(lines): line = lines[i] i += 1 if line[:1] == "#": continue if line == "": break body.append(line) if ":" in line: colons.append((i, key)) blocks[key] = "\n".join(body) return blocks, colons, cr, bom RE_NARR = re.compile(r"\\narr") RE_NL = re.compile(r"\\n(?!arr)") RE_BRACE = re.compile(r"\\\}") RE_EXPR = re.compile(r"#\{[^}]*\}") RE_COLOR = re.compile(r"\\[cC]\[[^\]]*\]") RE_OPTB = re.compile(r"\\optB\[") RE_TOKENS = [re.compile(r"(?%d}" % w for c, w in enumerate(widths)) out = [fmt.format(*header), fmt.format(*["-" * w for w in widths])] out.extend(fmt.format(*r) for r in rows) return "\n".join(out) def main(argv): only = None quiet = "--quiet" in argv if "--lang" in argv: only = argv[argv.index("--lang") + 1].upper() if not os.path.isdir(os.path.join(TEXT, BASE)): print("no Text/%s folder under %s" % (BASE, MOD)) return 1 eng = {} eng_notes = [0, 0, 0] # colon lines, CR, BOM for f in txt_files(BASE): blocks, colons, cr, bom = read_file(os.path.join(TEXT, BASE, f)) eng[f] = blocks eng_notes[0] += len(colons) eng_notes[1] += cr eng_notes[2] += 1 if bom else 0 eng_keys = sum(len(b) for b in eng.values()) eng_ctl = [(f, k) for f, bl in eng.items() for k, v in bl.items() if profile(v)["optchar"]] langs = sorted(d for d in os.listdir(TEXT) if os.path.isdir(os.path.join(TEXT, d)) and d != BASE) if only: langs = [l for l in langs if l == only] header = ["file", "ENG", "present", "missing", "extra", "same", "colon", "CR", "BOM"] print("%s: %d keys in %d files, colon lines %d, CR %d, BOM %d, optB entry problems %d" % (BASE, eng_keys, len(eng), eng_notes[0], eng_notes[1], eng_notes[2], len(eng_ctl))) summary = [] any_missing = False ctl_total = len(eng_ctl) details = [] ctl_details = ["%s %s optchar (key %s)" % (BASE, f, k) for f, k in eng_ctl] ctl_header = ["file"] + CTL + ["keys"] for lang in langs: rows = [] ctl_rows = [] ctl_tot = [0] * (len(CTL) + 1) tot = [0] * 8 # ENG, present, missing, extra, same, colon, cr, bom files = txt_files(lang) for f in sorted(set(eng) | set(files)): e = eng.get(f, {}) if f in files: blocks, colons, cr, bom = read_file(os.path.join(TEXT, lang, f)) else: blocks, colons, cr, bom = {}, [], 0, False present = sum(1 for k in e if k in blocks) missing = [k for k in e if k not in blocks] extra = [k for k in blocks if k not in e] same = sum(1 for k in e if k in blocks and blocks[k] == e[k]) rows.append([f if f in files else f + " (no file)", len(e), present, len(missing), len(extra), same, len(colons), cr, 1 if bom else 0]) for n, v in enumerate([len(e), present, len(missing), len(extra), same, len(colons), cr, 1 if bom else 0]): tot[n] += v counts = [0] * len(CTL) bad_keys = 0 for k in e: if k not in blocks: continue diff = ctl_diff(e[k], blocks[k]) if diff: bad_keys += 1 ctl_details.append("%s %s %s (key %s)" % (lang, f, ",".join(diff), k)) for c in diff: counts[CTL.index(c)] += 1 ctl_rows.append([f if f in files else f + " (no file)"] + counts + [bad_keys]) for n, v in enumerate(counts + [bad_keys]): ctl_tot[n] += v for k in missing: details.append("%s %s missing %s" % (lang, f, k)) for k in extra: details.append("%s %s extra %s" % (lang, f, k)) for ln, k in colons: details.append("%s %s colon line %d (key %s)" % (lang, f, ln, k)) rows.append(["TOTAL"] + tot) print("\n== %s ==" % lang) print(table([[str(c) for c in r] for r in rows], header)) ctl_rows.append(["TOTAL"] + ctl_tot) print("\ncontrol codes vs ENG, %s (keys that differ per check)" % lang) print(table([[str(c) for c in r] for r in ctl_rows], ctl_header)) ctl_total += ctl_tot[-1] translated = tot[1] - tot[4] pct = 100.0 * translated / tot[0] if tot[0] else 0.0 summary.append("%s %d/%d translated (%.0f%%), %d same as ENG, %d missing, %d extra, %d control-code mismatches" % (lang, translated, tot[0], pct, tot[4], tot[2], tot[3], ctl_tot[-1])) if tot[2]: any_missing = True if details and not quiet: print("\n-- details (first 60) --") for d in details[:60]: print(d) if len(details) > 60: print("... %d more" % (len(details) - 60)) if ctl_details and not quiet: print("\n-- control-code details (first 60) --") for d in ctl_details[:60]: print(d) if len(ctl_details) > 60: print("... %d more" % (len(ctl_details) - 60)) print("\nSUMMARY: " + "; ".join(summary) if summary else "\nSUMMARY: no language folders") return 1 if (any_missing or ctl_total) else 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))