Files
LonaRPG-Online-Games/ModScripts/_Mods/LonaCraft/tools/check_text.py
T
2026-10-29 21:59:06 +04:00

249 lines
10 KiB
Python

#!/usr/bin/env python3
r"""LonaCraft text checker: diffs Text/<lang>/*.txt against Text/ENG.
Usage: python tools/check_text.py [--lang CHT] [--quiet]
(run from anywhere; the mod folder is the parent of tools/)
--lang X check only Text/X (ENG is always read as the reference)
--quiet leave out the detail lines, keep the tables and the SUMMARY
Per language folder (every folder under Text/ except ENG) and per file:
missing keys in ENG that the language file lacks (or the whole file)
extra keys in the language file that ENG lacks
same keys whose text is identical to ENG (= untranslated)
colon body lines containing ASCII ':' outside comments (the game reads
"a: b" as speaker "a", Data/Scripts/Frames/121_Dialog_Control_System.rb:137)
CR / BOM carriage returns / byte order marks (files must be LF, UTF-8 no BOM)
ENG itself is checked for colon, CR and BOM too.
Control codes, per key present in both ENG and the language (second table per
language, one column per check, the number is the keys that differ from ENG):
lines number of body lines
nl count of \n (not counting \narr)
narr count of \narr
brace count of \}
optB number of \optB[...] lists
entries number of entries in each \optB list (split on ',')
expr the #{...} expressions, identical text (order does not matter)
color the \c[..] / \C[..] codes, identical text
fmt count of %s and of %d
token count of the words NUM, HP_NOW, HP_MAX
optchar an \optB entry holding '[' or ']', an empty entry, or no closing ']'
(a ',' inside an entry shows up as a wrong entry count; ENG is checked
for optchar too)
Parsing follows the game's Text#parse: blank-line separated blocks, the first
line is the key, lines starting with '#' are skipped.
Exit code 1 when any key is missing or any control-code mismatch is found, else 0.
"""
import os
import re
import sys
MOD = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
TEXT = os.path.join(MOD, "Text")
BASE = "ENG"
def read_file(path):
"""(blocks, colon_lines, cr_count, has_bom); blocks = {key: body text}."""
raw = open(path, "rb").read()
bom = raw.startswith(b"\xef\xbb\xbf")
cr = raw.count(b"\r")
text = raw.decode("utf-8", errors="replace")
if bom:
text = text[1:]
lines = text.replace("\r", "").split("\n")
blocks = {}
colons = []
i = 0
while i < len(lines):
line = lines[i]
i += 1
if line == "" or line[0] == "#":
continue
key = line
body = []
while i < len(lines):
line = lines[i]
i += 1
if line[:1] == "#":
continue
if line == "":
break
body.append(line)
if ":" in line:
colons.append((i, key))
blocks[key] = "\n".join(body)
return blocks, colons, cr, bom
RE_NARR = re.compile(r"\\narr")
RE_NL = re.compile(r"\\n(?!arr)")
RE_BRACE = re.compile(r"\\\}")
RE_EXPR = re.compile(r"#\{[^}]*\}")
RE_COLOR = re.compile(r"\\[cC]\[[^\]]*\]")
RE_OPTB = re.compile(r"\\optB\[")
RE_TOKENS = [re.compile(r"(?<![A-Za-z])" + t + r"(?![A-Za-z])") for t in ("NUM", "HP_NOW", "HP_MAX")]
CTL = ["lines", "nl", "narr", "brace", "optB", "entries", "expr", "color", "fmt", "token", "optchar"]
def opt_lists(body):
"""[(entries, bad)] for every \\optB[...] list of a body; bad = '[' or ']' inside,
an empty entry, or no closing ']'."""
out = []
for line in body.split("\n"):
starts = [m.start() for m in RE_OPTB.finditer(line)]
for n, s in enumerate(starts):
limit = starts[n + 1] if n + 1 < len(starts) else len(line)
seg = line[s + 6:limit]
close = seg.rfind("]")
if close < 0:
out.append((seg.split(","), True))
continue
inner = seg[:close]
entries = inner.split(",")
bad = "[" in inner or "]" in inner or any(e.strip() == "" for e in entries)
out.append((entries, bad))
return out
def profile(body):
"""Control-code profile of one key's body text (a dict, one value per CTL name)."""
lists = opt_lists(body)
return {
"lines": len(body.split("\n")) if body else 0,
"nl": len(RE_NL.findall(body)),
"narr": len(RE_NARR.findall(body)),
"brace": len(RE_BRACE.findall(body)),
"optB": len(lists),
"entries": [len(e) for e, _ in lists],
"expr": sorted(RE_EXPR.findall(body)),
"color": sorted(RE_COLOR.findall(body)),
"fmt": (body.count("%s"), body.count("%d")),
"token": tuple(len(r.findall(body)) for r in RE_TOKENS),
"optchar": any(bad for _, bad in lists),
}
def ctl_diff(eng_body, body):
"""Names of the checks where +body+ differs from +eng_body+ (optchar: true in body)."""
a, b = profile(eng_body), profile(body)
return [c for c in CTL if (b[c] if c == "optchar" else a[c] != b[c])]
def txt_files(lang):
folder = os.path.join(TEXT, lang)
if not os.path.isdir(folder):
return []
return sorted(f for f in os.listdir(folder) if f.lower().endswith(".txt"))
def table(rows, header):
widths = [max(len(str(r[c])) for r in rows + [header]) for c in range(len(header))]
fmt = " ".join("{:<%d}" % w if c == 0 else "{:>%d}" % w for c, w in enumerate(widths))
out = [fmt.format(*header), fmt.format(*["-" * w for w in widths])]
out.extend(fmt.format(*r) for r in rows)
return "\n".join(out)
def main(argv):
only = None
quiet = "--quiet" in argv
if "--lang" in argv:
only = argv[argv.index("--lang") + 1].upper()
if not os.path.isdir(os.path.join(TEXT, BASE)):
print("no Text/%s folder under %s" % (BASE, MOD))
return 1
eng = {}
eng_notes = [0, 0, 0] # colon lines, CR, BOM
for f in txt_files(BASE):
blocks, colons, cr, bom = read_file(os.path.join(TEXT, BASE, f))
eng[f] = blocks
eng_notes[0] += len(colons)
eng_notes[1] += cr
eng_notes[2] += 1 if bom else 0
eng_keys = sum(len(b) for b in eng.values())
eng_ctl = [(f, k) for f, bl in eng.items() for k, v in bl.items() if profile(v)["optchar"]]
langs = sorted(d for d in os.listdir(TEXT) if os.path.isdir(os.path.join(TEXT, d)) and d != BASE)
if only:
langs = [l for l in langs if l == only]
header = ["file", "ENG", "present", "missing", "extra", "same", "colon", "CR", "BOM"]
print("%s: %d keys in %d files, colon lines %d, CR %d, BOM %d, optB entry problems %d" %
(BASE, eng_keys, len(eng), eng_notes[0], eng_notes[1], eng_notes[2], len(eng_ctl)))
summary = []
any_missing = False
ctl_total = len(eng_ctl)
details = []
ctl_details = ["%s %s optchar (key %s)" % (BASE, f, k) for f, k in eng_ctl]
ctl_header = ["file"] + CTL + ["keys"]
for lang in langs:
rows = []
ctl_rows = []
ctl_tot = [0] * (len(CTL) + 1)
tot = [0] * 8 # ENG, present, missing, extra, same, colon, cr, bom
files = txt_files(lang)
for f in sorted(set(eng) | set(files)):
e = eng.get(f, {})
if f in files:
blocks, colons, cr, bom = read_file(os.path.join(TEXT, lang, f))
else:
blocks, colons, cr, bom = {}, [], 0, False
present = sum(1 for k in e if k in blocks)
missing = [k for k in e if k not in blocks]
extra = [k for k in blocks if k not in e]
same = sum(1 for k in e if k in blocks and blocks[k] == e[k])
rows.append([f if f in files else f + " (no file)", len(e), present, len(missing), len(extra),
same, len(colons), cr, 1 if bom else 0])
for n, v in enumerate([len(e), present, len(missing), len(extra), same, len(colons), cr, 1 if bom else 0]):
tot[n] += v
counts = [0] * len(CTL)
bad_keys = 0
for k in e:
if k not in blocks:
continue
diff = ctl_diff(e[k], blocks[k])
if diff:
bad_keys += 1
ctl_details.append("%s %s %s (key %s)" % (lang, f, ",".join(diff), k))
for c in diff:
counts[CTL.index(c)] += 1
ctl_rows.append([f if f in files else f + " (no file)"] + counts + [bad_keys])
for n, v in enumerate(counts + [bad_keys]):
ctl_tot[n] += v
for k in missing:
details.append("%s %s missing %s" % (lang, f, k))
for k in extra:
details.append("%s %s extra %s" % (lang, f, k))
for ln, k in colons:
details.append("%s %s colon line %d (key %s)" % (lang, f, ln, k))
rows.append(["TOTAL"] + tot)
print("\n== %s ==" % lang)
print(table([[str(c) for c in r] for r in rows], header))
ctl_rows.append(["TOTAL"] + ctl_tot)
print("\ncontrol codes vs ENG, %s (keys that differ per check)" % lang)
print(table([[str(c) for c in r] for r in ctl_rows], ctl_header))
ctl_total += ctl_tot[-1]
translated = tot[1] - tot[4]
pct = 100.0 * translated / tot[0] if tot[0] else 0.0
summary.append("%s %d/%d translated (%.0f%%), %d same as ENG, %d missing, %d extra, %d control-code mismatches" %
(lang, translated, tot[0], pct, tot[4], tot[2], tot[3], ctl_tot[-1]))
if tot[2]:
any_missing = True
if details and not quiet:
print("\n-- details (first 60) --")
for d in details[:60]:
print(d)
if len(details) > 60:
print("... %d more" % (len(details) - 60))
if ctl_details and not quiet:
print("\n-- control-code details (first 60) --")
for d in ctl_details[:60]:
print(d)
if len(ctl_details) > 60:
print("... %d more" % (len(ctl_details) - 60))
print("\nSUMMARY: " + "; ".join(summary) if summary else "\nSUMMARY: no language folders")
return 1 if (any_missing or ctl_total) else 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))