249 lines
10 KiB
Python
249 lines
10 KiB
Python
#!/usr/bin/env python3
|
|
r"""LonaCraft text checker: diffs Text/<lang>/*.txt against Text/ENG.
|
|
|
|
Usage: python tools/check_text.py [--lang CHT] [--quiet]
|
|
(run from anywhere; the mod folder is the parent of tools/)
|
|
--lang X check only Text/X (ENG is always read as the reference)
|
|
--quiet leave out the detail lines, keep the tables and the SUMMARY
|
|
|
|
Per language folder (every folder under Text/ except ENG) and per file:
|
|
missing keys in ENG that the language file lacks (or the whole file)
|
|
extra keys in the language file that ENG lacks
|
|
same keys whose text is identical to ENG (= untranslated)
|
|
colon body lines containing ASCII ':' outside comments (the game reads
|
|
"a: b" as speaker "a", Data/Scripts/Frames/121_Dialog_Control_System.rb:137)
|
|
CR / BOM carriage returns / byte order marks (files must be LF, UTF-8 no BOM)
|
|
ENG itself is checked for colon, CR and BOM too.
|
|
|
|
Control codes, per key present in both ENG and the language (second table per
|
|
language, one column per check, the number is the keys that differ from ENG):
|
|
lines number of body lines
|
|
nl count of \n (not counting \narr)
|
|
narr count of \narr
|
|
brace count of \}
|
|
optB number of \optB[...] lists
|
|
entries number of entries in each \optB list (split on ',')
|
|
expr the #{...} expressions, identical text (order does not matter)
|
|
color the \c[..] / \C[..] codes, identical text
|
|
fmt count of %s and of %d
|
|
token count of the words NUM, HP_NOW, HP_MAX
|
|
optchar an \optB entry holding '[' or ']', an empty entry, or no closing ']'
|
|
(a ',' inside an entry shows up as a wrong entry count; ENG is checked
|
|
for optchar too)
|
|
Parsing follows the game's Text#parse: blank-line separated blocks, the first
|
|
line is the key, lines starting with '#' are skipped.
|
|
Exit code 1 when any key is missing or any control-code mismatch is found, else 0.
|
|
"""
|
|
import os
|
|
import re
|
|
import sys
|
|
|
|
MOD = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
TEXT = os.path.join(MOD, "Text")
|
|
BASE = "ENG"
|
|
|
|
|
|
def read_file(path):
|
|
"""(blocks, colon_lines, cr_count, has_bom); blocks = {key: body text}."""
|
|
raw = open(path, "rb").read()
|
|
bom = raw.startswith(b"\xef\xbb\xbf")
|
|
cr = raw.count(b"\r")
|
|
text = raw.decode("utf-8", errors="replace")
|
|
if bom:
|
|
text = text[1:]
|
|
lines = text.replace("\r", "").split("\n")
|
|
blocks = {}
|
|
colons = []
|
|
i = 0
|
|
while i < len(lines):
|
|
line = lines[i]
|
|
i += 1
|
|
if line == "" or line[0] == "#":
|
|
continue
|
|
key = line
|
|
body = []
|
|
while i < len(lines):
|
|
line = lines[i]
|
|
i += 1
|
|
if line[:1] == "#":
|
|
continue
|
|
if line == "":
|
|
break
|
|
body.append(line)
|
|
if ":" in line:
|
|
colons.append((i, key))
|
|
blocks[key] = "\n".join(body)
|
|
return blocks, colons, cr, bom
|
|
|
|
|
|
RE_NARR = re.compile(r"\\narr")
|
|
RE_NL = re.compile(r"\\n(?!arr)")
|
|
RE_BRACE = re.compile(r"\\\}")
|
|
RE_EXPR = re.compile(r"#\{[^}]*\}")
|
|
RE_COLOR = re.compile(r"\\[cC]\[[^\]]*\]")
|
|
RE_OPTB = re.compile(r"\\optB\[")
|
|
RE_TOKENS = [re.compile(r"(?<![A-Za-z])" + t + r"(?![A-Za-z])") for t in ("NUM", "HP_NOW", "HP_MAX")]
|
|
CTL = ["lines", "nl", "narr", "brace", "optB", "entries", "expr", "color", "fmt", "token", "optchar"]
|
|
|
|
|
|
def opt_lists(body):
|
|
"""[(entries, bad)] for every \\optB[...] list of a body; bad = '[' or ']' inside,
|
|
an empty entry, or no closing ']'."""
|
|
out = []
|
|
for line in body.split("\n"):
|
|
starts = [m.start() for m in RE_OPTB.finditer(line)]
|
|
for n, s in enumerate(starts):
|
|
limit = starts[n + 1] if n + 1 < len(starts) else len(line)
|
|
seg = line[s + 6:limit]
|
|
close = seg.rfind("]")
|
|
if close < 0:
|
|
out.append((seg.split(","), True))
|
|
continue
|
|
inner = seg[:close]
|
|
entries = inner.split(",")
|
|
bad = "[" in inner or "]" in inner or any(e.strip() == "" for e in entries)
|
|
out.append((entries, bad))
|
|
return out
|
|
|
|
|
|
def profile(body):
|
|
"""Control-code profile of one key's body text (a dict, one value per CTL name)."""
|
|
lists = opt_lists(body)
|
|
return {
|
|
"lines": len(body.split("\n")) if body else 0,
|
|
"nl": len(RE_NL.findall(body)),
|
|
"narr": len(RE_NARR.findall(body)),
|
|
"brace": len(RE_BRACE.findall(body)),
|
|
"optB": len(lists),
|
|
"entries": [len(e) for e, _ in lists],
|
|
"expr": sorted(RE_EXPR.findall(body)),
|
|
"color": sorted(RE_COLOR.findall(body)),
|
|
"fmt": (body.count("%s"), body.count("%d")),
|
|
"token": tuple(len(r.findall(body)) for r in RE_TOKENS),
|
|
"optchar": any(bad for _, bad in lists),
|
|
}
|
|
|
|
|
|
def ctl_diff(eng_body, body):
|
|
"""Names of the checks where +body+ differs from +eng_body+ (optchar: true in body)."""
|
|
a, b = profile(eng_body), profile(body)
|
|
return [c for c in CTL if (b[c] if c == "optchar" else a[c] != b[c])]
|
|
|
|
|
|
def txt_files(lang):
|
|
folder = os.path.join(TEXT, lang)
|
|
if not os.path.isdir(folder):
|
|
return []
|
|
return sorted(f for f in os.listdir(folder) if f.lower().endswith(".txt"))
|
|
|
|
|
|
def table(rows, header):
|
|
widths = [max(len(str(r[c])) for r in rows + [header]) for c in range(len(header))]
|
|
fmt = " ".join("{:<%d}" % w if c == 0 else "{:>%d}" % w for c, w in enumerate(widths))
|
|
out = [fmt.format(*header), fmt.format(*["-" * w for w in widths])]
|
|
out.extend(fmt.format(*r) for r in rows)
|
|
return "\n".join(out)
|
|
|
|
|
|
def main(argv):
|
|
only = None
|
|
quiet = "--quiet" in argv
|
|
if "--lang" in argv:
|
|
only = argv[argv.index("--lang") + 1].upper()
|
|
if not os.path.isdir(os.path.join(TEXT, BASE)):
|
|
print("no Text/%s folder under %s" % (BASE, MOD))
|
|
return 1
|
|
eng = {}
|
|
eng_notes = [0, 0, 0] # colon lines, CR, BOM
|
|
for f in txt_files(BASE):
|
|
blocks, colons, cr, bom = read_file(os.path.join(TEXT, BASE, f))
|
|
eng[f] = blocks
|
|
eng_notes[0] += len(colons)
|
|
eng_notes[1] += cr
|
|
eng_notes[2] += 1 if bom else 0
|
|
eng_keys = sum(len(b) for b in eng.values())
|
|
eng_ctl = [(f, k) for f, bl in eng.items() for k, v in bl.items() if profile(v)["optchar"]]
|
|
langs = sorted(d for d in os.listdir(TEXT) if os.path.isdir(os.path.join(TEXT, d)) and d != BASE)
|
|
if only:
|
|
langs = [l for l in langs if l == only]
|
|
header = ["file", "ENG", "present", "missing", "extra", "same", "colon", "CR", "BOM"]
|
|
print("%s: %d keys in %d files, colon lines %d, CR %d, BOM %d, optB entry problems %d" %
|
|
(BASE, eng_keys, len(eng), eng_notes[0], eng_notes[1], eng_notes[2], len(eng_ctl)))
|
|
summary = []
|
|
any_missing = False
|
|
ctl_total = len(eng_ctl)
|
|
details = []
|
|
ctl_details = ["%s %s optchar (key %s)" % (BASE, f, k) for f, k in eng_ctl]
|
|
ctl_header = ["file"] + CTL + ["keys"]
|
|
for lang in langs:
|
|
rows = []
|
|
ctl_rows = []
|
|
ctl_tot = [0] * (len(CTL) + 1)
|
|
tot = [0] * 8 # ENG, present, missing, extra, same, colon, cr, bom
|
|
files = txt_files(lang)
|
|
for f in sorted(set(eng) | set(files)):
|
|
e = eng.get(f, {})
|
|
if f in files:
|
|
blocks, colons, cr, bom = read_file(os.path.join(TEXT, lang, f))
|
|
else:
|
|
blocks, colons, cr, bom = {}, [], 0, False
|
|
present = sum(1 for k in e if k in blocks)
|
|
missing = [k for k in e if k not in blocks]
|
|
extra = [k for k in blocks if k not in e]
|
|
same = sum(1 for k in e if k in blocks and blocks[k] == e[k])
|
|
rows.append([f if f in files else f + " (no file)", len(e), present, len(missing), len(extra),
|
|
same, len(colons), cr, 1 if bom else 0])
|
|
for n, v in enumerate([len(e), present, len(missing), len(extra), same, len(colons), cr, 1 if bom else 0]):
|
|
tot[n] += v
|
|
counts = [0] * len(CTL)
|
|
bad_keys = 0
|
|
for k in e:
|
|
if k not in blocks:
|
|
continue
|
|
diff = ctl_diff(e[k], blocks[k])
|
|
if diff:
|
|
bad_keys += 1
|
|
ctl_details.append("%s %s %s (key %s)" % (lang, f, ",".join(diff), k))
|
|
for c in diff:
|
|
counts[CTL.index(c)] += 1
|
|
ctl_rows.append([f if f in files else f + " (no file)"] + counts + [bad_keys])
|
|
for n, v in enumerate(counts + [bad_keys]):
|
|
ctl_tot[n] += v
|
|
for k in missing:
|
|
details.append("%s %s missing %s" % (lang, f, k))
|
|
for k in extra:
|
|
details.append("%s %s extra %s" % (lang, f, k))
|
|
for ln, k in colons:
|
|
details.append("%s %s colon line %d (key %s)" % (lang, f, ln, k))
|
|
rows.append(["TOTAL"] + tot)
|
|
print("\n== %s ==" % lang)
|
|
print(table([[str(c) for c in r] for r in rows], header))
|
|
ctl_rows.append(["TOTAL"] + ctl_tot)
|
|
print("\ncontrol codes vs ENG, %s (keys that differ per check)" % lang)
|
|
print(table([[str(c) for c in r] for r in ctl_rows], ctl_header))
|
|
ctl_total += ctl_tot[-1]
|
|
translated = tot[1] - tot[4]
|
|
pct = 100.0 * translated / tot[0] if tot[0] else 0.0
|
|
summary.append("%s %d/%d translated (%.0f%%), %d same as ENG, %d missing, %d extra, %d control-code mismatches" %
|
|
(lang, translated, tot[0], pct, tot[4], tot[2], tot[3], ctl_tot[-1]))
|
|
if tot[2]:
|
|
any_missing = True
|
|
if details and not quiet:
|
|
print("\n-- details (first 60) --")
|
|
for d in details[:60]:
|
|
print(d)
|
|
if len(details) > 60:
|
|
print("... %d more" % (len(details) - 60))
|
|
if ctl_details and not quiet:
|
|
print("\n-- control-code details (first 60) --")
|
|
for d in ctl_details[:60]:
|
|
print(d)
|
|
if len(ctl_details) > 60:
|
|
print("... %d more" % (len(ctl_details) - 60))
|
|
print("\nSUMMARY: " + "; ".join(summary) if summary else "\nSUMMARY: no language folders")
|
|
return 1 if (any_missing or ctl_total) else 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|