Release 0.0.1.8.8.7
This commit is contained in:
1 parent
3c8a17aa78
commit
c7beab414c
16283 files changed
+768761
-2
No files matched your search
@@ -0,0 +1,248 @@
|
||||
#!/usr/bin/env python3
|
||||
r"""LonaCraft text checker: diffs Text/<lang>/*.txt against Text/ENG.
|
||||
|
||||
Usage: python tools/check_text.py [--lang CHT] [--quiet]
|
||||
(run from anywhere; the mod folder is the parent of tools/)
|
||||
--lang X check only Text/X (ENG is always read as the reference)
|
||||
--quiet leave out the detail lines, keep the tables and the SUMMARY
|
||||
|
||||
Per language folder (every folder under Text/ except ENG) and per file:
|
||||
missing keys in ENG that the language file lacks (or the whole file)
|
||||
extra keys in the language file that ENG lacks
|
||||
same keys whose text is identical to ENG (= untranslated)
|
||||
colon body lines containing ASCII ':' outside comments (the game reads
|
||||
"a: b" as speaker "a", Data/Scripts/Frames/121_Dialog_Control_System.rb:137)
|
||||
CR / BOM carriage returns / byte order marks (files must be LF, UTF-8 no BOM)
|
||||
ENG itself is checked for colon, CR and BOM too.
|
||||
|
||||
Control codes, per key present in both ENG and the language (second table per
|
||||
language, one column per check, the number is the keys that differ from ENG):
|
||||
lines number of body lines
|
||||
nl count of \n (not counting \narr)
|
||||
narr count of \narr
|
||||
brace count of \}
|
||||
optB number of \optB[...] lists
|
||||
entries number of entries in each \optB list (split on ',')
|
||||
expr the #{...} expressions, identical text (order does not matter)
|
||||
color the \c[..] / \C[..] codes, identical text
|
||||
fmt count of %s and of %d
|
||||
token count of the words NUM, HP_NOW, HP_MAX
|
||||
optchar an \optB entry holding '[' or ']', an empty entry, or no closing ']'
|
||||
(a ',' inside an entry shows up as a wrong entry count; ENG is checked
|
||||
for optchar too)
|
||||
Parsing follows the game's Text#parse: blank-line separated blocks, the first
|
||||
line is the key, lines starting with '#' are skipped.
|
||||
Exit code 1 when any key is missing or any control-code mismatch is found, else 0.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
MOD = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
TEXT = os.path.join(MOD, "Text")
|
||||
BASE = "ENG"
|
||||
|
||||
|
||||
def read_file(path):
|
||||
"""(blocks, colon_lines, cr_count, has_bom); blocks = {key: body text}."""
|
||||
raw = open(path, "rb").read()
|
||||
bom = raw.startswith(b"\xef\xbb\xbf")
|
||||
cr = raw.count(b"\r")
|
||||
text = raw.decode("utf-8", errors="replace")
|
||||
if bom:
|
||||
text = text[1:]
|
||||
lines = text.replace("\r", "").split("\n")
|
||||
blocks = {}
|
||||
colons = []
|
||||
i = 0
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
i += 1
|
||||
if line == "" or line[0] == "#":
|
||||
continue
|
||||
key = line
|
||||
body = []
|
||||
while i < len(lines):
|
||||
line = lines[i]
|
||||
i += 1
|
||||
if line[:1] == "#":
|
||||
continue
|
||||
if line == "":
|
||||
break
|
||||
body.append(line)
|
||||
if ":" in line:
|
||||
colons.append((i, key))
|
||||
blocks[key] = "\n".join(body)
|
||||
return blocks, colons, cr, bom
|
||||
|
||||
|
||||
RE_NARR = re.compile(r"\\narr")
|
||||
RE_NL = re.compile(r"\\n(?!arr)")
|
||||
RE_BRACE = re.compile(r"\\\}")
|
||||
RE_EXPR = re.compile(r"#\{[^}]*\}")
|
||||
RE_COLOR = re.compile(r"\\[cC]\[[^\]]*\]")
|
||||
RE_OPTB = re.compile(r"\\optB\[")
|
||||
RE_TOKENS = [re.compile(r"(?<![A-Za-z])" + t + r"(?![A-Za-z])") for t in ("NUM", "HP_NOW", "HP_MAX")]
|
||||
CTL = ["lines", "nl", "narr", "brace", "optB", "entries", "expr", "color", "fmt", "token", "optchar"]
|
||||
|
||||
|
||||
def opt_lists(body):
|
||||
"""[(entries, bad)] for every \\optB[...] list of a body; bad = '[' or ']' inside,
|
||||
an empty entry, or no closing ']'."""
|
||||
out = []
|
||||
for line in body.split("\n"):
|
||||
starts = [m.start() for m in RE_OPTB.finditer(line)]
|
||||
for n, s in enumerate(starts):
|
||||
limit = starts[n + 1] if n + 1 < len(starts) else len(line)
|
||||
seg = line[s + 6:limit]
|
||||
close = seg.rfind("]")
|
||||
if close < 0:
|
||||
out.append((seg.split(","), True))
|
||||
continue
|
||||
inner = seg[:close]
|
||||
entries = inner.split(",")
|
||||
bad = "[" in inner or "]" in inner or any(e.strip() == "" for e in entries)
|
||||
out.append((entries, bad))
|
||||
return out
|
||||
|
||||
|
||||
def profile(body):
|
||||
"""Control-code profile of one key's body text (a dict, one value per CTL name)."""
|
||||
lists = opt_lists(body)
|
||||
return {
|
||||
"lines": len(body.split("\n")) if body else 0,
|
||||
"nl": len(RE_NL.findall(body)),
|
||||
"narr": len(RE_NARR.findall(body)),
|
||||
"brace": len(RE_BRACE.findall(body)),
|
||||
"optB": len(lists),
|
||||
"entries": [len(e) for e, _ in lists],
|
||||
"expr": sorted(RE_EXPR.findall(body)),
|
||||
"color": sorted(RE_COLOR.findall(body)),
|
||||
"fmt": (body.count("%s"), body.count("%d")),
|
||||
"token": tuple(len(r.findall(body)) for r in RE_TOKENS),
|
||||
"optchar": any(bad for _, bad in lists),
|
||||
}
|
||||
|
||||
|
||||
def ctl_diff(eng_body, body):
|
||||
"""Names of the checks where +body+ differs from +eng_body+ (optchar: true in body)."""
|
||||
a, b = profile(eng_body), profile(body)
|
||||
return [c for c in CTL if (b[c] if c == "optchar" else a[c] != b[c])]
|
||||
|
||||
|
||||
def txt_files(lang):
|
||||
folder = os.path.join(TEXT, lang)
|
||||
if not os.path.isdir(folder):
|
||||
return []
|
||||
return sorted(f for f in os.listdir(folder) if f.lower().endswith(".txt"))
|
||||
|
||||
|
||||
def table(rows, header):
|
||||
widths = [max(len(str(r[c])) for r in rows + [header]) for c in range(len(header))]
|
||||
fmt = " ".join("{:<%d}" % w if c == 0 else "{:>%d}" % w for c, w in enumerate(widths))
|
||||
out = [fmt.format(*header), fmt.format(*["-" * w for w in widths])]
|
||||
out.extend(fmt.format(*r) for r in rows)
|
||||
return "\n".join(out)
|
||||
|
||||
|
||||
def main(argv):
|
||||
only = None
|
||||
quiet = "--quiet" in argv
|
||||
if "--lang" in argv:
|
||||
only = argv[argv.index("--lang") + 1].upper()
|
||||
if not os.path.isdir(os.path.join(TEXT, BASE)):
|
||||
print("no Text/%s folder under %s" % (BASE, MOD))
|
||||
return 1
|
||||
eng = {}
|
||||
eng_notes = [0, 0, 0] # colon lines, CR, BOM
|
||||
for f in txt_files(BASE):
|
||||
blocks, colons, cr, bom = read_file(os.path.join(TEXT, BASE, f))
|
||||
eng[f] = blocks
|
||||
eng_notes[0] += len(colons)
|
||||
eng_notes[1] += cr
|
||||
eng_notes[2] += 1 if bom else 0
|
||||
eng_keys = sum(len(b) for b in eng.values())
|
||||
eng_ctl = [(f, k) for f, bl in eng.items() for k, v in bl.items() if profile(v)["optchar"]]
|
||||
langs = sorted(d for d in os.listdir(TEXT) if os.path.isdir(os.path.join(TEXT, d)) and d != BASE)
|
||||
if only:
|
||||
langs = [l for l in langs if l == only]
|
||||
header = ["file", "ENG", "present", "missing", "extra", "same", "colon", "CR", "BOM"]
|
||||
print("%s: %d keys in %d files, colon lines %d, CR %d, BOM %d, optB entry problems %d" %
|
||||
(BASE, eng_keys, len(eng), eng_notes[0], eng_notes[1], eng_notes[2], len(eng_ctl)))
|
||||
summary = []
|
||||
any_missing = False
|
||||
ctl_total = len(eng_ctl)
|
||||
details = []
|
||||
ctl_details = ["%s %s optchar (key %s)" % (BASE, f, k) for f, k in eng_ctl]
|
||||
ctl_header = ["file"] + CTL + ["keys"]
|
||||
for lang in langs:
|
||||
rows = []
|
||||
ctl_rows = []
|
||||
ctl_tot = [0] * (len(CTL) + 1)
|
||||
tot = [0] * 8 # ENG, present, missing, extra, same, colon, cr, bom
|
||||
files = txt_files(lang)
|
||||
for f in sorted(set(eng) | set(files)):
|
||||
e = eng.get(f, {})
|
||||
if f in files:
|
||||
blocks, colons, cr, bom = read_file(os.path.join(TEXT, lang, f))
|
||||
else:
|
||||
blocks, colons, cr, bom = {}, [], 0, False
|
||||
present = sum(1 for k in e if k in blocks)
|
||||
missing = [k for k in e if k not in blocks]
|
||||
extra = [k for k in blocks if k not in e]
|
||||
same = sum(1 for k in e if k in blocks and blocks[k] == e[k])
|
||||
rows.append([f if f in files else f + " (no file)", len(e), present, len(missing), len(extra),
|
||||
same, len(colons), cr, 1 if bom else 0])
|
||||
for n, v in enumerate([len(e), present, len(missing), len(extra), same, len(colons), cr, 1 if bom else 0]):
|
||||
tot[n] += v
|
||||
counts = [0] * len(CTL)
|
||||
bad_keys = 0
|
||||
for k in e:
|
||||
if k not in blocks:
|
||||
continue
|
||||
diff = ctl_diff(e[k], blocks[k])
|
||||
if diff:
|
||||
bad_keys += 1
|
||||
ctl_details.append("%s %s %s (key %s)" % (lang, f, ",".join(diff), k))
|
||||
for c in diff:
|
||||
counts[CTL.index(c)] += 1
|
||||
ctl_rows.append([f if f in files else f + " (no file)"] + counts + [bad_keys])
|
||||
for n, v in enumerate(counts + [bad_keys]):
|
||||
ctl_tot[n] += v
|
||||
for k in missing:
|
||||
details.append("%s %s missing %s" % (lang, f, k))
|
||||
for k in extra:
|
||||
details.append("%s %s extra %s" % (lang, f, k))
|
||||
for ln, k in colons:
|
||||
details.append("%s %s colon line %d (key %s)" % (lang, f, ln, k))
|
||||
rows.append(["TOTAL"] + tot)
|
||||
print("\n== %s ==" % lang)
|
||||
print(table([[str(c) for c in r] for r in rows], header))
|
||||
ctl_rows.append(["TOTAL"] + ctl_tot)
|
||||
print("\ncontrol codes vs ENG, %s (keys that differ per check)" % lang)
|
||||
print(table([[str(c) for c in r] for r in ctl_rows], ctl_header))
|
||||
ctl_total += ctl_tot[-1]
|
||||
translated = tot[1] - tot[4]
|
||||
pct = 100.0 * translated / tot[0] if tot[0] else 0.0
|
||||
summary.append("%s %d/%d translated (%.0f%%), %d same as ENG, %d missing, %d extra, %d control-code mismatches" %
|
||||
(lang, translated, tot[0], pct, tot[4], tot[2], tot[3], ctl_tot[-1]))
|
||||
if tot[2]:
|
||||
any_missing = True
|
||||
if details and not quiet:
|
||||
print("\n-- details (first 60) --")
|
||||
for d in details[:60]:
|
||||
print(d)
|
||||
if len(details) > 60:
|
||||
print("... %d more" % (len(details) - 60))
|
||||
if ctl_details and not quiet:
|
||||
print("\n-- control-code details (first 60) --")
|
||||
for d in ctl_details[:60]:
|
||||
print(d)
|
||||
if len(ctl_details) > 60:
|
||||
print("... %d more" % (len(ctl_details) - 60))
|
||||
print("\nSUMMARY: " + "; ".join(summary) if summary else "\nSUMMARY: no language folders")
|
||||
return 1 if (any_missing or ctl_total) else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
Reference in new issue
Block a user