#!/usr/bin/env python3 """Compare a source JSON language file with its translation. Usage: python3 check_i18n_json.py en.json # inspect the source file only python3 check_i18n_json.py en.json fr.json # compare source and translation Standard library only. Exit code 1 when an ERROR is found. """ import json import re import sys from collections import Counter SIMPLE_ARG = re.compile(r"\{\{?\s*([A-Za-z_][\w.]*)\s*\}?\}") # {name} {{name}} PRINTF = re.compile(r"%(?:\d+\$)?[-+0#]*\d*(?:\.\d+)?[sdifuxXeEgGc@]") # %s %1$s %d %@ ICU_ARG = re.compile(r"\{\s*([A-Za-z_]\w*)\s*,\s*(plural|select|selectordinal)\s*,") ICU_CASE = re.compile(r"(?]*>") URL = re.compile(r"https?://\S+") LETTER = re.compile(r"[^\W\d_]") MAX_SAFE = 2 ** 53 - 1 problems = [] def report(level, path, message): problems.append(level) print(f"{level:5} {path or '(root)'}: {message}") def load(path): pairs_seen = [] def hook(pairs): keys = [k for k, _ in pairs] pairs_seen.append([k for k, n in Counter(keys).items() if n > 1]) return dict(pairs) with open(path, encoding="utf-8-sig") as fh: text = fh.read() try: data = json.loads(text, object_pairs_hook=hook) except json.JSONDecodeError as err: print(f"ERROR {path}: not strict JSON ({err}). Comments, trailing commas " "and single quotes are not JSON.") sys.exit(1) dups = sorted({k for group in pairs_seen for k in group}) if dups: report("WARN", path, f"duplicate keys, only the last value survives: {dups}") return data def leaves(value, path=""): """Unambiguous paths: keys that contain dots are written as ["a.b"].""" if isinstance(value, dict): for key, child in value.items(): part = f'["{key}"]' if "." in key else key yield from leaves(child, f"{path}.{part}" if path and "." not in key else f"{path}{part}") elif isinstance(value, list): for i, child in enumerate(value): yield from leaves(child, f"{path}[{i}]") else: yield path, value def dotted_paths(value, path=""): """Dot-joined paths, the way a dot-path segment id would name them.""" if isinstance(value, dict): for key, child in value.items(): yield from dotted_paths(child, f"{path}.{key}" if path else key) elif isinstance(value, list): for i, child in enumerate(value): yield from dotted_paths(child, f"{path}.{i}" if path else str(i)) elif isinstance(value, str): yield path def icu_simple_args(text): """Simple {name} arguments of an ICU message, skipping case bodies like one {# file}.""" found = [] i = 0 def message(end_on_brace): nonlocal i while i < len(text): ch = text[i] if ch == "'" and i + 1 < len(text) and text[i + 1] in "{}'": i += 2 elif ch == "{": i += 1 argument() elif ch == "}": if end_on_brace: i += 1 return raise ValueError("unbalanced }") else: i += 1 if end_on_brace: raise ValueError("unclosed {") def argument(): nonlocal i close = text.find("}", i) comma = text.find(",", i) if close == -1: raise ValueError("unclosed {") if comma == -1 or close < comma: found.append(text[i:close].strip().strip("{}").strip()) i = close + 1 return kind = text[comma + 1:].split(",", 1)[0].strip() if kind not in ("plural", "select", "selectordinal"): found.append(text[i:comma].strip()) depth = 1 while i < len(text) and depth: depth += {"{": 1, "}": -1}.get(text[i], 0) i += 1 return i = text.index(",", comma + 1) + 1 while True: while i < len(text) and text[i].isspace(): i += 1 if i < len(text) and text[i] == "}": i += 1 return brace = text.find("{", i) if brace == -1: raise ValueError("ICU case without a body") i = brace + 1 message(True) message(False) return found def tokens(text): if ICU_ARG.search(text): try: simple = Counter(icu_simple_args(text)) except ValueError: simple = Counter(["(broken ICU syntax)"]) else: simple = Counter(SIMPLE_ARG.findall(text)) return { "placeholders": simple + Counter(PRINTF.findall(text)), "icu_args": Counter(ICU_ARG.findall(text)), "tags": Counter(TAG.findall(text)), "urls": Counter(URL.findall(text)), } def inspect_source(data): strings = numbers = 0 for path, value in leaves(data): if isinstance(value, str): strings += 1 elif isinstance(value, int) and not isinstance(value, bool): numbers += 1 if abs(value) > MAX_SAFE: report("WARN", path, f"{value} is above 2^53 and can change in a " "JavaScript round trip; store IDs as strings") collisions = [p for p, n in Counter(dotted_paths(data)).items() if n > 1] for path in collisions: report("WARN", path, "a flat key with dots and a nested key resolve to the " "same dot path; tools that address strings by dot path can mix them up") numeric_keys = [p for p, _ in leaves(data) if re.search(r"(^|\.)\d+$", p)] if numeric_keys: report("INFO", "", f"{len(numeric_keys)} keys look like integers; JavaScript " "sorts those first and in numeric order when it rewrites the object") print(f"INFO source: {strings} strings, {numbers} numbers") def compare(src, dst): src_leaves = dict(leaves(src)) dst_leaves = dict(leaves(dst)) for path in sorted(set(src_leaves) - set(dst_leaves)): report("ERROR", path, "missing in translation") for path in sorted(set(dst_leaves) - set(src_leaves)): report("ERROR", path, "not in source (renamed, added or a _translated sibling?)") same = 0 for path in sorted(set(src_leaves) & set(dst_leaves)): a, b = src_leaves[path], dst_leaves[path] if type(a) is not type(b): report("ERROR", path, f"type changed from {type(a).__name__} to {type(b).__name__}") continue if not isinstance(a, str): if a != b: report("ERROR", path, f"non-text value changed: {a!r} -> {b!r}") continue ta, tb = tokens(a), tokens(b) for kind in ("placeholders", "icu_args", "tags", "urls"): if ta[kind] != tb[kind]: lost = sorted((ta[kind] - tb[kind]).elements()) extra = sorted((tb[kind] - ta[kind]).elements()) report("ERROR", path, f"{kind} differ; lost {lost} extra {extra}") if ta["icu_args"]: ca, cb = set(ICU_CASE.findall(a)), set(ICU_CASE.findall(b)) if "other" not in cb: report("ERROR", path, "ICU message has no 'other' case") elif ca != cb: report("WARN", path, f"plural cases {sorted(ca)} -> {sorted(cb)}; fine if " "they match the target language's CLDR categories") if a == b and LETTER.search(a) and not URL.fullmatch(a.strip()): same += 1 report("INFO", path, f"identical to source: {a[:60]!r}") print(f"INFO compared {len(src_leaves)} source values, {same} unchanged strings") def main(): if len(sys.argv) not in (2, 3): print(__doc__) sys.exit(2) src = load(sys.argv[1]) inspect_source(src) if len(sys.argv) == 3: compare(src, load(sys.argv[2])) sys.exit(1 if "ERROR" in problems else 0) if __name__ == "__main__": main()