#!/usr/bin/env python3 """Prepare a long .txt file for translation and check the translated file afterwards. Commands: inspect FILE encoding, BOM, line endings, hard wrapping, chapter headings convert FILE OUT write UTF-8 (no BOM) with LF line endings --encoding NAME use this encoding instead of the detected one --unwrap join hard-wrapped lines inside each paragraph compare SOURCE TRANSLATED paragraph, chapter, and line counts side by side, plus leftover source-script characters --source-encoding / --translated-encoding when a file is not UTF-8 Chapter detection knows English and Chinese/Japanese heading patterns; add your target language's word for "chapter" to CHAPTER if it differs. The leftover-script check only helps when source and target use different scripts. Python 3.8+ standard library only. Written by BookTranslator (https://www.booktranslator.app). Free to reuse. """ import argparse import codecs import os import re import sys import unicodedata from collections import Counter BOMS = [ (codecs.BOM_UTF8, "utf-8-sig"), (codecs.BOM_UTF32_LE, "utf-32"), (codecs.BOM_UTF32_BE, "utf-32"), (codecs.BOM_UTF16_LE, "utf-16"), (codecs.BOM_UTF16_BE, "utf-16"), ] # Legacy encodings worth trying when the file is not valid UTF-8. CANDIDATES = ["gb18030", "big5", "cp932", "euc-kr", "cp1251", "cp1252"] CHAPTER = re.compile( r"^\s*(" r"(chapter|part|book|prologue|epilogue)\b.*" # Chapter 1, Part Two, Prologue r"|第[0-9０-９一二三四五六七八九十百千零〇两]+[章节回卷話话部]" # 第一章, 第12回, 第三話 r"|[0-9]+\.?\s*$" # a bare number on its own line r"|[ivxlcdm]+\.?\s*$" # a bare roman numeral r")", re.IGNORECASE, ) SENTENCE_END = tuple('.!?。！？…"”’」』)）') def detect(raw): for bom, name in BOMS: if raw.startswith(bom): return name, "byte order mark", [] if raw[:200].count(b"\x00") > 20: return None, "many NUL bytes: probably UTF-16 or UTF-32 without a BOM", [] try: raw.decode("utf-8") ascii_only = all(b < 128 for b in raw) return "utf-8", "plain ASCII" if ascii_only else "valid UTF-8", [] except UnicodeDecodeError: pass clean = [] for name in CANDIDATES: try: clean.append((name, raw.decode(name))) except UnicodeDecodeError: continue return None, "not UTF-8", clean def paragraphs(text): return [p for p in re.split(r"\n\s*\n", text) if p.strip()] def wrap_report(text): lines = [l for l in text.split("\n") if l.strip()] if not lines: return None lengths = Counter(len(l) for l in lines) longest = max(lengths) near_max = sum(n for length, n in lengths.items() if length >= longest - 12) multi = [p for p in paragraphs(text) if p.count("\n") >= 1] broken = 0 for p in multi: for line in p.split("\n")[:-1]: if not line.rstrip().endswith(SENTENCE_END): broken += 1 return longest, near_max, len(lines), broken def unwrap(text, width_ratio=0.75): """Join lines inside a paragraph when most of its lines run close to the wrap width. A paragraph is joined when at least 70% of its lines (not counting the last) are at least 75% of the file's longest line. Blocks made of short lines (verse, letters, addresses, lists) are left alone. """ out = [] lines = [l for l in text.split("\n") if l.strip()] width = max((len(l) for l in lines), default=0) for p in re.split(r"\n\s*\n", text.strip("\n")): rows = p.split("\n") body = rows[:-1] near = sum(1 for r in body if len(r.rstrip()) >= width * width_ratio) if len(rows) > 1 and body and near / len(body) >= 0.7: joined = rows[0].rstrip() for r in rows[1:]: if re.search(r"[　-ヿ㐀-鿿＀-￯]$", joined): joined += r.strip() # no space between CJK lines else: joined += " " + r.strip() out.append(joined) else: out.append(p) return "\n\n".join(out) def read(path, encoding=None): raw = open(path, "rb").read() if encoding: return raw.decode(encoding), encoding name, _, _ = detect(raw) if name is None: sys.exit(f"{path}: encoding not certain; run inspect and pass --encoding") return raw.decode(name), name def normalize(text): return text.replace("\r\n", "\n").replace("\r", "\n").lstrip("") def cmd_inspect(args): raw = open(args.file, "rb").read() name, why, clean = detect(raw) print(f"file: {args.file} ({len(raw)} bytes)") if name: print(f"encoding: {name} ({why})") else: print(f"encoding: unknown ({why})") if clean: print("these legacy encodings decode without errors; read the previews,") print("because a wrong one can still produce real-looking characters:") for enc, text in clean: first = next((l for l in text.splitlines() if l.strip()), "")[:40] print(f" {enc:8} {first}") else: print("no candidate decodes cleanly; the file may be damaged or mixed") return text = raw.decode(name) crlf, cr = text.count("\r\n"), text.count("\r") - text.count("\r\n") lf = text.count("\n") - crlf print(f"line endings: LF={lf} CRLF={crlf} CR={cr}") text = normalize(text) bad = text.count("�") if bad: print(f"replacement characters (U+FFFD): {bad} <- text was already damaged") paras = paragraphs(text) print(f"paragraphs (blank-line separated): {len(paras)}") report = wrap_report(text) if report: longest, near, total, broken = report print(f"non-empty lines: {total}; longest line: {longest} chars; " f"{near} lines within 12 chars of the longest") print(f"line breaks inside paragraphs that do not end a sentence: {broken}") if broken and near / total > 0.5: print("looks hard-wrapped: consider convert --unwrap") heads = [l.strip() for l in text.split("\n") if CHAPTER.match(l) and len(l.strip()) < 60] print(f"chapter-like headings: {len(heads)}") for h in heads: print(f" {h}") def cmd_convert(args): if os.path.abspath(args.file) == os.path.abspath(args.out): sys.exit("OUT must be a different file; convert never overwrites the original") text, used = read(args.file, args.encoding) text = normalize(text) if args.unwrap: text = unwrap(text) with open(args.out, "w", encoding="utf-8", newline="\n") as f: f.write(text if text.endswith("\n") else text + "\n") print(f"wrote {args.out}: UTF-8, LF, from {used}" + (", unwrapped" if args.unwrap else "")) def script_of(ch): name = unicodedata.name(ch, "") for s in ("CJK", "HIRAGANA", "KATAKANA", "HANGUL", "CYRILLIC", "ARABIC", "HEBREW", "GREEK", "THAI", "LATIN"): if name.startswith(s): return s return None def cmd_compare(args): a, _ = read(args.source, args.source_encoding) b, _ = read(args.translated, args.translated_encoding) a, b = normalize(a), normalize(b) def stats(t): lines = t.split("\n") return { "paragraphs": len(paragraphs(t)), "chapter headings": sum(1 for l in lines if CHAPTER.match(l) and len(l.strip()) < 60), "non-empty lines": sum(1 for l in lines if l.strip()), "scene breaks (* * *)": sum(1 for l in lines if l.strip() in ("* * *", "***", "＊＊＊")), "replacement chars": t.count("�"), } sa, sb = stats(a), stats(b) print(f"{'':24}{'source':>10}{'translated':>12}") for k in sa: flag = "" if sa[k] == sb[k] else " <- differs" print(f"{k:24}{sa[k]:>10}{sb[k]:>12}{flag}") src_scripts = Counter(filter(None, map(script_of, a))) tgt_scripts = Counter(filter(None, map(script_of, b))) main_src = src_scripts.most_common(1)[0][0] if src_scripts else None main_tgt = tgt_scripts.most_common(1)[0][0] if tgt_scripts else None if main_src and main_src != main_tgt: left = tgt_scripts.get(main_src, 0) print(f"source script {main_src} characters left in translation: {left}") def main(): ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) sub = ap.add_subparsers(dest="cmd", required=True) p = sub.add_parser("inspect"); p.add_argument("file"); p.set_defaults(fn=cmd_inspect) p = sub.add_parser("convert"); p.add_argument("file"); p.add_argument("out") p.add_argument("--encoding"); p.add_argument("--unwrap", action="store_true"); p.set_defaults(fn=cmd_convert) p = sub.add_parser("compare"); p.add_argument("source"); p.add_argument("translated") p.add_argument("--source-encoding"); p.add_argument("--translated-encoding"); p.set_defaults(fn=cmd_compare) args = ap.parse_args() args.fn(args) if __name__ == "__main__": main()