#!/usr/bin/env python3
"""Compare a translated HTML file with its source and report structural damage.

Usage:
    python3 check_html_translation.py source.html translated.html [more.html ...]
    python3 check_html_translation.py source.html out.html --lang zh-Hans --keep Shelfmark

Checks (Python standard library only):
  tags          same start tags in the same order
  attributes    every attribute that is not reader-facing text is unchanged
                (href, src, id, class, for, name, type, data-*, width, ...)
  text attrs    alt, title, placeholder, aria-label and the description/og:title
                meta content were translated (still identical to English = flagged)
  lang          the html element's lang attribute equals --lang
  code          text inside <code>, <pre>, <script> and <style> is unchanged,
                except the JSON-LD block, which only needs to stay valid JSON
                with the same keys
  json-ld       JSON-LD still parses and has the same keys
  no-translate  text inside translate="no" elements is unchanged
  comments      HTML comments are unchanged
  fragments     every href="#id" still points at an id in the file
  english       visible text runs that still look like untranslated English
  spacing       a space between CJK text and CJK text across an inline tag
  inline code   <code> spans keep their text; a changed order is a warning

Exit code is 0 when every file passes the hard checks (tags, attributes, code,
inline code text, json-ld, no-translate, comments, fragments). Text attrs, lang, english and
spacing are reported as warnings because they can be deliberate decisions.

Written by BookTranslator (https://www.booktranslator.app). Free to reuse.
"""

import argparse
import json
import re
import sys
from html.parser import HTMLParser

TEXT_ATTRS = {"alt", "title", "placeholder", "aria-label"}
INLINE = {"a", "b", "strong", "em", "i", "code", "span"}
VOID = {"meta", "link", "img", "input", "br", "hr", "source", "wbr"}
CJK = re.compile(r"[　-〿぀-ヿ㐀-鿿가-힯＀-￯]")


class Doc(HTMLParser):
    def __init__(self, raw):
        super().__init__(convert_charrefs=True)
        self.raw = raw
        self.tags, self.attrs, self.text_attrs = [], [], []
        self.code, self.inline_code, self.comments, self.notrans, self.visible = [], [], [], [], []
        self.ids, self.fragments, self.jsonld, self.lang = set(), [], [], None
        self.stack = []

    def handle_starttag(self, tag, attrs):
        a = dict(attrs)
        self.tags.append(tag)
        meta_text = tag == "meta" and (a.get("name") == "description" or a.get("property") == "og:title")
        fixed = {k: v for k, v in a.items() if k not in TEXT_ATTRS and not (meta_text and k == "content")
                 and not (tag == "html" and k == "lang")}
        self.attrs.append((tag, sorted(fixed.items())))
        for k in sorted(TEXT_ATTRS & a.keys()):
            self.text_attrs.append((tag, k, a[k]))
        if meta_text:
            self.text_attrs.append((tag, "content", a.get("content", "")))
        if tag == "html":
            self.lang = a.get("lang")
        if "id" in a:
            self.ids.add(a["id"])
        if a.get("href", "").startswith("#"):
            self.fragments.append(a["href"][1:])
        if tag not in VOID:
            kind = "jsonld" if tag == "script" and a.get("type") == "application/ld+json" else tag
            self.stack.append((tag, kind, a.get("translate") == "no"))

    def handle_endtag(self, tag):
        for i in range(len(self.stack) - 1, -1, -1):
            if self.stack[i][0] == tag:
                del self.stack[i:]
                break

    def handle_data(self, data):
        kinds = {k for _, k, _ in self.stack}
        if "jsonld" in kinds:
            self.jsonld.append(data)
        elif kinds & {"pre", "script", "style"}:
            self.code.append(data)
        elif "code" in kinds:
            self.inline_code.append(data)
        elif any(n for _, _, n in self.stack):
            self.notrans.append(data.strip())
        elif data.strip():
            self.visible.append(data.strip())

    def handle_comment(self, data):
        self.comments.append(data)


def parse(path):
    raw = open(path, encoding="utf-8").read()
    d = Doc(raw)
    d.feed(raw)
    d.close()
    return d


def spacing_issues(raw):
    """Spaces where CJK text meets CJK text across an inline tag, e.g. '， <strong>备份'."""
    names = "|".join(INLINE)
    hits = []
    for m in re.finditer(r"(\S)( +)(<(?:%s)\b[^>]*>)(\S)" % names, raw):
        if CJK.search(m.group(1)) and CJK.search(m.group(4)):
            hits.append(m.group(0))
    for m in re.finditer(r"(\S)(</(?:%s)>)( +)(\S)" % names, raw):
        if CJK.search(m.group(1)) and CJK.search(m.group(4)):
            hits.append(m.group(0))
    return sorted(set(hits))


def jsonld_keys(blocks):
    out = []
    for b in blocks:
        try:
            out.append(sorted(json.loads(b).keys()))
        except ValueError:
            out.append(None)
    return out


def check(src, out, path, lang, keep):
    errors, warnings = [], []
    if src.tags != out.tags:
        errors.append(f"tags: {len(src.tags)} in source, {len(out.tags)} in translation, "
                      f"first difference at #{next((i for i, (a, b) in enumerate(zip(src.tags, out.tags)) if a != b), min(len(src.tags), len(out.tags)))}")
    else:
        for (tag, a), (_, b) in zip(src.attrs, out.attrs):
            if a != b:
                errors.append(f"attributes changed on <{tag}>: {dict(a)} -> {dict(b)}")
    same = [(t, k, v) for (t, k, v), (_, _, w) in zip(src.text_attrs, out.text_attrs) if v == w]
    if same:
        warnings.append("text attrs left in English: " + "; ".join(f"<{t} {k}=\"{v}\">" for t, k, v in same))
    if lang and out.lang != lang:
        warnings.append(f"lang is {out.lang!r}, expected {lang!r}")
    if src.code != out.code:
        changed = []
        for a, b in zip(src.code, out.code):
            if a != b:
                al, bl = a.splitlines(), b.splitlines()
                changed += [f"{x.strip()!r} -> {y.strip()!r}" for x, y in zip(al, bl) if x != y] or ["line count differs"]
        errors.append("pre/script/style changed: " + "; ".join(changed or ["block count differs"]))
    if sorted(src.inline_code) != sorted(out.inline_code):
        errors.append(f"inline code changed: {sorted(set(src.inline_code) ^ set(out.inline_code))}")
    elif src.inline_code != out.inline_code:
        warnings.append("inline code reordered (normal for some languages; read the sentence): "
                        + " ".join(out.inline_code))
    if jsonld_keys(src.jsonld) != jsonld_keys(out.jsonld):
        errors.append("json-ld no longer parses or keys changed")
    elif src.jsonld == out.jsonld:
        warnings.append("json-ld name/description left in English")
    if src.notrans != out.notrans:
        errors.append(f"translate=\"no\" text changed: {src.notrans} -> {out.notrans}")
    if src.comments != out.comments:
        errors.append("HTML comments changed")
    broken = [f for f in out.fragments if f not in out.ids]
    if broken:
        errors.append("fragment links without a target: " + ", ".join("#" + f for f in broken))
    english = [t for t in out.visible
               if re.search(r"[A-Za-z]{3,}", re.sub("|".join(map(re.escape, keep)) or "$^", "", t))
               and not CJK.search(t) and not re.fullmatch(r"[\w.@+-]+", t)]
    if english:
        warnings.append("visible text still in English: " + "; ".join(repr(t[:40]) for t in english))
    sp = spacing_issues(out.raw)
    if sp:
        warnings.append(f"{len(sp)} space(s) between CJK text across an inline tag: " + "; ".join(repr(s) for s in sp))
    return errors, warnings


def main():
    ap = argparse.ArgumentParser()
    ap.add_argument("source")
    ap.add_argument("translated", nargs="+")
    ap.add_argument("--lang", default=None, help="expected lang attribute, e.g. zh-Hans")
    ap.add_argument("--keep", action="append", default=[], help="term allowed to stay in English (repeatable)")
    args = ap.parse_args()
    src = parse(args.source)
    failed = False
    for path in args.translated:
        errors, warnings = check(src, parse(path), path, args.lang, args.keep)
        print(f"== {path}: {'FAIL' if errors else 'PASS'} ({len(errors)} errors, {len(warnings)} warnings)")
        for e in errors:
            print("  ERROR  " + e)
        for w in warnings:
            print("  warn   " + w)
        failed = failed or bool(errors)
    sys.exit(1 if failed else 0)


if __name__ == "__main__":
    main()