Merge pull request #30 from thunderbird/sync2upstream

Sync update workflow to upstream

.github/CODEOWNERS

@@ -2,5 +2,4 @@
 * @thunderbird/l10n-thunderbird
 
 # Code reviewers (workflows, scripts)
-/_scripts/ @thunderbird/build-release @jfx2006
-/.github/ @thunderbird/build-release @jfx2006
+/.github/ @jfx2006

.github/scripts/diff.py

@@ -0,0 +1,90 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+"""
+Compare messages in localization files between two paths
+"""
+
+from __future__ import annotations
+
+from argparse import ArgumentParser
+from os import walk
+from os.path import commonpath, exists, join, relpath
+from sys import exit
+
+from moz.l10n.resource import UnsupportedResource, l10n_equal, parse_resource
+
+
+def diff_tree(a_root: str, b_root: str, ignore: set[str]) -> int:
+    common_root = commonpath((a_root, b_root))
+    diff_count = 0
+    b_seen: set[str] = set()
+    for dirpath, dirnames, filenames in walk(a_root):
+        dirnames[:] = [dn for dn in dirnames if not dn.startswith((".", "_"))]
+        for fn in filenames:
+            if not fn.startswith((".", "_")):
+                a_path = join(dirpath, fn)
+                rel_path = relpath(a_path, a_root)
+                if rel_path in ignore:
+                    continue
+                b_path = join(b_root, rel_path)
+                if not exists(b_path):
+                    print(f"Missing file: {relpath(b_path, common_root)}")
+                    diff_count += 1
+                elif not l10n_equal_paths(a_path, b_path):
+                    print(f"Files at {rel_path} differ")
+                    diff_count += 1
+                b_seen.add(b_path)
+    for dirpath, dirnames, filenames in walk(b_root):
+        dirnames[:] = [dn for dn in dirnames if not dn.startswith((".", "_"))]
+        for fn in filenames:
+            if not fn.startswith((".", "_")):
+                b_path = join(dirpath, fn)
+                rel_path = relpath(b_path, b_root)
+                if rel_path not in ignore and b_path not in b_seen:
+                    a_path = join(a_root, rel_path)
+                    print(f"Missing file: {relpath(a_path, common_root)}")
+                    diff_count += 1
+    return diff_count
+
+
+def l10n_equal_paths(a_path: str, b_path: str) -> bool:
+    with open(a_path, "rb") as a_file:
+        a_bytes = a_file.read()
+    with open(b_path, "rb") as b_file:
+        b_bytes = b_file.read()
+    if a_bytes == b_bytes:
+        return True
+
+    try:
+        a_res = parse_resource(a_path, a_bytes)
+    except UnsupportedResource:
+        a_res = None
+    except Exception as error:
+        print(f"Parse error at {a_path}: {error}")
+        return False
+    try:
+        b_res = parse_resource(b_path, b_bytes)
+    except Exception as error:
+        if isinstance(error, UnsupportedResource) and a_res is None:
+            return True
+        else:
+            print(f"Parse error at {b_path}: {error}")
+            return False
+    if a_res is None:
+        print(f"Parse error at {a_path}")
+        return False
+
+    return l10n_equal(a_res, b_res)
+
+
+if __name__ == "__main__":
+    parser = ArgumentParser(description=__doc__)
+    parser.add_argument("path", nargs=2, help="Root directories for the comparison")
+    parser.add_argument("--ignore", nargs="*", help="Relative paths to ignore")
+    args = parser.parse_args()
+
+    ignore = set(args.ignore) if args.ignore else set()
+    if diff_tree(args.path[0], args.path[1], ignore):
+        exit(1)

_scripts/prune.py → .github/scripts/prune.py

@@ -14,41 +14,39 @@
 
 import json
 from argparse import ArgumentParser
-from codecs import encode
-from os import getcwd, remove, scandir, walk
+from os import getcwd, remove, scandir
 from os.path import join, relpath, splitext
 from sys import exit
-from compare_locales.parser import Entity, getParser
+from moz.l10n.paths import L10nDiscoverPaths
+from moz.l10n.resource import parse_resource, serialize_resource
+from moz.l10n.resource.data import Entry
 
 
 def prune_file(path: str, msg_refs: set[str]):
-    parser = getParser(path)
     with open(path, "+rb") as file:
-        parser.readContents(file.read())
-        resource = [entity for entity in parser.walk()]
-        drop = set(
-            e.key for e in resource if isinstance(e, Entity) and e.key not in msg_refs
-        )
-        if drop:
-            print(f"drop {len(drop)} from {path}")
-            res = ""
-            trim = False
-            for entity in resource:
-                # As whitespace is handled as a separate entity,
-                # a trailing newline after a dropped entity needs to be
-                # trimmed separately.
-                if isinstance(entity, Entity) and entity.key in drop:
-                    trim = True
-                elif trim:
-                    el_str = entity.all
-                    res += el_str[1:] if el_str[0] == "\n" else el_str
-                    trim = False
-                else:
-                    res += entity.all
+        resource = parse_resource(path, file.read())
+        drop_count = 0
+        for section in resource.sections:
+            next = [
+                entry
+                for entry in section.entries
+                if not isinstance(entry, Entry)
+                or ".".join(section.id + entry.id) in msg_refs
+            ]
+            diff = len(section.entries) - len(next)
+            if diff > 0:
+                drop_count += diff
+                section.entries = next
+        resource.sections = [
+            section for section in resource.sections if section.entries
+        ]
+        if drop_count:
+            print(f"drop {drop_count} from {path}")
             file.seek(0)
-            file.write(encode(res, parser.encoding))
+            for line in serialize_resource(resource):
+                file.write(line.encode("utf-8"))
             file.truncate()
-    return len(drop)
+    return drop_count
 
 
 def prune(branches: list[str]):
@@ -80,17 +78,14 @@ def prune(branches: list[str]):
     if expected:
         exit(f"Incomplete data! Not found: {expected}")
 
-    for entry in scandir(cwd):
-        if entry.is_dir() and not entry.name.startswith(("_", ".")):
-            for root, _, files in walk(entry.path):
-                for file in files:
-                    path = relpath(join(root, file), cwd)
-                    if path not in refs:
-                        print(f"remove {path}")
-                        remove(path)
-                        removed_files += 1
-                    elif refs[path]:
-                        removed_messages += prune_file(path, refs[path])
+    for ref_path in L10nDiscoverPaths(cwd, ref_root=cwd).ref_paths:
+        path = relpath(ref_path, cwd)
+        if path not in refs:
+            print(f"remove {path}")
+            remove(path)
+            removed_files += 1
+        elif refs[path]:
+            removed_messages += prune_file(path, refs[path])
     return removed_data, removed_files, removed_messages
 
 
@@ -109,15 +104,11 @@ def write_commit_msg(data: list[str], files: int, messages: int):
 
 
 if __name__ == "__main__":
-    prog = "python -m _scripts.prune"
-    parser = ArgumentParser(
-        prog=prog,
-        description=__doc__,
-        epilog=f"Example: {prog} master beta release",
-    )
+    prog = "python .github/scripts/prune.py"
+    parser = ArgumentParser(prog=prog, description=__doc__)
     args = parser.parse_args()
 
-    config_file = join("_configs", "config.json")
+    config_file = join(".github", "update-config.json")
     with open(config_file) as f:
         cfg_automation = json.load(f)
 

.github/scripts/requirements.txt

@@ -0,0 +1,2 @@
+moz.l10n ~= 0.3.0
+tomli-w ~= 1.0