build(gettext): extend the "translate-desktop" command to cover the G…

…NOME Shell extension

In the original (since lost) 1cfd35b, we tried to treat the GNOME Shell
extension as a separate "securedrop/extension" Weblate component with
its own "translate-extension" command, replicating both
"--extract-update" and "--compile" modes.

In f3d3f04, we tried to treat the GNOME Shell extension as an extension
(sorry) of the main "securedrop/securedrop" component, taking advantage
of Babel's mapping feature to parse the ".js.in" file as JavaScript.

Here we harmonize the two approaches.  "translate-desktop
--extract-update" is now a two-step process, first via Babel over the
".js.in" JavaScript file, then as usual via xgettext over the ".j2.in"
desktop templates.  (This is logically reversed, but Babel appears to
have no equivalent of xgettext's appending "--join-existing" mode.)
"translate-desktop --compile" similarly does two passes with msgfmt,
first to create the standard gettext directory layout and then to update
the desktop templates.

NB. We also add "msgmerge --no-fuzzy-matching" after #6772, which
required rebasing from "develop".

.gitignore

@@ -94,8 +94,8 @@ coverage.xml
 .hypothesis/
 .mypy_cache/
 
-# Translations
-*.mo
+# Translations compiled during packaging:
+securedrop/translations/**/*.mo
 
 # Flask stuff:
 instance/

securedrop/i18n_tool.py

@@ -27,7 +27,6 @@
 LOCALE_DIR = {
     "securedrop": "securedrop/translations",
     "desktop": "install_files/ansible-base/roles/tails-config/templates",
-    "extension": "install_files/ansible-base/roles/tails-config/templates",
 }
 
 
@@ -142,9 +141,44 @@ def translate_desktop(self, args: argparse.Namespace) -> None:
 
         if args.extract_update:
             sources = args.sources.split(",")
+            xgettext_flags = []
+
+            # First extract from JavaScript sources via Babel ("xgettext
+            # --language=JavaScript" can't parse gettext as exposed by GNOME
+            # JavaScript):
+            js_sources = [source for source in sources if ".js" in source]
+            if js_sources:
+                xgettext_flags.append("--join-existing")
+                subprocess.check_call(
+                    [
+                        "pybabel",
+                        "extract",
+                        "--charset=utf-8",
+                        "--mapping",
+                        args.mapping,
+                        "--output",
+                        "desktop.pot",
+                        "--project=SecureDrop",
+                        "--version",
+                        args.version,
+                        "[email protected]",
+                        "--copyright-holder=Freedom of the Press Foundation",
+                        "--add-comments=Translators:",
+                        "--strip-comments",
+                        "--add-location=never",
+                        "--no-wrap",
+                        *js_sources,
+                    ],
+                    cwd=args.translations_dir,
+                )
+
+            # Then extract from desktop templates via xgettext in appending
+            # "--join-existing" mode:
+            desktop_sources = [source for source in sources if ".js" not in source]
             subprocess.check_call(
                 [
                     "xgettext",
+                    *xgettext_flags,
                     "--output=desktop.pot",
                     "--language=Desktop",
                     "--keyword",
@@ -153,7 +187,7 @@ def translate_desktop(self, args: argparse.Namespace) -> None:
                     args.version,
                     "[email protected]",
                     "--copyright-holder=Freedom of the Press Foundation",
-                    *sources,
+                    *desktop_sources,
                 ],
                 cwd=args.translations_dir,
             )
@@ -168,7 +202,9 @@ def translate_desktop(self, args: argparse.Namespace) -> None:
                     if not f.endswith(".po"):
                         continue
                     po_file = os.path.join(args.translations_dir, f)
-                    subprocess.check_call(["msgmerge", "--update", po_file, messages_file])
+                    subprocess.check_call(
+                        ["msgmerge", "--no-fuzzy-matching", "--update", po_file, messages_file]
+                    )
                 log.warning(f"messages translations updated in {messages_file}")
             else:
                 log.warning("desktop translations are already up to date")
@@ -181,7 +217,34 @@ def translate_desktop(self, args: argparse.Namespace) -> None:
             try:
                 open(linguas_file, "w").write(content)
 
-                for source in args.sources.split(","):
+                # First, compile each message catalog for normal gettext use.
+                # We have to iterate over them, rather than using "pybabel
+                # compile --directory", in order to replicate gettext's
+                # standard "{locale}/LC_MESSAGES/{domain}.mo" structure (like
+                # "securedrop/translations").
+                locale_dir = os.path.join(args.translations_dir, "locale")
+                for po in pos:
+                    locale = po.replace(".po", "")
+                    output_dir = os.path.join(locale_dir, locale, "LC_MESSAGES")
+                    subprocess.run(["mkdir", "--parents", output_dir], check=True)
+                    subprocess.run(
+                        [
+                            "msgfmt",
+                            "--output-file",
+                            os.path.join(output_dir, "messages.mo"),
+                            po,
+                        ],
+                        check=True,
+                        stdout=subprocess.DEVNULL,
+                        stderr=subprocess.DEVNULL,
+                        cwd=args.translations_dir,
+                    )
+
+                # Then use msgfmt to update the desktop files as usual.
+                desktop_sources = [
+                    source for source in args.sources.split(",") if ".js" not in source
+                ]
+                for source in desktop_sources:
                     target = source.rstrip(".in")
                     subprocess.check_call(
                         [
@@ -247,8 +310,7 @@ def set_translate_messages_parser(self, subps: _SubParsersAction) -> None:
             "translate-messages", help=("Update and compile " "source and template translations")
         )
         translations_dir = join(dirname(realpath(__file__)), "translations")
-        extension_location = join(dirname(realpath(__file__)), "..", LOCALE_DIR["extension"])
-        sources = join(".,source_templates,journalist_templates,", extension_location)
+        sources = ".,source_templates,journalist_templates"
         self.set_translate_parser(parser, translations_dir, sources)
         mapping = "babel.cfg"
         parser.add_argument(
@@ -267,7 +329,19 @@ def set_translate_desktop_parser(self, subps: _SubParsersAction) -> None:
             "..",
             LOCALE_DIR["desktop"],
         )
-        sources = "desktop-journalist-icon.j2.in,desktop-source-icon.j2.in"
+        sources = ",".join(
+            [
+                "desktop-journalist-icon.j2.in",
+                "desktop-source-icon.j2.in",
+                "extension.js.in",
+            ]
+        )
+        mapping = join(dirname(realpath(__file__)), "babel.cfg")
+        parser.add_argument(
+            "--mapping",
+            default=mapping,
+            help=f"Mapping of files to consider (default {mapping})",
+        )
         self.set_translate_parser(parser, translations_dir, sources)
         parser.set_defaults(func=self.translate_desktop)