Add support for searchfox data

src/fuzzfetch/args.py

@@ -27,17 +27,13 @@ def __init__(self) -> None:
         if not hasattr(self, "parser"):
             self.parser = ArgumentParser(conflict_handler="resolve", prog="fuzzfetch")
 
-        self.parser.set_defaults(
-            target="firefox", build="latest", tests=None
-        )  # branch default is set after parsing
-
         target_group = self.parser.add_argument_group("Target")
         target_group.add_argument(
             "--target",
             nargs="*",
-            default=FetcherArgs.DEFAULT_TARGETS,
+            default=[],
             help="Specify the build artifacts to download. "
-            "Valid options: firefox js common gtest mozharness"
+            "Valid options: firefox js common gtest mozharness searchfox "
             f"(default: {' '.join(FetcherArgs.DEFAULT_TARGETS)})",
         )
         target_group.add_argument(
@@ -62,6 +58,7 @@ def __init__(self) -> None:
         type_group = self.parser.add_argument_group("Build")
         type_group.add_argument(
             "--build",
+            default="latest",
             metavar="DATE|REV|NS",
             help="Specify the build to download, (default: %(default)s)"
             " Accepts values in format YYYY-MM-DD (2017-01-01)"
@@ -157,6 +154,9 @@ def __init__(self) -> None:
         build_group.add_argument(
             "--nyx", action="store_true", help="Download Nyx snapshot builds."
         )
+        build_group.add_argument(
+            "--searchfox", action="store_true", help="Download searchfox data."
+        )
 
         self.parser.add_argument(
             "--gtest",
@@ -251,13 +251,8 @@ def sanity_check(self, args: Namespace) -> None:
                 self.parser.error("Cannot specify --build namespace and --fuzzilli")
             if args.nyx:
                 self.parser.error("Cannot specify --build namespace and --nyx")
-
-        if args.gtest:
-            LOG.warning(
-                "--gtest is deprecated, add 'gtest' to --target instead "
-                "(e.g. --target firefox gtest)"
-            )
-            args.target.append("gtest")
+            if args.searchfox:
+                self.parser.error("Cannot specify --build namespace and --searchfox")
 
         if "firefox" in args.target and args.fuzzilli:
             self.parser.error("Cannot specify --target firefox and --fuzzilli")
@@ -269,5 +264,22 @@ def parse_args(self, argv: Optional[Sequence[str]] = None) -> Namespace:
             argv: a list of arguments
         """
         args = self.parser.parse_args(argv)
+
+        if not args.target and (
+            args.searchfox
+            or (self.is_build_ns(args.build) and "-searchfox" in args.build)
+        ):
+            args.target.append("searchfox")
+
+        if not args.target:
+            args.target.extend(self.DEFAULT_TARGETS)
+
+        if args.gtest:
+            LOG.warning(
+                "--gtest is deprecated, add 'gtest' to --target instead "
+                "(e.g. --target firefox gtest)"
+            )
+            args.target.append("gtest")
+
         self.sanity_check(args)
         return args

src/fuzzfetch/core.py

@@ -114,6 +114,7 @@ def __init__(
                     no_opt,
                     fuzzilli,
                     nyx,
+                    searchfox,
                 ) = self._flags
                 if not debug:
                     debug = "-debug" in build or "-dbg" in build
@@ -133,6 +134,8 @@ def __init__(
                     fuzzilli = "-fuzzilli" in build
                 if not nyx:
                     nyx = "-nyx" in build
+                if not searchfox:
+                    searchfox = "-searchfox" in build
 
                 self._flags = BuildFlags(
                     asan,
@@ -144,6 +147,7 @@ def __init__(
                     no_opt,
                     fuzzilli,
                     nyx,
+                    searchfox,
                 )
 
                 # Validate flags
@@ -192,6 +196,11 @@ def __init__(
                         "'build' is not a Nyx build, but nyx=True given "
                         f"(build={build})"
                     )
+                if self._flags.searchfox and "-searchfox" not in build:
+                    raise FetcherException(
+                        "'build' is not a searchfox build, but searchfox=True given "
+                        f"(build={build})"
+                    )
 
             # Attempt to fetch the build.  If it fails and nearest is set, try and find
             # the nearest build that matches
@@ -482,6 +491,12 @@ def resolve_targets(self, targets: Sequence[str]) -> None:
                     if not (self._flags.fuzzing or self._flags.fuzzilli):
                         raise
 
+        if "searchfox" in targets_remaining:
+            targets_remaining.remove("searchfox")
+            resolve_url(self.artifact_url("mozsearch-index.zip"))
+            resolve_url(self.artifact_url("generated-files.tar.gz"))
+            resolve_url(self.artifact_url("mozsearch-distinclude.map"))
+
         for target in targets_remaining:
             try:
                 resolve_url(self.artifact_url(f"{target}.tests.tar.gz"))
@@ -583,6 +598,15 @@ def extract_build(self, path: PathArg) -> None:
                     # we want to maintain support for older builds for now
                     pass
 
+        if "searchfox" in targets_remaining:
+            targets_remaining.remove("searchfox")
+            self.extract_zip(self.artifact_url("mozsearch-index.zip"), path=path)
+            self.extract_tar(self.artifact_url("generated-files.tar.gz"), path=path)
+            download_url(
+                self.artifact_url("mozsearch-distinclude.map"),
+                outfile=path / "mozsearch-distinclude.map",
+            )
+
         # any still remaining targets are assumed to be test artifacts
         for target in targets_remaining:
             try:
@@ -766,6 +790,7 @@ def from_args(
             args.no_opt,
             args.fuzzilli,
             args.nyx,
+            args.searchfox,
         )
         obj = cls(
             args.branch,

src/fuzzfetch/models.py

@@ -35,6 +35,7 @@ class BuildFlags(
             "no_opt",
             "fuzzilli",
             "nyx",
+            "searchfox",
         ),
     )
 ):
@@ -55,6 +56,7 @@ def build_string(self) -> str:
             + ("-nyx" if self.nyx else "")
             + ("-valgrind" if self.valgrind else "")
             + ("-noopt" if self.no_opt else "")
+            + ("-searchfox" if self.searchfox else "")
             + ("-debug" if self.debug else "")
             + ("-opt" if not self.no_opt and not self.debug else "")
         )

tests/mock-firefoxci/api/index/v1/task/gecko.v2.mozilla-central.latest.firefox.linux64-searchfox-debug

@@ -0,0 +1,7 @@
+{
+  "namespace": "gecko.v2.mozilla-central.latest.firefox.linux64-searchfox-debug",
+  "taskId": "fcYf8jKOQ9yV6VtZtbFNJQ",
+  "rank": 0,
+  "data": {},
+  "expires": "2025-01-16T09:31:05.987Z"
+}

tests/mock-firefoxci/api/queue/v1/task/fcYf8jKOQ9yV6VtZtbFNJQ/artifacts/.get

@@ -0,0 +1,148 @@
+{
+  "artifacts": [
+    {
+      "storageType": "s3",
+      "name": "public/build/buildhub.json",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/json"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/config.status",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/octet-stream"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/host/bin/mar",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/octet-stream"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/host/bin/mbsdiff",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/octet-stream"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/mozharness.zip",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/zip"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/profile_build_resources.json",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/json"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.checksums",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/octet-stream"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.generated-files.tar.gz",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/gzip"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target_info.txt",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "text/plain"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.json",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/json"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.langpack.xpi",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/x-xpinstall"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.mozinfo.json",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/json"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.mozsearch-distinclude.map",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/json"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.mozsearch-index.zip",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/zip"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.mozsearch-scip-index.zip",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/zip"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.tar.bz2",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/x-bzip2"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.txt",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "text/plain"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/build/target.xpt_artifacts.zip",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/zip"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/chain-of-trust.json",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "text/plain"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/chain-of-trust.json.sig",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/octet-stream"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/logs/certified.log",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "text/plain"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/logs/live_backing.log",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "text/plain; charset=utf-8"
+    },
+    {
+      "storageType": "reference",
+      "name": "public/logs/live.log",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "text/plain; charset=utf-8"
+    },
+    {
+      "storageType": "s3",
+      "name": "public/logs/localconfig.json",
+      "expires": "2025-01-16T09:31:05.987Z",
+      "contentType": "application/json"
+    }
+  ]
+}

tests/mock-firefoxci/api/queue/v1/task/fcYf8jKOQ9yV6VtZtbFNJQ/artifacts/public/build/target.json

@@ -0,0 +1,17 @@
+{
+  "as": "/builds/worker/fetches/clang/bin/clang --sysroot /builds/worker/fetches/sysroot-x86_64-linux-gnu -std=gnu99",
+  "buildid": "20240117092715",
+  "cc": "/builds/worker/fetches/clang/bin/clang --sysroot /builds/worker/fetches/sysroot-x86_64-linux-gnu -std=gnu99",
+  "cxx": "/builds/worker/fetches/clang/bin/clang++ --sysroot /builds/worker/fetches/sysroot-x86_64-linux-gnu",
+  "host_alias": "x86_64-pc-linux-gnu",
+  "moz_app_id": "{ec8030f7-c20a-464f-9b0e-13a3a9e97384}",
+  "moz_app_maxversion": "123.0a1",
+  "moz_app_name": "firefox",
+  "moz_app_vendor": "Mozilla",
+  "moz_app_version": "123.0a1",
+  "moz_pkg_platform": "linux-x86_64",
+  "moz_source_repo": "https://hg.mozilla.org/mozilla-central",
+  "moz_source_stamp": "71000174812fc0992b6793e53ac5f11f1b87bdc0",
+  "moz_update_channel": "default",
+  "target_alias": "x86_64-pc-linux-gnu"
+}

tests/test_fetch.py

@@ -281,3 +281,16 @@ def test_nyx_builds():
         build_flags_factory(asan=True, fuzzing=True, nyx=True),
         DEFAULT_TARGETS,
     )
+
+
+@pytest.mark.usefixtures("fetcher_mock_resolve_targets", "requests_mock_cache")
+def test_searchfox_data():
+    """
+    Test for retrieving SearchFox source data
+    """
+    Fetcher(
+        "central",
+        "latest",
+        build_flags_factory(searchfox=True, debug=True),
+        ["searchfox"],
+    )