Merge branch 'main' into 1426-optional-step

Signed-off-by: Ayan Sinha Mahapatra <[email protected]>

CHANGELOG.rst

@@ -22,6 +22,12 @@ v34.9.0 (unreleased)
   The steps decorated as optional are not included by default anymore.
   https://github.com/aboutcode-org/scancode.io/issues/386
 
+- Add a new ``PublishToFederatedCode`` pipeline (addon) to push scan result
+  to FederatedCode.
+  https://github.com/nexB/scancode.io/pull/1400
+
+- Add new ``purl`` field to project model. https://github.com/nexB/scancode.io/pull/1400
+
 v34.8.3 (2024-10-30)
 --------------------
 

docs/application-settings.rst

@@ -338,6 +338,28 @@ API key using ``MATCHCODEIO_API_KEY``::
 
     MATCHCODEIO_API_KEY=insert_your_api_key_here
 
+.. _scancodeio_settings_federatedcode:
+
+FEDERATEDCODE
+^^^^^^^^^^^^^
+
+FederatedCode is decentralized and federated metadata for software applications
+stored in Git repositories.
+
+
+To configure your local environment, set the following in your ``.env`` file::
+
+    FEDERATEDCODE_GIT_ACCOUNT_URL=https://<Address to your git account>/
+
+    FEDERATEDCODE_GIT_SERVICE_TOKEN=insert_your_git_api_key_here
+
+Also provide the name and email that will be used to sign off on commits to Git repositories::
+
+    FEDERATEDCODE_GIT_SERVICE_NAME=insert_name_here
+
+    FEDERATEDCODE_GIT_SERVICE_EMAIL=insert_email_here
+
+
 .. _scancodeio_settings_fetch_authentication:
 
 Fetch Authentication

docs/built-in-pipelines.rst

@@ -188,6 +188,20 @@ Populate PurlDB (addon)
     :members:
     :member-order: bysource
 
+.. _pipeline_publish_to_federatedcode:
+
+Publish To FederatedCode (addon)
+--------------------------------
+
+.. warning::
+    This pipeline requires access to a FederatedCode service.
+    Refer to :ref:`scancodeio_settings_federatedcode` to configure access to
+    FederatedCode in your ScanCode.io instance.
+
+.. autoclass:: scanpipe.pipelines.publish_to_federatedcode.PublishToFederatedCode()
+    :members:
+    :member-order: bysource
+
 .. _pipeline_scan_codebase:
 
 Scan Codebase

docs/faq.rst

@@ -16,7 +16,7 @@ and run the appropriate pipeline.
 ScanCode.io offers several :ref:`built_in_pipelines` depending on your input, see
 the :ref:`faq_which_pipeline` below.
 
-As an alternative, I you simply which to run a pipeline without installing ScanCode.io
+As an alternative, If you simply which to run a pipeline without installing ScanCode.io
 you may use the Docker image to run pipelines as a single command:
 
 .. code-block:: bash
@@ -41,7 +41,7 @@ Here are some general guidelines based on different input scenarios:
 - If you have a **Docker image** as input, use the
   :ref:`analyze_docker_image <pipeline_analyze_docker_image>` pipeline.
 - For a full **codebase compressed as an archive**, optionally also with
-  it's **pre-resolved dependenices**, and want to detect all the packages
+  it's **pre-resolved dependencies**, and want to detect all the packages
   present linked with their respective files, use the
   :ref:`scan_codebase <pipeline_scan_codebase>` pipeline.
 - If you have a **single package archive**, and you want to get information
@@ -58,7 +58,7 @@ Here are some general guidelines based on different input scenarios:
   resolve packages from their package requirements, use the
   :ref:`resolve_dependencies <pipeline_resolve_dependencies>` pipeline.
 - When you have application **package archives/codebases** and optionally also
-  their **pre-resolved dependenices** and you want to **inspect packages**
+  their **pre-resolved dependencies** and you want to **inspect packages**
   present in the package manifests and dependency, use the
   :ref:`inspect_packages <pipeline_inspect_packages>` pipeline.
 - For scenarios involving both a **development and deployment codebase**, consider using
@@ -102,8 +102,8 @@ data:
 .. image:: images/license-clarity-scan-summary.png
 
 In contrast, the :ref:`scan_codebase <pipeline_scan_codebase>` pipeline is more of a
-general purpose pipeline and make no such single package assumption.
-It does not not compute such summary.
+general purpose pipeline and makes no such single package assumption.
+It does not compute such summary.
 
 You can also have a look at the different steps for each pipeline from the
 :ref:`built_in_pipelines` documentation.

scancodeio/settings.py

@@ -418,3 +418,10 @@
 MATCHCODEIO_USER = env.str("MATCHCODEIO_USER", default="")
 MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="")
 MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="")
+
+# FederatedCode integration
+
+FEDERATEDCODE_GIT_ACCOUNT_URL = env.str("FEDERATEDCODE_GIT_ACCOUNT_URL", default="")
+FEDERATEDCODE_GIT_SERVICE_TOKEN = env.str("FEDERATEDCODE_GIT_SERVICE_TOKEN", default="")
+FEDERATEDCODE_GIT_SERVICE_NAME = env.str("FEDERATEDCODE_GIT_SERVICE_NAME", default="")
+FEDERATEDCODE_GIT_SERVICE_EMAIL = env.str("FEDERATEDCODE_GIT_SERVICE_EMAIL", default="")

scanpipe/api/serializers.py

@@ -209,6 +209,7 @@ class Meta:
             "name",
             "url",
             "uuid",
+            "purl",
             "upload_file",
             "upload_file_tag",
             "input_urls",

scanpipe/forms.py

@@ -25,6 +25,7 @@
 from django.core.exceptions import ObjectDoesNotExist
 from django.core.exceptions import ValidationError
 
+from packageurl import PackageURL
 from taggit.forms import TagField
 from taggit.forms import TagWidget
 
@@ -480,12 +481,31 @@ class Meta:
         fields = [
             "name",
             "notes",
+            "purl",
         ]
         widgets = {
             "name": forms.TextInput(attrs={"class": "input"}),
             "notes": forms.Textarea(attrs={"rows": 3, "class": "textarea is-dynamic"}),
+            "purl": forms.TextInput(
+                attrs={
+                    "class": "input",
+                    "placeholder": "pkg:npm/[email protected]",
+                }
+            ),
         }
 
+    def clean_purl(self):
+        """Validate the Project PURL."""
+        purl = self.cleaned_data.get("purl")
+
+        if purl:
+            try:
+                PackageURL.from_string(purl)
+            except ValueError:
+                raise forms.ValidationError("PURL must be a valid PackageURL")
+
+        return purl
+
     def __init__(self, *args, **kwargs):
         """Load initial values from Project ``settings`` field."""
         super().__init__(*args, **kwargs)

scanpipe/migrations/0069_project_purl.py

@@ -0,0 +1,18 @@
+# Generated by Django 5.1.3 on 2024-11-08 12:47
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0068_rename_discovered_dependencies_attribute'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='project',
+            name='purl',
+            field=models.CharField(blank=True, help_text='Package URL for the project, used for pushing project scan result to FederatedCode. This should be the PURL of the input.', max_length=2048),
+        ),
+    ]

scanpipe/models.py

@@ -562,6 +562,16 @@ class Project(UUIDPKModel, ExtraDataFieldMixin, UpdateMixin, models.Model):
     notes = models.TextField(blank=True)
     settings = models.JSONField(default=dict, blank=True)
     labels = TaggableManager(through=UUIDTaggedItem)
+    purl = models.CharField(
+        max_length=2048,
+        blank=True,
+        help_text=_(
+            "Package URL (PURL) for the project, required for pushing the project's "
+            "scan result to FederatedCode. For example, if the input is an input URL "
+            "like https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz, the "
+            "corresponding PURL would be pkg:npm/[email protected]."
+        ),
+    )
 
     objects = ProjectQuerySet.as_manager()
 
@@ -705,6 +715,7 @@ def clone(
         """Clone this project using the provided ``clone_name`` as new project name."""
         new_project = Project.objects.create(
             name=clone_name,
+            purl=self.purl,
             settings=self.settings if copy_settings else {},
         )
 

scanpipe/pipelines/publish_to_federatedcode.py

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import federatedcode
+
+
+class PublishToFederatedCode(Pipeline):
+    """
+    Publish package scan to FederatedCode.
+
+    This pipeline commits the project scan result in FederatedCode Git repository.
+    It uses ``Project PURL`` to determine the Git repository and the
+    exact directory path where the scan should be stored.
+    """
+
+    download_inputs = False
+    is_addon = True
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.check_federatedcode_eligibility,
+            cls.get_package_repository,
+            cls.clone_repository,
+            cls.add_scan_result,
+            cls.commit_and_push_changes,
+            cls.delete_local_clone,
+        )
+
+    def check_federatedcode_eligibility(self):
+        """
+        Check if the project fulfills the following criteria for
+        pushing the project result to FederatedCode.
+        """
+        federatedcode.check_federatedcode_eligibility(project=self.project)
+
+    def get_package_repository(self):
+        """Get the Git repository URL and scan path for a given package."""
+        self.package_git_repo, self.package_scan_file = (
+            federatedcode.get_package_repository(
+                project_purl=self.project.purl, logger=self.log
+            )
+        )
+
+    def clone_repository(self):
+        """Clone repository to local_path."""
+        self.repo = federatedcode.clone_repository(
+            repo_url=self.package_git_repo,
+            logger=self.log,
+        )
+
+    def add_scan_result(self):
+        """Add package scan result to the local Git repository."""
+        self.relative_file_path = federatedcode.add_scan_result(
+            project=self.project,
+            repo=self.repo,
+            package_scan_file=self.package_scan_file,
+            logger=self.log,
+        )
+
+    def commit_and_push_changes(self):
+        """Commit and push changes to remote repository."""
+        federatedcode.commit_and_push_changes(
+            repo=self.repo,
+            file_to_commit=str(self.relative_file_path),
+            purl=self.project.purl,
+            logger=self.log,
+        )
+        self.log(
+            f"Scan result for '{self.project.purl}' "
+            f"pushed to '{self.package_git_repo}'"
+        )
+
+    def delete_local_clone(self):
+        """Remove local clone."""
+        federatedcode.delete_local_clone(repo=self.repo)