Remove the dependency on Metaflow #82 (#89)

* Remove the dependency on Metaflow #82 Signed-off-by: Thomas Druez <[email protected]> * Unify Pipeline subclasses name #82 Signed-off-by: Thomas Druez <[email protected]> * Move logging to the base Pipeline class #82 Signed-off-by: Thomas Druez <[email protected]> * Add support for exception during Pipeline execution #82 Signed-off-by: Thomas Druez <[email protected]> * Add entry in the CHANGELOG #82 Signed-off-by: Thomas Druez <[email protected]> * Update the profiling method to the new log format #82 Signed-off-by: Thomas Druez <[email protected]> * Replace indent function by built-in Python textwrap.indent #82 Signed-off-by: Thomas Druez <[email protected]>
aboutcode-org · Feb 8, 2021 · b270bec · b270bec
1 parent 12e7579
commit b270bec
Show file tree

Hide file tree

Showing 35 changed files with 409 additions and 686 deletions.
diff --git a/CHANGELOG.rst b/CHANGELOG.rst
@@ -1,6 +1,12 @@
 // Release notes
 // -------------
 
+### v1.1.0 (unreleased)
+
+- Remove the dependency on Metaflow
+  WARNING: The new Pipelines syntax is not backward compatible with v1.0.x
+  https://github.com/nexB/scancode.io/issues/82
+
 ### v1.0.7 (2021-02-01)
 
 - Add user interface to manage Projects from a web browser

diff --git a/Makefile b/Makefile
@@ -45,7 +45,6 @@ conf:
 	@echo "-> Configure the Python venv and install dependencies"
 	${PYTHON_EXE} -m venv .
 	@${ACTIVATE} pip install -r etc/requirements/base.txt
-	@${ACTIVATE} pip install --editable .
 	# Workaround https://github.com/python/typing/issues/573#issuecomment-405986724
 	@${ACTIVATE} pip uninstall --yes typing
 
@@ -68,13 +67,13 @@ check:
 	@echo "-> Run pycodestyle (PEP8) validation"
 	@${ACTIVATE} pycodestyle --max-line-length=88 --exclude=lib,thirdparty,docs,bin,migrations,settings,data,pipelines,var .
 	@echo "-> Run isort imports ordering validation"
-	@${ACTIVATE} isort --recursive --check-only .
+	@${ACTIVATE} isort --check-only .
 	@echo "-> Run black validation"
 	@${ACTIVATE} black --check ${BLACK_ARGS}
 
 isort:
 	@echo "-> Apply isort changes to ensure proper imports ordering"
-	bin/isort --recursive --apply .
+	bin/isort .
 
 black:
 	@echo "-> Apply black code formatter"

diff --git a/docs/scanpipe-pipelines.rst b/docs/scanpipe-pipelines.rst
@@ -10,17 +10,17 @@ Pipeline Base Class
 
 Docker Image Analysis
 ---------------------
-.. autoclass:: scanpipe.pipelines.docker.DockerPipeline()
+.. autoclass:: scanpipe.pipelines.docker.Docker()
     :members:
 
 Load Inventory From Scan
 ------------------------
-.. autoclass:: scanpipe.pipelines.load_inventory.LoadInventoryFromScanCodeScan()
+.. autoclass:: scanpipe.pipelines.load_inventory.LoadInventory()
     :members:
 
 Root Filesystem Analysis
 ------------------------
-.. autoclass:: scanpipe.pipelines.root_filesystems.RootfsPipeline()
+.. autoclass:: scanpipe.pipelines.root_filesystems.RootFS()
     :members:
 
 Scan Codebase

diff --git a/docs/scanpipe-pipes.rst b/docs/scanpipe-pipes.rst
@@ -18,6 +18,11 @@ Codebase
 .. automodule:: scanpipe.pipes.codebase
     :members:
 
+Compliance
+----------
+.. automodule:: scanpipe.pipes.compliance
+    :members:
+
 Debian
 ------
 .. automodule:: scanpipe.pipes.debian
@@ -44,8 +49,3 @@ ScanCode
 --------
 .. automodule:: scanpipe.pipes.scancode
     :members:
-
-Utilities
----------
-.. automodule:: scanpipe.pipes.utilities
-    :members:
diff --git a/etc/requirements/base.txt b/etc/requirements/base.txt
@@ -26,9 +26,6 @@ kombu==4.6.11
 # WSGI server
 gunicorn==20.0.4
 
-# Metaflow
-metaflow==2.2.6
-
 # Docker
 container_inspector==3.1.2
 

diff --git a/etc/requirements/dev.txt b/etc/requirements/dev.txt
@@ -3,6 +3,7 @@
 # Code validation
 pycodestyle==2.6.0
 black==20.8b1
+isort==5.7.0
 
 # Documentation
 Sphinx==3.4.3

diff --git a/scancodeio/settings/base.py b/scancodeio/settings/base.py
@@ -129,6 +129,9 @@
     },
 ]
 
+# True if running tests through `./manage test`
+IS_TESTS = "test" in sys.argv
+
 # Logging
 
 LOGGING = {
@@ -144,15 +147,16 @@
     },
     "loggers": {
         "scanner.tasks": {
-            "handlers": ["console"],
+            "handlers": ["null"] if IS_TESTS else ["console"],
+            "level": env.str("DJANGO_LOG_LEVEL", "INFO"),
+        },
+        "scanpipe.pipelines": {
+            "handlers": ["null"] if IS_TESTS else ["console"],
             "level": env.str("DJANGO_LOG_LEVEL", "INFO"),
         },
     },
 }
 
-# True if running tests through `./manage test`
-IS_TESTS = "test" in sys.argv
-
 # Instead of sending out real emails the console backend just writes the emails
 # that would be sent to the standard output.
 EMAIL_BACKEND = "django.core.mail.backends.console.EmailBackend"

diff --git a/scanpipe/api/serializers.py b/scanpipe/api/serializers.py
@@ -56,7 +56,6 @@ class RunSerializer(SerializerExcludeFieldsMixin, serializers.ModelSerializer):
     project = serializers.HyperlinkedRelatedField(
         view_name="project-detail", read_only=True
     )
-    task_output = serializers.SerializerMethodField()
 
     class Meta:
         model = Run
@@ -66,7 +65,6 @@ class Meta:
             "description",
             "project",
             "uuid",
-            "run_id",
             "created_date",
             "task_id",
             "task_start_date",
@@ -77,9 +75,6 @@ class Meta:
             "execution_time",
         ]
 
-    def get_task_output(self, run):
-        return run.task_output.split("\n")[1:]
-
 
 class ProjectSerializer(ExcludeFromListViewMixin, serializers.ModelSerializer):
     pipeline = serializers.ChoiceField(

diff --git a/scanpipe/api/views.py b/scanpipe/api/views.py
@@ -40,7 +40,8 @@
 from scanpipe.models import Project
 from scanpipe.models import ProjectError
 from scanpipe.models import Run
-from scanpipe.pipelines import get_pipeline_description
+from scanpipe.pipelines import get_pipeline_class
+from scanpipe.pipelines import get_pipeline_graph
 from scanpipe.views import project_results_json_response
 
 scanpipe_app_config = apps.get_app_config("scanpipe")
@@ -92,7 +93,8 @@ def pipelines(self, request, *args, **kwargs):
         for location, name in scanpipe_app_config.pipelines:
             data[name] = {
                 "location": location,
-                "description": get_pipeline_description(location).split("\n"),
+                "description": get_pipeline_class(location).get_doc(),
+                "steps": get_pipeline_graph(location),
             }
         return Response(data)
 
@@ -188,18 +190,3 @@ def start_pipeline(self, request, *args, **kwargs):
         transaction.on_commit(run.run_pipeline_task_async)
 
         return Response({"status": f"Pipeline {run.pipeline} started."})
-
-    @action(detail=True, methods=["get"])
-    def resume_pipeline(self, request, *args, **kwargs):
-        run = self.get_object()
-
-        if run.task_succeeded:
-            message = {"status": "Cannot resume a successful pipeline run."}
-            return Response(message, status=status.HTTP_400_BAD_REQUEST)
-        elif not run.task_start_date:
-            message = {"status": "Cannot resume never started pipeline run."}
-            return Response(message, status=status.HTTP_400_BAD_REQUEST)
-
-        transaction.on_commit(run.resume_pipeline_task_async)
-
-        return Response({"status": f"Pipeline {run.pipeline} resumed."})
diff --git a/scanpipe/apps.py b/scanpipe/apps.py
@@ -25,9 +25,6 @@
 from django.apps import AppConfig
 from django.utils.translation import gettext_lazy as _
 
-from scanpipe.logging import RunLogger
-from scanpipe.logging import extra_logging
-
 dot_py_suffix = ".py"
 
 
@@ -58,12 +55,6 @@ def ready(self):
                 name = remove_dot_py_suffix(child.name)
                 self.pipelines.append((location, name))
 
-        # Decorates the default metaflow logger to capture log messages
-        # Warning: This import cannot be moved outside this method
-        from metaflow import cli
-
-        cli.logger = extra_logging(cli.logger, RunLogger())
-
     def is_valid(self, pipeline):
         """
         Return True if the pipeline is valid and available.

diff --git a/scanpipe/forms.py b/scanpipe/forms.py
@@ -21,7 +21,6 @@
 # Visit https://github.com/nexB/scancode.io for support and download.
 
 from django import forms
-from django.db import transaction
 
 import django_filters
 

diff --git a/scanpipe/logging.py b/scanpipe/logging.py
diff --git a/scanpipe/management/commands/delete-project.py b/scanpipe/management/commands/delete-project.py
@@ -20,7 +20,6 @@
 # ScanCode.io is a free software code scanning tool from nexB Inc. and others.
 # Visit https://github.com/nexB/scancode.io for support and download.
 
-import shutil
 import sys
 
 from scanpipe.management.commands import ProjectCommand