Resolve dependencies and improve JS support

CHANGELOG.rst

@@ -34,6 +34,26 @@ v33.0.0 (next next, roadmap)
   of these in other summary plugins.
   See https://github.com/nexB/scancode-toolkit/issues/1745
 
+v32.2.0 - (next, roadmap)
+-------------------------
+
+- New and improved package/dependency data:
+  - Added new attribute in DependentPackage `is_direct` to aid
+    package resolution and dependency graph creation.
+  - Added new attributes in PackageData: `is_private` and
+    `is_virtual`. #3102 #3811
+  https://github.com/nexB/scancode-toolkit/pull/3779
+
+- Improved javascript package detection:
+  - Add support for pnpm manifests and lockfiles #3766
+  - Add support for npm, pnpm and yarn workspaces #3746
+  - Improve resolved package and dependencies support in lockfiles for
+    yarn.lock, package-lock.json, and pnpm. #3780
+  - Add support for private packages. #3120
+  - Add support for new dependency scopes across javascript
+  - Lots of misc bugfixes in yarn and npm parsers.
+  https://github.com/nexB/scancode-toolkit/pull/3779
+
 - Improve cargo package detection support with various improvements
   and bugfixes:
   - Fix for parser crashing on cargo workspaces
@@ -43,14 +63,10 @@ v33.0.0 (next next, roadmap)
   - Better handle workspace data thorugh extra_data attribute
   See https://github.com/nexB/scancode-toolkit/pull/3783
 
-- We now support parsing the Swift manifest JSON dump and the ``Package.resolved`` file https://github.com/nexB/scancode-toolkit/issues/2657.
-  - Run the commands below on your local Swift project before running the scan.
-    - ::
-
-        swift package dump-package > Package.swift.json
-    - ::
-
-        swift package resolve
+- We now support parsing the Swift manifest JSON dump and the
+  ``Package.resolved`` file https://github.com/nexB/scancode-toolkit/issues/2657.
+  Run the command below on your local Swift project before running the scan:
+    `swift package dump-package > Package.swift.json && swift package resolve``
 
 - New and updated licenses, including support for newly released
   SPDX license list versions:

docs/source/reference/available_package_parsers.rst

@@ -539,6 +539,24 @@ parsers in scancode-toolkit during documentation builds.
      - ``npm_shrinkwrap_json``
      - JavaScript
      - https://docs.npmjs.com/cli/v8/configuring-npm/npm-shrinkwrap-json
+   * - pnpm pnpm-lock.yaml lockfile
+     - ``*/pnpm-lock.yaml``
+     - ``npm``
+     - ``pnpm_lock_yaml``
+     - JavaScript
+     - https://github.com/pnpm/spec/blob/master/lockfile/6.0.md
+   * - pnpm shrinkwrap.yaml lockfile
+     - ``*/shrinkwrap.yaml``
+     - ``npm``
+     - ``pnpm_shrinkwrap_yaml``
+     - JavaScript
+     - https://github.com/pnpm/spec/blob/master/lockfile/4.md
+   * - pnpm workspace yaml file
+     - ``*/pnpm-workspace.yaml``
+     - ``npm``
+     - ``pnpm_workspace_yaml``
+     - JavaScript
+     - https://pnpm.io/pnpm-workspace_yaml
    * - yarn.lock lockfile v1 format
      - ``*/yarn.lock``
      - ``npm``
@@ -748,7 +766,7 @@ parsers in scancode-toolkit during documentation builds.
      - ``squashfs_disk_image``
      - None
      - https://en.wikipedia.org/wiki/SquashFS
-   * - JSON dump of Package.swift created with ``swift package dump-package > Package.swift.json``
+   * - JSON dump of Package.swift created with ``swift package dump-package > Package.swift.json``
      - ``*/Package.swift.json``
      - ``swift``
      - ``swift_package_manifest_json``

src/packagedcode/__init__.py

@@ -150,6 +150,9 @@
     npm.NpmShrinkwrapJsonHandler,
     npm.YarnLockV1Handler,
     npm.YarnLockV2Handler,
+    npm.PnpmShrinkwrapYamlHandler,
+    npm.PnpmLockYamlHandler,
+    npm.PnpmWorkspaceYamlHandler,
 
     nuget.NugetNupkgHandler,
     nuget.NugetNuspecHandler,

src/packagedcode/bower.py

@@ -29,8 +29,10 @@ def parse(cls, location, package_only=False):
         with io.open(location, encoding='utf-8') as loc:
             package_data = json.load(loc)
 
-        # note: having no name is not a problem for private packages. See #1514
         name = package_data.get('name')
+        is_private = False
+        if not name:
+            is_private = True
 
         description = package_data.get('description')
         version = package_data.get('version')
@@ -99,5 +101,6 @@ def parse(cls, location, package_only=False):
             homepage_url=homepage_url,
             vcs_url=vcs_url,
             dependencies=dependencies,
+            is_private=is_private,
         )
         yield models.PackageData.from_data(package_data, package_only)

src/packagedcode/models.py

@@ -371,6 +371,15 @@ class DependentPackage(ModelMixin):
              'been resolved and this dependency url points to an '
              'exact version.')
 
+    is_direct = Boolean(
+        default=True,
+        label='is direct flag',
+        help='True if this is a direct, first-level dependency, '
+             'defined in the manifest of a package. False if this '
+             'is an indirect, transitive dependency resolved from '
+             'first level dependencies.'
+    )
+
     resolved_package = Mapping(
         label='resolved package data',
         help='A mapping of resolved package data for this dependent package, '
@@ -682,6 +691,24 @@ class PackageData(IdentifiablePackageData):
              'package type or datafile format.'
     )
 
+    is_private = Boolean(
+        default=False,
+        label='is private flag',
+        help='True if this is a private package, either not meant to be '
+             'published on a repository, and/or a local package without a '
+             'name and version used primarily to track dependencies and '
+             'other information, and build this package, for instance with '
+             'JavaScript and PHP applications.'
+    )
+
+    is_virtual = Boolean(
+        default=False,
+        label='is virtual flag',
+        help='True if this package is created only from a manifest or lockfile, '
+             'and not from its actual packaged code. The files of this package '
+             'are not present in the codebase.'
+    )
+
     extra_data = Mapping(
         label='extra data',
         help='A mapping of arbitrary extra package data.',
@@ -1026,6 +1053,9 @@ class DatafileHandler:
     # Informational: Default primary language for this parser.
     default_primary_language = None
 
+    # If the datafilehandler contains only resolved dependencies
+    is_lockfile = False
+
     # Informational: Description of this parser
     description = None