fix: allow packages to be captured from DIST/EGG case (#2239)

* fix: allow packages to be captured from DIST/EGG case Signed-off-by: Christopher Phillips <[email protected]> * test: update expected glob paths Signed-off-by: Christopher Phillips <[email protected]> --------- Signed-off-by: Christopher Phillips <[email protected]>
anchore · Oct 20, 2023 · e1ad340 · e1ad340
1 parent 07f1304
commit e1ad340
Show file tree

Hide file tree

Showing 12 changed files with 355 additions and 1 deletion.
diff --git a/syft/pkg/cataloger/python/cataloger.go b/syft/pkg/cataloger/python/cataloger.go
@@ -29,5 +29,12 @@ func NewPythonIndexCataloger(cfg CatalogerConfig) *generic.Cataloger {
 // NewPythonPackageCataloger returns a new cataloger for python packages within egg or wheel installation directories.
 func NewPythonPackageCataloger() *generic.Cataloger {
 	return generic.NewCataloger("python-package-cataloger").
-		WithParserByGlobs(parseWheelOrEgg, eggInfoGlob, "**/*dist-info/METADATA", "**/*egg-info/PKG-INFO")
+		WithParserByGlobs(
+			parseWheelOrEgg,
+			eggInfoGlob,
+			"**/*dist-info/METADATA",
+			"**/*egg-info/PKG-INFO",
+			"**/*DIST-INFO/METADATA",
+			"**/*EGG-INFO/PKG-INFO",
+		)
 }
diff --git a/syft/pkg/cataloger/python/cataloger_test.go b/syft/pkg/cataloger/python/cataloger_test.go
@@ -69,6 +69,43 @@ func Test_PackageCataloger(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "egg-info directory case sensitive",
+			fixtures: []string{
+				"test-fixtures/casesensitive/EGG-INFO/PKG-INFO",
+				"test-fixtures/casesensitive/EGG-INFO/RECORD",
+				"test-fixtures/casesensitive/EGG-INFO/top_level.txt",
+			},
+			expectedPackage: pkg.Package{
+				Name:     "requests",
+				Version:  "2.22.0",
+				PURL:     "pkg:pypi/[email protected]",
+				Type:     pkg.PythonPkg,
+				Language: pkg.Python,
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicenseFromLocations("Apache 2.0", file.NewLocation("test-fixtures/casesensitive/EGG-INFO/PKG-INFO")),
+				),
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "requests",
+					Version:              "2.22.0",
+					Platform:             "UNKNOWN",
+					Author:               "Kenneth Reitz",
+					AuthorEmail:          "[email protected]",
+					SitePackagesRootPath: "test-fixtures/casesensitive",
+					Files: []pkg.PythonFileRecord{
+						{Path: "requests-2.22.0.dist-info/INSTALLER", Digest: &pkg.PythonFileDigest{"sha256", "zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg"}, Size: "4"},
+						{Path: "requests/__init__.py", Digest: &pkg.PythonFileDigest{"sha256", "PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA"}, Size: "3921"},
+						{Path: "requests/__pycache__/__version__.cpython-38.pyc"},
+						{Path: "requests/__pycache__/utils.cpython-38.pyc"},
+						{Path: "requests/__version__.py", Digest: &pkg.PythonFileDigest{"sha256", "Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc"}, Size: "436"},
+						{Path: "requests/utils.py", Digest: &pkg.PythonFileDigest{"sha256", "LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A"}, Size: "30049"},
+					},
+					TopLevelPackages: []string{"requests"},
+				},
+			},
+		},
 		{
 			name: "dist-info directory",
 			fixtures: []string{
@@ -109,6 +146,46 @@ func Test_PackageCataloger(t *testing.T) {
 				},
 			},
 		},
+		{
+			name: "dist-info directory case sensitive",
+			fixtures: []string{
+				"test-fixtures/casesensitive/DIST-INFO/METADATA",
+				"test-fixtures/casesensitive/DIST-INFO/RECORD",
+				"test-fixtures/casesensitive/DIST-INFO/top_level.txt",
+				"test-fixtures/casesensitive/DIST-INFO/direct_url.json",
+			},
+			expectedPackage: pkg.Package{
+				Name:     "Pygments",
+				Version:  "2.6.1",
+				PURL:     "pkg:pypi/[email protected]?vcs_url=git+https://github.com/python-test/test.git%40aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+				Type:     pkg.PythonPkg,
+				Language: pkg.Python,
+				Licenses: pkg.NewLicenseSet(
+					pkg.NewLicenseFromLocations("BSD License", file.NewLocation("test-fixtures/casesensitive/DIST-INFO/METADATA")),
+				),
+				FoundBy:      "python-package-cataloger",
+				MetadataType: pkg.PythonPackageMetadataType,
+				Metadata: pkg.PythonPackageMetadata{
+					Name:                 "Pygments",
+					Version:              "2.6.1",
+					Platform:             "any",
+					Author:               "Georg Brandl",
+					AuthorEmail:          "[email protected]",
+					SitePackagesRootPath: "test-fixtures/casesensitive",
+					Files: []pkg.PythonFileRecord{
+						{Path: "../../../bin/pygmentize", Digest: &pkg.PythonFileDigest{"sha256", "dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8"}, Size: "220"},
+						{Path: "Pygments-2.6.1.dist-info/AUTHORS", Digest: &pkg.PythonFileDigest{"sha256", "PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY"}, Size: "8449"},
+						{Path: "Pygments-2.6.1.dist-info/RECORD"},
+						{Path: "pygments/__pycache__/__init__.cpython-38.pyc"},
+						{Path: "pygments/util.py", Digest: &pkg.PythonFileDigest{"sha256", "586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA"}, Size: "10778"},
+
+						{Path: "pygments/x_util.py", Digest: &pkg.PythonFileDigest{"sha256", "qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI="}, Size: "10778"},
+					},
+					TopLevelPackages: []string{"pygments", "something_else"},
+					DirectURLOrigin:  &pkg.PythonDirectURLOriginInfo{URL: "https://github.com/python-test/test.git", VCS: "git", CommitID: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},
+				},
+			},
+		},
 		{
 			name: "malformed-record",
 			fixtures: []string{
@@ -278,6 +355,8 @@ func Test_PackageCataloger_Globs(t *testing.T) {
 			name:    "obtain index files",
 			fixture: "test-fixtures/glob-paths",
 			expected: []string{
+				"site-packages/v.DIST-INFO/METADATA",
+				"site-packages/w.EGG-INFO/PKG-INFO",
 				"site-packages/x.dist-info/METADATA",
 				"site-packages/y.egg-info/PKG-INFO",
 				"site-packages/z.egg-info",

diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/METADATA
@@ -0,0 +1,47 @@
+Metadata-Version: 2.1
+Name: Pygments
+Version: 2.6.1
+Summary: Pygments is a syntax highlighting package written in Python.
+Home-page: https://pygments.org/
+Author: Georg Brandl
+Author-email: [email protected]
+License: BSD License
+Keywords: syntax highlighting
+Platform: any
+Classifier: License :: OSI Approved :: BSD License
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: End Users/Desktop
+Classifier: Intended Audience :: System Administrators
+Classifier: Development Status :: 6 - Mature
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: 3.8
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Text Processing :: Filters
+Classifier: Topic :: Utilities
+Requires-Python: >=3.5
+
+
+Pygments
+~~~~~~~~
+
+Pygments is a syntax highlighting package written in Python.
+
+It is a generic syntax highlighter suitable for use in code hosting, forums,
+wikis or other applications that need to prettify source code.  Highlights
+are:
+
+* a wide range of over 500 languages and other text formats is supported
+* special attention is paid to details, increasing quality by a fair amount
+* support for new languages and formats are added easily
+* a number of output formats, presently HTML, LaTeX, RTF, SVG, all image     formats that PIL supports and ANSI sequences
+* it is usable as a command-line tool and as a library
+
+:copyright: Copyright 2006-2019 by the Pygments team, see AUTHORS.
+:license: BSD, see LICENSE for details.
+
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/RECORD
@@ -0,0 +1,6 @@
+../../../bin/pygmentize,sha256=dDhv_U2jiCpmFQwIRHpFRLAHUO4R1jIJPEvT_QYTFp8,220
+Pygments-2.6.1.dist-info/AUTHORS,sha256=PVpa2_Oku6BGuiUvutvuPnWGpzxqFy2I8-NIrqCvqUY,8449
+Pygments-2.6.1.dist-info/RECORD,,
+pygments/__pycache__/__init__.cpython-38.pyc,,
+pygments/util.py,sha256=586xXHiJGGZxqk5PMBu3vBhE68DLuAe5MBARWrSPGxA,10778
+pygments/x_util.py,sha256=qpzzsOW31KT955agi-7NS--90I0iNiJCyLJQnRCHgKI=,10778
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/direct_url.json
@@ -0,0 +1 @@
+{"url": "https://github.com/python-test/test.git", "vcs_info": {"commit_id": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "vcs": "git"}}
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/casesensitive/DIST-INFO/top_level.txt
@@ -0,0 +1,2 @@
+pygments
+something_else
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO
@@ -0,0 +1,134 @@
+Metadata-Version: 2.1
+Name: requests
+Version: 2.22.0
+Summary: Python HTTP for Humans.
+Home-page: http://python-requests.org
+Author: Kenneth Reitz
+Author-email: [email protected]
+License: Apache 2.0
+Description: Requests: HTTP for Humans™
+        ==========================
+
+        [![image](https://img.shields.io/pypi/v/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/l/requests.svg)](https://pypi.org/project/requests/)
+        [![image](https://img.shields.io/pypi/pyversions/requests.svg)](https://pypi.org/project/requests/)
+        [![codecov.io](https://codecov.io/github/requests/requests/coverage.svg?branch=master)](https://codecov.io/github/requests/requests)
+        [![image](https://img.shields.io/github/contributors/requests/requests.svg)](https://github.com/requests/requests/graphs/contributors)
+        [![image](https://img.shields.io/badge/Say%20Thanks-!-1EAEDB.svg)](https://saythanks.io/to/kennethreitz)
+
+        Requests is the only *Non-GMO* HTTP library for Python, safe for human
+        consumption.
+
+        ![image](https://farm5.staticflickr.com/4317/35198386374_1939af3de6_k_d.jpg)
+
+        Behold, the power of Requests:
+
+        ``` {.sourceCode .python}
+        >>> import requests
+        >>> r = requests.get('https://api.github.com/user', auth=('user', 'pass'))
+        >>> r.status_code
+        200
+        >>> r.headers['content-type']
+        'application/json; charset=utf8'
+        >>> r.encoding
+        'utf-8'
+        >>> r.text
+        u'{"type":"User"...'
+        >>> r.json()
+        {u'disk_usage': 368627, u'private_gists': 484, ...}
+        ```
+
+        See [the similar code, sans Requests](https://gist.github.com/973705).
+
+        [![image](https://raw.githubusercontent.com/requests/requests/master/docs/_static/requests-logo-small.png)](http://docs.python-requests.org/)
+
+        Requests allows you to send *organic, grass-fed* HTTP/1.1 requests,
+        without the need for manual labor. There's no need to manually add query
+        strings to your URLs, or to form-encode your POST data. Keep-alive and
+        HTTP connection pooling are 100% automatic, thanks to
+        [urllib3](https://github.com/shazow/urllib3).
+
+        Besides, all the cool kids are doing it. Requests is one of the most
+        downloaded Python packages of all time, pulling in over 11,000,000
+        downloads every month. You don't want to be left out!
+
+        Feature Support
+        ---------------
+
+        Requests is ready for today's web.
+
+        -   International Domains and URLs
+        -   Keep-Alive & Connection Pooling
+        -   Sessions with Cookie Persistence
+        -   Browser-style SSL Verification
+        -   Basic/Digest Authentication
+        -   Elegant Key/Value Cookies
+        -   Automatic Decompression
+        -   Automatic Content Decoding
+        -   Unicode Response Bodies
+        -   Multipart File Uploads
+        -   HTTP(S) Proxy Support
+        -   Connection Timeouts
+        -   Streaming Downloads
+        -   `.netrc` Support
+        -   Chunked Requests
+
+        Requests officially supports Python 2.7 & 3.4–3.7, and runs great on
+        PyPy.
+
+        Installation
+        ------------
+
+        To install Requests, simply use [pipenv](http://pipenv.org/) (or pip, of
+        course):
+
+        ``` {.sourceCode .bash}
+        $ pipenv install requests
+        ✨🍰✨
+        ```
+
+        Satisfaction guaranteed.
+
+        Documentation
+        -------------
+
+        Fantastic documentation is available at
+        <http://docs.python-requests.org/>, for a limited time only.
+
+        How to Contribute
+        -----------------
+
+        1.  Become more familiar with the project by reading our [Contributor's Guide](http://docs.python-requests.org/en/latest/dev/contributing/) and our [development philosophy](http://docs.python-requests.org/en/latest/dev/philosophy/).
+        2.  Check for open issues or open a fresh issue to start a discussion
+            around a feature idea or a bug. There is a [Contributor
+            Friendly](https://github.com/requests/requests/issues?direction=desc&labels=Contributor+Friendly&page=1&sort=updated&state=open)
+            tag for issues that should be ideal for people who are not very
+            familiar with the codebase yet.
+        3.  Fork [the repository](https://github.com/requests/requests) on
+            GitHub to start making your changes to the **master** branch (or
+            branch off of it).
+        4.  Write a test which shows that the bug was fixed or that the feature
+            works as expected.
+        5.  Send a pull request and bug the maintainer until it gets merged and
+            published. :) Make sure to add yourself to
+            [AUTHORS](https://github.com/requests/requests/blob/master/AUTHORS.rst).
+
+
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Natural Language :: English
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Programming Language :: Python :: Implementation :: PyPy
+Requires-Python: >=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*
+Description-Content-Type: text/markdown
+Provides-Extra: security
+Provides-Extra: socks
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/PKG-INFO-INVALID
@@ -0,0 +1,69 @@
+Metadata-Version: 1.1
+Name: mxnet
+Version: 1.8.0
+Summary: <!--- Licensed to the Apache Software Foundation (ASF) under one -->
+<!--- or more contributor license agreements.  See the NOTICE file -->
+<!--- distributed with this work for additional information -->
+<!--- regarding copyright ownership.  The ASF licenses this file -->
+<!--- to you under the Apache License, Version 2.0 (the -->
+<!--- "License"); you may not use this file except in compliance -->
+<!--- with the License.  You may obtain a copy of the License at -->
+
+<!---   http://www.apache.org/licenses/LICENSE-2.0 -->
+
+<!--- Unless required by applicable law or agreed to in writing, -->
+<!--- software distributed under the License is distributed on an -->
+<!--- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -->
+<!--- KIND, either express or implied.  See the License for the -->
+<!--- specific language governing permissions and limitations -->
+<!--- under the License. -->
+
+MXNet Python Package
+====================
+This directory and nested files contain MXNet Python package and language binding.
+
+## Installation
+To install MXNet Python package, visit MXNet [Install Instruction](https://mxnet.apache.org/get_started)
+
+
+## Running the unit tests
+
+For running unit tests, you will need the [nose PyPi package](https://pypi.python.org/pypi/nose). To install:
+```bash
+pip install --upgrade nose
+```
+
+Once ```nose``` is installed, run the following from MXNet root directory (please make sure the installation path of ```nosetests``` is included in your ```$PATH``` environment variable):
+```
+nosetests tests/python/unittest
+nosetests tests/python/train
+
+```
+
+Home-page: https://github.com/apache/incubator-mxnet
+Author: UNKNOWN
+Author-email: UNKNOWN
+License: UNKNOWN
+Description: UNKNOWN
+Platform: UNKNOWN
+Classifier: Development Status :: 5 - Production/Stable
+Classifier: Intended Audience :: Developers
+Classifier: Intended Audience :: Education
+Classifier: Intended Audience :: Science/Research
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Programming Language :: C++
+Classifier: Programming Language :: Cython
+Classifier: Programming Language :: Other
+Classifier: Programming Language :: Perl
+Classifier: Programming Language :: Python
+Classifier: Programming Language :: Python :: 2.7
+Classifier: Programming Language :: Python :: 3.4
+Classifier: Programming Language :: Python :: 3.5
+Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: Implementation :: CPython
+Classifier: Topic :: Scientific/Engineering
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Scientific/Engineering :: Mathematics
+Classifier: Topic :: Software Development
+Classifier: Topic :: Software Development :: Libraries
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/RECORD
@@ -0,0 +1,6 @@
+requests-2.22.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4
+requests/__init__.py,sha256=PnKCgjcTq44LaAMzB-7--B2FdewRrE8F_vjZeaG9NhA,3921
+requests/__pycache__/__version__.cpython-38.pyc,,
+requests/__pycache__/utils.cpython-38.pyc,,
+requests/__version__.py,sha256=Bm-GFstQaFezsFlnmEMrJDe8JNROz9n2XXYtODdvjjc,436
+requests/utils.py,sha256=LtPJ1db6mJff2TJSJWKi7rBpzjPS3mSOrjC9zRhoD3A,30049
diff --git a/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt b/syft/pkg/cataloger/python/test-fixtures/casesensitive/EGG-INFO/top_level.txt
@@ -0,0 +1 @@
+requests
diff --git a/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/v.DIST-INFO/METADATA b/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/v.DIST-INFO/METADATA
@@ -0,0 +1 @@
+bogus
diff --git a/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/w.EGG-INFO/PKG-INFO b/syft/pkg/cataloger/python/test-fixtures/glob-paths/site-packages/w.EGG-INFO/PKG-INFO
@@ -0,0 +1 @@
+bogus