Disable scorecard vulnerabilities check (#534)

This check is a bit limited at the moment, so we may end up displaying misleading results (100% of the projects are actually passing this check as a consequence of those limitations). Signed-off-by: Sergio Castaño Arteaga <[email protected]>
cncf · Aug 25, 2022 · 04071b0 · 04071b0
1 parent e758b8c
commit 04071b0
Show file tree

Hide file tree

Showing 18 changed files with 19 additions and 155 deletions.
diff --git a/clomonitor-core/src/config.rs b/clomonitor-core/src/config.rs
@@ -33,7 +33,6 @@ pub const SLACK_PRESENCE: &str = "slack_presence";
 pub const SIGNED_RELEASES: &str = "signed_releases";
 pub const TRADEMARK_DISCLAIMER: &str = "trademark_disclaimer";
 pub const TOKEN_PERMISSIONS: &str = "token_permissions";
-pub const VULNERABILITIES: &str = "vulnerabilities";
 pub const WEBSITE: &str = "website";
 
 // Checks weights
@@ -70,15 +69,14 @@ lazy_static! {
 
         // Security
         m.insert(BINARY_ARTIFACTS, 2);
-        m.insert(CODE_REVIEW, 2);
+        m.insert(CODE_REVIEW, 3);
         m.insert(DANGEROUS_WORKFLOW, 2);
         m.insert(DEPENDENCY_UPDATE_TOOL, 2);
-        m.insert(MAINTAINED, 2);
+        m.insert(MAINTAINED, 3);
         m.insert(SBOM, 1);
-        m.insert(SECURITY_POLICY, 2);
+        m.insert(SECURITY_POLICY, 3);
         m.insert(SIGNED_RELEASES, 2);
         m.insert(TOKEN_PERMISSIONS, 2);
-        m.insert(VULNERABILITIES, 2);
 
         // Legal
         m.insert(TRADEMARK_DISCLAIMER, 5);
@@ -117,7 +115,6 @@ lazy_static! {
                 SECURITY_POLICY,
                 SIGNED_RELEASES,
                 TOKEN_PERMISSIONS,
-                VULNERABILITIES,
             ],
         );
 
@@ -175,7 +172,6 @@ lazy_static! {
         m.insert(MAINTAINED, "Maintained");
         m.insert(SIGNED_RELEASES, "Signed-Releases");
         m.insert(TOKEN_PERMISSIONS, "Token-Permissions");
-        m.insert(VULNERABILITIES, "Vulnerabilities");
 
         m
     };

diff --git a/clomonitor-core/src/linter/check/mod.rs b/clomonitor-core/src/linter/check/mod.rs
@@ -603,14 +603,6 @@ pub(crate) async fn trademark_disclaimer(input: &CheckInput<'_>) -> Result<Check
     Ok(false.into())
 }
 
-/// Vulnerabilities check (from OpenSSF Scorecard).
-pub(crate) fn vulnerabilities(input: &CheckInput) -> Result<CheckOutput> {
-    Ok(match input.scorecard.get_check(VULNERABILITIES) {
-        Some(sc_check) => sc_check.into(),
-        None => false.into(),
-    })
-}
-
 /// Website check.
 pub(crate) fn website(input: &CheckInput) -> Result<CheckOutput> {
     // Website in Github

diff --git a/clomonitor-core/src/linter/check/scorecard.rs b/clomonitor-core/src/linter/check/scorecard.rs
@@ -39,7 +39,7 @@ pub(crate) async fn scorecard(repo_url: &str, github_token: &str) -> Result<Scor
         .arg(format!("--repo={repo_url}"))
         .arg("--format=json")
         .arg("--show-details")
-        .arg("--checks=Binary-Artifacts,Code-Review,Dangerous-Workflow,Dependency-Update-Tool,Maintained,Signed-Releases,Token-Permissions,Vulnerabilities")
+        .arg("--checks=Binary-Artifacts,Code-Review,Dangerous-Workflow,Dependency-Update-Tool,Maintained,Signed-Releases,Token-Permissions")
         .output()
         .await?;
     if !output.status.success() {

diff --git a/clomonitor-core/src/linter/mod.rs b/clomonitor-core/src/linter/mod.rs
@@ -132,7 +132,6 @@ pub struct Security {
     pub security_policy: Option<CheckOutput>,
     pub signed_releases: Option<CheckOutput>,
     pub token_permissions: Option<CheckOutput>,
-    pub vulnerabilities: Option<CheckOutput>,
 }
 
 /// Legal section of the report.
@@ -231,7 +230,6 @@ pub async fn lint(opts: &LintOptions, svc: &LintServices) -> Result<Report> {
             security_policy: run_check(SECURITY_POLICY, security_policy, &input),
             signed_releases: run_check(SIGNED_RELEASES, signed_releases, &input),
             token_permissions: run_check(TOKEN_PERMISSIONS, token_permissions, &input),
-            vulnerabilities: run_check(VULNERABILITIES, vulnerabilities, &input),
         },
         legal: Legal {
             trademark_disclaimer,

diff --git a/clomonitor-core/src/score/mod.rs b/clomonitor-core/src/score/mod.rs
@@ -110,7 +110,6 @@ pub fn calculate(report: &Report) -> Score {
         (SECURITY_POLICY, should_score(&s.security_policy)),
         (SIGNED_RELEASES, should_score(&s.signed_releases)),
         (TOKEN_PERMISSIONS, should_score(&s.token_permissions)),
-        (VULNERABILITIES, should_score(&s.vulnerabilities)),
     ]);
 
     // Legal
@@ -350,23 +349,22 @@ mod tests {
                     security_policy: Some(true.into()),
                     signed_releases: Some(true.into()),
                     token_permissions: Some(true.into()),
-                    vulnerabilities: Some(true.into()),
                 },
                 legal: Legal {
                     trademark_disclaimer: Some(true.into()),
                 },
             }),
             Score {
                 global: 99.99999999999999,
-                global_weight: 94,
+                global_weight: 95,
                 documentation: Some(100.0),
                 documentation_weight: Some(30),
                 license: Some(100.0),
                 license_weight: Some(20),
                 best_practices: Some(100.0),
                 best_practices_weight: Some(20),
                 security: Some(100.0),
-                security_weight: Some(19),
+                security_weight: Some(20),
                 legal: Some(100.0),
                 legal_weight: Some(5),
             }
@@ -417,23 +415,22 @@ mod tests {
                     security_policy: Some(false.into()),
                     signed_releases: Some(false.into()),
                     token_permissions: Some(false.into()),
-                    vulnerabilities: Some(false.into()),
                 },
                 legal: Legal {
                     trademark_disclaimer: Some(false.into()),
                 },
             }),
             Score {
                 global: 0.0,
-                global_weight: 94,
+                global_weight: 95,
                 documentation: Some(0.0),
                 documentation_weight: Some(30),
                 license: Some(0.0),
                 license_weight: Some(20),
                 best_practices: Some(0.0),
                 best_practices_weight: Some(20),
                 security: Some(0.0),
-                security_weight: Some(19),
+                security_weight: Some(20),
                 legal: Some(0.0),
                 legal_weight: Some(5),
             }
@@ -490,23 +487,22 @@ mod tests {
                     security_policy: Some(true.into()),
                     signed_releases: Some(true.into()),
                     token_permissions: Some(true.into()),
-                    vulnerabilities: Some(true.into()),
                 },
                 legal: Legal {
                     trademark_disclaimer: None,
                 },
             }),
             Score {
-                global: 100.0,
-                global_weight: 74,
+                global: 100.00000000000001,
+                global_weight: 75,
                 documentation: Some(100.0),
                 documentation_weight: Some(18),
                 license: Some(100.0),
                 license_weight: Some(20),
                 best_practices: Some(100.0),
                 best_practices_weight: Some(17),
                 security: Some(100.0),
-                security_weight: Some(19),
+                security_weight: Some(20),
                 legal: None,
                 legal_weight: None,
             }

diff --git a/clomonitor-linter/src/table.rs b/clomonitor-linter/src/table.rs
@@ -207,10 +207,6 @@ pub(crate) fn display(
             cell_entry("Security / Token permissions"),
             cell_check(&report.security.token_permissions),
         ])
-        .add_row(vec![
-            cell_entry("Security / Vulnerabilities"),
-            cell_check(&report.security.vulnerabilities),
-        ])
         .add_row(vec![
             cell_entry("Legal / Trademark disclaimer"),
             cell_check(&report.legal.trademark_disclaimer),
@@ -359,7 +355,6 @@ mod tests {
                 security_policy: Some(true.into()),
                 signed_releases: Some(true.into()),
                 token_permissions: Some(true.into()),
-                vulnerabilities: Some(true.into()),
             },
             legal: Legal {
                 trademark_disclaimer: Some(true.into()),

diff --git a/clomonitor-linter/src/testdata/display.golden b/clomonitor-linter/src/testdata/display.golden
@@ -94,8 +94,6 @@ Checks summary
 ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
 │ Security / Token permissions         ┆      ✓     │
 ├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
-│ Security / Vulnerabilities           ┆      ✓     │
-├╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌┼╌╌╌╌╌╌╌╌╌╌╌╌┤
 │ Legal / Trademark disclaimer         ┆      ✓     │
 ╰──────────────────────────────────────┴────────────╯
 

diff --git a/database/migrations/functions/repositories/get_repositories_with_checks.sql b/database/migrations/functions/repositories/get_repositories_with_checks.sql
@@ -40,15 +40,14 @@ returns setof text as $$
             (rp.data->'security'->'security_policy'->'passed')::boolean as security_policy,
             (rp.data->'security'->'signed_releases'->'passed')::boolean as signed_releases,
             (rp.data->'security'->'token_permissions'->'passed')::boolean as token_permissions,
-            (rp.data->'security'->'vulnerabilities'->'passed')::boolean as vulnerabilities,
             (rp.data->'legal'->'trademark_disclaimer'->'passed')::boolean as trademark_disclaimer
         from organization o
         join project p using (organization_id)
         join repository r using (project_id)
         join report rp using (repository_id)
         order by o.foundation asc, p.name asc
     )
-    select 'Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Website,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF Badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependency Update Tool,Maintained,SBOM,Security Policy,Signed Releases,Token Permissions,Vulnerabilities,Trademark Disclaimer'
+    select 'Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Website,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF Badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependency Update Tool,Maintained,SBOM,Security Policy,Signed Releases,Token Permissions,Trademark Disclaimer'
     union all
     select rtrim(ltrim(r.*::text, '('), ')') from repositories r;
 $$ language sql;
diff --git a/database/migrations/functions/stats/get_stats.sql b/database/migrations/functions/stats/get_stats.sql
@@ -165,8 +165,7 @@ returns json as $$
                     'sbom', repositories_passing_check(p_foundation, 'security', 'sbom'),
                     'security_policy', repositories_passing_check(p_foundation, 'security', 'security_policy'),
                     'signed_releases', repositories_passing_check(p_foundation, 'security', 'signed_releases'),
-                    'token_permissions', repositories_passing_check(p_foundation, 'security', 'token_permissions'),
-                    'vulnerabilities', repositories_passing_check(p_foundation, 'security', 'vulnerabilities')
+                    'token_permissions', repositories_passing_check(p_foundation, 'security', 'token_permissions')
                 ),
                 'legal', json_build_object(
                     'trademark_disclaimer', repositories_passing_check(p_foundation, 'legal', 'trademark_disclaimer')

diff --git a/database/tests/functions/projects/get_project_checks.sql b/database/tests/functions/projects/get_project_checks.sql
@@ -100,9 +100,6 @@ insert into report (
             "signed_releases": {
                 "passed": false
             },
-            "vulnerabilities": {
-                "passed": true
-            },
             "binary_artifacts": {
                 "passed": true
             },
@@ -233,7 +230,6 @@ select results_eq(
             ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'code_review', true),
             ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'security_policy', true),
             ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'signed_releases', false),
-            ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'vulnerabilities', true),
             ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'binary_artifacts', true),
             ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'token_permissions', false),
             ('00000000-0000-0001-0000-000000000000'::uuid, 'security', 'dangerous_workflow', true),

diff --git a/database/tests/functions/projects/get_project_passed_checks.sql b/database/tests/functions/projects/get_project_passed_checks.sql
@@ -100,9 +100,6 @@ insert into report (
             "signed_releases": {
                 "passed": false
             },
-            "vulnerabilities": {
-                "passed": true
-            },
             "binary_artifacts": {
                 "passed": true
             },
@@ -243,7 +240,6 @@ select results_eq(
             openssf_badge,
             recent_release,
             security_policy,
-            vulnerabilities,
             website
         }'::text[])
     $$,

diff --git a/database/tests/functions/repositories/get_repositories_with_checks.sql b/database/tests/functions/repositories/get_repositories_with_checks.sql
@@ -100,9 +100,6 @@ insert into report (
             "signed_releases": {
                 "passed": false
             },
-            "vulnerabilities": {
-                "passed": true
-            },
             "binary_artifacts": {
                 "passed": true
             },
@@ -225,9 +222,9 @@ select results_eq(
     $$,
     $$
         values
-            ('Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Website,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF Badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependency Update Tool,Maintained,SBOM,Security Policy,Signed Releases,Token Permissions,Vulnerabilities,Trademark Disclaimer'),
-            ('cncf,project1,https://repo1.url,"{code,community}",t,t,t,t,t,t,t,f,t,t,f,Apache-2.0,GA4,f,t,f,t,t,t,t,f,t,t,t,f,t,f,t,f,f,t,f'),
-            ('cncf,project1,https://repo2.url,{docs},,,,,,,f,,,t,,Apache-2.0,,,,,,,,,,,,,,,,,,,,')
+            ('Foundation,Project,Repository URL,Check Sets,Adopters,Changelog,Code of Conduct,Contributing,Governance,Maintainers,Readme,Roadmap,Website,License Approved,License Scanning,License SPDX ID,Analytics,ArtifactHub Badge,CLA,Community Meeting,DCO,GitHub discussions,OpenSSF Badge,Recent Release,Slack Presence,Binary Artifacts,Code Review,Dangerous Workflow,Dependency Update Tool,Maintained,SBOM,Security Policy,Signed Releases,Token Permissions,Trademark Disclaimer'),
+            ('cncf,project1,https://repo1.url,"{code,community}",t,t,t,t,t,t,t,f,t,t,f,Apache-2.0,GA4,f,t,f,t,t,t,t,f,t,t,t,f,t,f,t,f,f,f'),
+            ('cncf,project1,https://repo2.url,{docs},,,,,,,f,,,t,,Apache-2.0,,,,,,,,,,,,,,,,,,,')
     $$,
     'Return all repositories with all checks'
 );

diff --git a/database/tests/functions/stats/get_stats.sql b/database/tests/functions/stats/get_stats.sql
@@ -151,9 +151,6 @@ insert into report (
             "signed_releases": {
                 "passed": false
             },
-            "vulnerabilities": {
-                "passed": true
-            },
             "binary_artifacts": {
                 "passed": true
             },
@@ -282,9 +279,6 @@ insert into report (
             "signed_releases": {
                 "passed": false
             },
-            "vulnerabilities": {
-                "passed": true
-            },
             "binary_artifacts": {
                 "passed": true
             },
@@ -410,9 +404,6 @@ insert into report (
             "signed_releases": {
                 "passed": false
             },
-            "vulnerabilities": {
-                "passed": false
-            },
             "binary_artifacts": {
                 "passed": false
             },
@@ -577,8 +568,7 @@ select is(
                     "sbom": 0,
                     "security_policy": 67,
                     "signed_releases": 0,
-                    "token_permissions": 0,
-                    "vulnerabilities": 67
+                    "token_permissions": 0
                 },
                 "legal": {
                     "trademark_disclaimer": 0

diff --git a/docs/checks.md b/docs/checks.md
@@ -27,7 +27,6 @@ Checks are organized in `check sets`. Each `check set` defines a number of check
   - Security / Policy
   - Security / Signed releases
   - Security / Token permissions
-  - Security / Vulnerabilities
 
 - **code-lite** (subset of *code*, recommended for secondary code repositories)
 
@@ -612,14 +611,6 @@ This check determines whether the project's automated workflows tokens are set t
 
 *This is an OpenSSF Scorecard check. For more details please see the [check documentation](https://github.com/ossf/scorecard/blob/main/docs/checks.md#token-permissions) in the ossf/scorecard repository.*
 
-### Vulnerabilities (from OpenSSF Scorecard)
-
-**ID**: `vulnerabilities`
-
-This check determines whether the project has open, unfixed vulnerabilities using the [OSV (Open Source Vulnerabilities)](https://osv.dev/) service.
-
-*This is an OpenSSF Scorecard check. For more details please see the [check documentation](https://github.com/ossf/scorecard/blob/main/docs/checks.md#vulnerabilities) in the ossf/scorecard repository.*
-
 ## Legal
 
 ### Trademark disclaimer

diff --git a/web/src/data.tsx b/web/src/data.tsx
@@ -1,4 +1,4 @@
-import { BiLock, BiMedal, BiShieldQuarter, BiTargetLock, BiTrophy, BiWorld } from 'react-icons/bi';
+import { BiLock, BiMedal, BiShieldQuarter, BiTrophy, BiWorld } from 'react-icons/bi';
 import { BsCalendar3 } from 'react-icons/bs';
 import { CgFileDocument, CgReadme } from 'react-icons/cg';
 import {
@@ -354,16 +354,6 @@ export const REPORT_OPTIONS: ReportOptionInfo = {
     legend: <span>Projects sites should have the Linux Foundation trademark disclaimer</span>,
     reference: '/docs/topics/checks/#trademark-disclaimer',
   },
-  [ReportOption.Vulnerabilities]: {
-    icon: <BiTargetLock />,
-    name: 'Vulnerabilities',
-    legend: (
-      <span>
-        Whether the project has open, unfixed vulnerabilities (uses the OSV -Open Source Vulnerabilities- service)
-      </span>
-    ),
-    reference: '/docs/topics/checks/#vulnerabilities-from-openssf-scorecard',
-  },
   [ReportOption.Website]: {
     icon: <BiWorld />,
     name: 'Website',
@@ -415,7 +405,6 @@ export const CHECKS_PER_CATEGORY: ChecksPerCategory = {
     ReportOption.SecurityPolicy,
     ReportOption.SignedReleases,
     ReportOption.TokenPermissions,
-    ReportOption.Vulnerabilities,
   ],
   [ScoreType.Legal]: [ReportOption.TrademarkDisclaimer],
 };
diff --git a/web/src/layout/stats/__fixtures__/index/1.json b/web/src/layout/stats/__fixtures__/index/1.json
@@ -430,8 +430,7 @@
         "sbom": 4,
         "security_policy": 57,
         "signed_releases": 5,
-        "token_permissions": 18,
-        "vulnerabilities": 100
+        "token_permissions": 18
       },
       "legal": {
         "trademark_disclaimer": 40