Ensure license headers adher to Cargo manifest (#58)

README.md

@@ -305,6 +305,16 @@ Can be used to exclude files or directories from the scan.
 - Most useful in the combination with `--ensure-licenses`.
 - The excluded path can be absolute or relative.
 
+## `--file-extensions` <a name="file-extensions"></a>
+
+Scan only files with the specified extensions.
+
+Examples:
+
+```bash
+yarn start -- scan --ensure-licenses Apache-2.0 --file-extensions '.rs' -- /directory/or/file
+```
+
 # Implementation <a name="implementation"></a>
 
 [`scan`](https://github.com/paritytech/license-scanner/blob/668b8c5f1cfa1dfc8f22170562f648a344cb60ef/license-scanner/scanner.ts#L141)

license-scanner/cli/scan.ts

@@ -87,6 +87,7 @@ export const executeScan = async function ({
   logLevel,
   ensureLicenses,
   ensureProduct,
+  fileExtensions,
   exclude,
 }: ScanCliArgs) {
   const licenses = await loadLicensesNormalized(joinPath(buildRoot, "licenses"), {
@@ -113,6 +114,7 @@ export const executeScan = async function ({
       matchLicense,
       root: scanRoot,
       initialRoot: scanRoot,
+      fileExtensions,
       exclude,
       dirs: { crates: cratesDir, repositories: repositoriesDir },
       rust: { shouldCheckForCargoLock: true, cargoExecPath: "cargo", rustCrateScannerRoot },

license-scanner/license.ts

@@ -278,6 +278,7 @@ export const ensureLicensesInResult = function ({
   file,
   result,
   ensureLicenses,
+  manifestLicense,
 }: EnsureLicensesInResultOptions): Error | undefined {
   if (ensureLicenses === false) return;
   if (result === undefined) {
@@ -297,6 +298,13 @@ export const ensureLicensesInResult = function ({
         `, expected one of: ${ensureLicenses.join(",")}. Exact file path: "${file.path}"`,
     );
   }
+
+  if (manifestLicense && result.license !== manifestLicense) {
+    return new Error(
+      `${file.name} has ${result.license} license` +
+        `, expected ${manifestLicense} as in cargo manifest. Exact file path: "${file.path}"`,
+    );
+  }
 };
 
 /**

license-scanner/main.ts

@@ -53,6 +53,7 @@ program
       "If configured, the scan will make sure the product mentioned in the license headers is correct.",
     ),
   )
+  .option("--file-extensions <fileExtensions...>", "Scan only files with the specified extensions.")
   .option("--exclude <exclude...>", "Can be used to exclude files or directories from the scan.")
   // It's actually correct usage but @commander-js/extra-typings is wrong on this one.
   // eslint-disable-next-line @typescript-eslint/no-misused-promises
@@ -63,6 +64,7 @@ program
         scanRoots: scanRoots.map((scanRoot) => resolvePath(scanRoot)),
         startLinesExcludes: await readStartLinesExcludes(options.startLinesExcludes),
         detectionOverrides: await readDetectionOverrides(options.detectionOverrides),
+        fileExtensions: options.fileExtensions ?? [],
         exclude: options.exclude ?? [],
         logLevel: options.logLevel as LogLevel,
         ensureLicenses: readEnsureLicenses(options),

license-scanner/scanner.ts

@@ -120,6 +120,7 @@ const scanCrates = async function (rust: ScanOptionsRust, options: Omit<ScanOpti
 export const scan = async function (options: ScanOptions): Promise<ScanResult> {
   const {
     saveResult,
+    fileExtensions,
     exclude,
     root,
     rust,
@@ -143,6 +144,9 @@ export const scan = async function (options: ScanOptions): Promise<ScanResult> {
       logger.debug(`Excluding file ${file.path}`);
       continue toNextFile;
     }
+    if (fileExtensions.length > 0 && !fileExtensions.some((ext) => file.path.endsWith(ext))) {
+      continue toNextFile;
+    }
     tracker.setFileKey(file.path, key);
 
     logger.debug(`Enqueueing file ${file.path}`);
@@ -178,7 +182,12 @@ export const scan = async function (options: ScanOptions): Promise<ScanResult> {
 
     await scanQueue.add(async () => {
       const result = await matchLicense(file.path);
-      const licensingError = ensureLicensesInResult({ file, result, ensureLicenses });
+      const licensingError = ensureLicensesInResult({
+        file,
+        result,
+        ensureLicenses,
+        manifestLicense: file.manifestLicense,
+      });
       if (licensingError) licensingErrors.push(licensingError);
       const productError = ensureProductInFile(file.path, ensureProduct);
       if (productError) licensingErrors.push(productError);
@@ -192,8 +201,12 @@ export const scan = async function (options: ScanOptions): Promise<ScanResult> {
 
   if (rust !== null) {
     const rootCargoToml = joinPath(root, "Cargo.toml");
+    const rootCargoLock = joinPath(root, "Cargo.lock");
     if (await existsAsync(rootCargoToml)) {
-      await scanCrates(rust, options);
+      await scanCrates(
+        { ...rust, shouldCheckForCargoLock: rust.shouldCheckForCargoLock && (await existsAsync(rootCargoLock)) },
+        options,
+      );
     }
   }
 

license-scanner/types.ts

@@ -59,6 +59,7 @@ export type ScanOptions = {
   saveResult: (projectId: string, filePathFromRoot: string, result: ScanResultItem) => Promise<void>;
   root: string;
   initialRoot: string;
+  fileExtensions: string[];
   exclude: string[];
   dirs: {
     repositories: string;
@@ -101,6 +102,7 @@ export type EnsureLicensesInResultOptions = {
   file: { path: string; name: string };
   result: ScanResultItem | undefined;
   ensureLicenses: boolean | string[];
+  manifestLicense?: string;
 };
 
 export class DatabaseSaveError extends Error {
@@ -150,6 +152,7 @@ export type CargoMetadataOutputV1 = {
 
 export interface ScanCliArgs {
   scanRoots: string[];
+  fileExtensions: string[];
   exclude: string[];
   startLinesExcludes: string[];
   detectionOverrides: DetectionOverride[];

license-scanner/utils.ts

@@ -16,25 +16,39 @@ export const lstatAsync = promisify(fs.lstat);
 export const writeFileAsync = promisify(fs.writeFile);
 const mkdirAsync = promisify(fs.mkdir);
 
+/**
+ * Recursively traverses down the given directory and yields all found files.
+ * The files contained within the score of a Cargo.toml manifest are annotated with
+ * the corresponding license, if it exists.
+ */
 export const walkFiles: (
   dir: string,
   options?: {
+    manifestLicense?: string;
     excluding?: { initialRoot: string; exclude: string[] };
   },
-) => AsyncGenerator<{ path: string; name: string }> = async function* (dir, { excluding } = {}) {
+) => AsyncGenerator<{ path: string; name: string; manifestLicense?: string }> = async function* (dir, options = {}) {
+  const { excluding } = options;
   if ((await lstatAsync(dir)).isFile()) {
     if (excluding && shouldExclude({ targetPath: dir, ...excluding })) return;
-    yield { path: dir, name: path.basename(dir) };
+    yield { path: dir, name: path.basename(dir), manifestLicense: options.manifestLicense };
     return;
   }
 
+  let manifestLicense = options.manifestLicense;
+  const rootCargoToml = path.join(dir, "Cargo.toml");
+  if (await existsAsync(rootCargoToml)) {
+    const manifest = fs.readFileSync(rootCargoToml, "utf-8");
+    manifestLicense = manifest.match(/license = "(.*)"/)?.[1];
+  }
+
   for await (const d of await fs.promises.opendir(dir)) {
     const fullPath = path.join(dir, d.name);
     if (excluding && shouldExclude({ targetPath: fullPath, ...excluding })) continue;
     if (d.isDirectory()) {
-      yield* walkFiles(fullPath, { excluding });
+      yield* walkFiles(fullPath, { excluding, manifestLicense });
     } else {
-      yield { path: fullPath, name: d.name };
+      yield { path: fullPath, name: d.name, manifestLicense };
     }
   }
 };

package.json

@@ -1,6 +1,6 @@
 {
   "name": "@paritytech/license-scanner",
-  "version": "0.0.6",
+  "version": "0.0.7",
   "author": "Parity <[email protected]> (https://parity.io)",
   "type": "module",
   "license": "Apache-2.0",