Merge pull request #75 from snyk/feat/yarn-v2

Feat/yarn v2

lib/errors/out-of-sync-error.ts

@@ -1,3 +1,5 @@
+import { LockfileType } from '../parsers';
+
 const LOCK_FILE_NAME = {
   npm: 'package-lock.json',
   yarn: 'yarn.lock',
@@ -14,7 +16,7 @@ export class OutOfSyncError extends Error {
   public dependencyName: string;
   public lockFileType: string;
 
-  constructor(dependencyName: string, lockFileType: 'yarn' | 'npm') {
+  constructor(dependencyName: string, lockFileType: LockfileType) {
     super(
       `Dependency ${dependencyName} was not found in ` +
         `${LOCK_FILE_NAME[lockFileType]}. Your package.json and ` +

lib/index.ts

@@ -13,6 +13,7 @@ import {
 } from './parsers';
 import { PackageLockParser } from './parsers/package-lock-parser';
 import { YarnLockParser } from './parsers/yarn-lock-parse';
+import { Yarn2LockParser } from './parsers/yarn2-lock-parse';
 import getRuntimeVersion from './get-node-runtime-version';
 import {
   UnsupportedRuntimeError,
@@ -51,13 +52,16 @@ async function buildDepTree(
       lockfileParser = new PackageLockParser();
       break;
     case LockfileType.yarn:
-      // parsing yarn.lock is supported for Node.js v6 and higher
-      if (getRuntimeVersion() >= 6) {
-        lockfileParser = new YarnLockParser();
+      lockfileParser = new YarnLockParser();
+      break;
+    case LockfileType.yarn2:
+      // parsing yarn.lock is supported for Node.js v10 and higher
+      if (getRuntimeVersion() >= 10) {
+        lockfileParser = new Yarn2LockParser();
       } else {
         throw new UnsupportedRuntimeError(
           'Parsing `yarn.lock` is not ' +
-            'supported on Node.js version less than 6. Please upgrade your ' +
+            'supported on Node.js version less than 10. Please upgrade your ' +
             'Node.js environment or use `package-lock.json`',
         );
       }
@@ -95,18 +99,6 @@ async function buildDepTreeFromFiles(
     throw new Error('Missing required parameters for buildDepTreeFromFiles()');
   }
 
-  let lockFileType: LockfileType;
-  if (lockFilePath.endsWith('package-lock.json')) {
-    lockFileType = LockfileType.npm;
-  } else if (lockFilePath.endsWith('yarn.lock')) {
-    lockFileType = LockfileType.yarn;
-  } else {
-    throw new InvalidUserInputError(
-      `Unknown lockfile ${lockFilePath}. ` +
-        'Please provide either package-lock.json or yarn.lock.',
-    );
-  }
-
   const manifestFileFullPath = path.resolve(root, manifestFilePath);
   const lockFileFullPath = path.resolve(root, lockFilePath);
 
@@ -125,6 +117,27 @@ async function buildDepTreeFromFiles(
   const manifestFileContents = fs.readFileSync(manifestFileFullPath, 'utf-8');
   const lockFileContents = fs.readFileSync(lockFileFullPath, 'utf-8');
 
+  let lockFileType: LockfileType;
+  if (lockFilePath.endsWith('package-lock.json')) {
+    lockFileType = LockfileType.npm;
+  } else if (lockFilePath.endsWith('yarn.lock')) {
+    if (
+      lockFileContents.includes('__metadata') ||
+      fs.existsSync(
+        path.resolve(root, lockFilePath.replace('yarn.lock', '.yarnrc.yml')),
+      )
+    ) {
+      lockFileType = LockfileType.yarn2;
+    } else {
+      lockFileType = LockfileType.yarn;
+    }
+  } else {
+    throw new InvalidUserInputError(
+      `Unknown lockfile ${lockFilePath}. ` +
+        'Please provide either package-lock.json or yarn.lock.',
+    );
+  }
+
   return await buildDepTree(
     manifestFileContents,
     lockFileContents,

lib/parsers/index.ts

@@ -1,6 +1,7 @@
 import { PackageLock } from './package-lock-parser';
 import { YarnLock } from './yarn-lock-parse';
 import { InvalidUserInputError } from '../errors';
+import { Yarn2Lock } from './yarn2-lock-parse';
 
 export interface Dep {
   name: string;
@@ -53,6 +54,7 @@ export interface PkgTree extends DepTreeDep {
   };
   meta?: {
     nodeVersion: string;
+    packageManagerVersion?: string;
   };
   hasDevDependencies?: boolean;
   cyclic?: boolean;
@@ -67,6 +69,7 @@ export enum Scope {
 export enum LockfileType {
   npm = 'npm',
   yarn = 'yarn',
+  yarn2 = 'yarn2',
 }
 
 export interface LockfileParser {
@@ -79,7 +82,7 @@ export interface LockfileParser {
   ) => Promise<PkgTree>;
 }
 
-export type Lockfile = PackageLock | YarnLock;
+export type Lockfile = PackageLock | YarnLock | Yarn2Lock;
 
 export function parseManifestFile(manifestFileContents: string): ManifestFile {
   try {

lib/parsers/package-lock-parser.ts

@@ -171,7 +171,7 @@ export class PackageLockParser implements LockfileParser {
         // TODO: also check the package version
         // for a stricter check
         if (strict) {
-          throw new OutOfSyncError(depName, 'npm');
+          throw new OutOfSyncError(depName, LockfileType.npm);
         }
         depTree.dependencies[dep.name] = createDepTreeDepFromDep(dep);
         if (!depTree.dependencies[dep.name].labels) {
@@ -363,7 +363,7 @@ export class PackageLockParser implements LockfileParser {
     }
 
     if (!depMap[depName]) {
-      throw new OutOfSyncError(depName, 'npm');
+      throw new OutOfSyncError(depName, LockfileType.npm);
     }
 
     return depName;

lib/parsers/yarn-lock-parse-base.ts

@@ -23,7 +23,7 @@ import { config } from '../config';
 
 const EVENT_PROCESSING_CONCURRENCY = 5;
 
-export type YarnLockFileTypes = LockfileType.yarn;
+export type YarnLockFileTypes = LockfileType.yarn | LockfileType.yarn2;
 
 export interface YarnLockDeps {
   [depName: string]: YarnLockDep;
@@ -84,6 +84,10 @@ export abstract class YarnLockParseBase<T extends YarnLockFileTypes>
       _set(depTree, 'meta.nodeVersion', nodeVersion);
     }
 
+    const packageManagerVersion =
+      lockfile.type === LockfileType.yarn ? '1' : '2';
+    _set(depTree, 'meta.packageManagerVersion', packageManagerVersion);
+
     const topLevelDeps: Dep[] = getTopLevelDeps(manifestFile, includeDev);
     // asked to process empty deps
     if (_isEmpty(manifestFile.dependencies) && !includeDev) {

lib/parsers/yarn-utils.ts

@@ -0,0 +1,87 @@
+import { structUtils } from '@yarnpkg/core';
+import * as _ from 'lodash';
+
+const BUILTIN_PLACEHOLDER = 'builtin';
+const MULTIPLE_KEYS_REGEXP = / *, */g;
+
+export type ParseDescriptor = typeof structUtils.parseDescriptor;
+export type ParseRange = typeof structUtils.parseRange;
+
+const keyNormalizer = (
+  parseDescriptor: ParseDescriptor,
+  parseRange: ParseRange,
+) => (rawDescriptor: string): string[] => {
+  // See https://yarnpkg.com/features/protocols
+  const descriptors: string[] = [rawDescriptor];
+  const descriptor = parseDescriptor(rawDescriptor);
+  const name = `${descriptor.scope ? '@' + descriptor.scope + '/' : ''}${
+    descriptor.name
+  }`;
+  const range = parseRange(descriptor.range);
+  const protocol = range.protocol;
+  switch (protocol) {
+    case 'npm:':
+    case 'file:':
+      descriptors.push(`${name}@${range.selector}`);
+      descriptors.push(`${name}@${protocol}${range.selector}`);
+      break;
+    case 'git:':
+    case 'git+ssh:':
+    case 'git+http:':
+    case 'git+https:':
+    case 'github:':
+      if (range.source) {
+        descriptors.push(
+          `${name}@${protocol}${range.source}${
+            range.selector ? '#' + range.selector : ''
+          }`,
+        );
+      } else {
+        descriptors.push(`${name}@${protocol}${range.selector}`);
+      }
+      break;
+    case 'patch:':
+      if (range.source && range.selector.indexOf(BUILTIN_PLACEHOLDER) === 0) {
+        descriptors.push(range.source);
+      } else {
+        descriptors.push(
+          `${name}@${protocol}${range.source}${
+            range.selector ? '#' + range.selector : ''
+          }`,
+        );
+      }
+      break;
+    case null:
+    case undefined:
+      if (range.source) {
+        descriptors.push(`${name}@${range.source}#${range.selector}`);
+      } else {
+        descriptors.push(`${name}@${range.selector}`);
+      }
+      break;
+    case 'http:':
+    case 'https:':
+    case 'link:':
+    case 'portal:':
+    case 'exec:':
+    case 'workspace:':
+    case 'virtual:':
+    default:
+      // For user defined plugins
+      descriptors.push(`${name}@${protocol}${range.selector}`);
+      break;
+  }
+  return descriptors;
+};
+
+export type YarnLockFileKeyNormalizer = (fullDescriptor: string) => Set<string>;
+
+export const yarnLockFileKeyNormalizer = (
+  parseDescriptor: ParseDescriptor,
+  parseRange: ParseRange,
+): YarnLockFileKeyNormalizer => (fullDescriptor: string) => {
+  const allKeys = fullDescriptor
+    .split(MULTIPLE_KEYS_REGEXP)
+    .map(keyNormalizer(parseDescriptor, parseRange));
+  return new Set<string>(_.flatMap(allKeys));
+};

lib/parsers/yarn2-lock-parse.ts

@@ -0,0 +1,55 @@
+import * as _ from 'lodash';
+import * as yaml from 'yaml';
+
+import { LockfileType } from './';
+import getRuntimeVersion from '../get-node-runtime-version';
+import { InvalidUserInputError, UnsupportedRuntimeError } from '../errors';
+import { YarnLockBase, YarnLockDeps } from './yarn-lock-parse-base';
+import { YarnLockParseBase } from './yarn-lock-parse-base';
+import {
+  YarnLockFileKeyNormalizer,
+  yarnLockFileKeyNormalizer,
+} from './yarn-utils';
+
+export type Yarn2Lock = YarnLockBase<LockfileType.yarn2>;
+
+export class Yarn2LockParser extends YarnLockParseBase<LockfileType.yarn2> {
+  private keyNormalizer: YarnLockFileKeyNormalizer;
+
+  constructor() {
+    super(LockfileType.yarn2);
+    // @yarnpkg/core doesn't work with Node.js < 10
+    if (getRuntimeVersion() < 10) {
+      throw new UnsupportedRuntimeError(
+        `yarn.lock parsing is supported for Node.js v10 and higher.`,
+      );
+    }
+    const structUtils = require('@yarnpkg/core').structUtils;
+    const parseDescriptor = structUtils.parseDescriptor;
+    const parseRange = structUtils.parseRange;
+    this.keyNormalizer = yarnLockFileKeyNormalizer(parseDescriptor, parseRange);
+  }
+
+  public parseLockFile(lockFileContents: string): Yarn2Lock {
+    try {
+      const rawYarnLock: any = yaml.parse(lockFileContents);
+      delete rawYarnLock.__metadata;
+      const dependencies: YarnLockDeps = {};
+      _.forEach(rawYarnLock, (versionData, fullDescriptor) => {
+        this.keyNormalizer(fullDescriptor).forEach((descriptor) => {
+          dependencies[descriptor] = versionData;
+        });
+      });
+      return {
+        dependencies,
+        lockfileType: LockfileType.yarn2,
+        object: dependencies,
+        type: LockfileType.yarn2,
+      };
+    } catch (e) {
+      throw new InvalidUserInputError(
+        `yarn.lock parsing failed with an error: ${e.message}`,
+      );
+    }
+  }
+}

package.json

@@ -31,7 +31,7 @@
   "homepage": "https://github.com/snyk/nodejs-lockfile-parser#readme",
   "dependencies": {
     "@snyk/graphlib": "2.1.9-patch",
-    "@yarnpkg/core": "^2.0.0-rc.26",
+    "@yarnpkg/core": "^2.0.0-rc.29",
     "@yarnpkg/lockfile": "^1.1.0",
     "event-loop-spinner": "^2.0.0",
     "lodash.clonedeep": "^4.5.0",
@@ -46,7 +46,7 @@
     "yaml": "^1.9.2"
   },
   "devDependencies": {
-    "@types/node": "^6.14.7",
+    "@types/node": "^10.17.26",
     "@types/uuid": "^3.4.4",
     "@typescript-eslint/eslint-plugin": "^2.29.0",
     "@typescript-eslint/parser": "^2.29.0",

test/lib/fixtures/cyclic-dep-simple/yarn2/.yarnrc.yml

@@ -0,0 +1 @@
+yarnPath: .yarn/releases/yarn-rc.js

test/lib/fixtures/cyclic-dep-simple/yarn2/yarn.lock

@@ -0,0 +1,23 @@
+# This file is generated by running "yarn install" inside your project.
+# Manual changes might be lost - proceed with caution!
+
+__metadata:
+  version: 4
+
+"debug@npm:2.0.x":
+  version: 2.0.0
+  resolution: "debug@npm:2.0.0"
+  dependencies:
+    ms: 0.6.2
+  checksum: 2/0d3aafea6f4d2fac3e1ad295c17927aeb1188b51184cc1a39a061911fde969c1ff9139a14f895b592f2a82035fc507ef9ec78ebddcba6aba4be0fe6e195abde7
+  languageName: node
+  linkType: hard
+
+"ms@npm:0.6.2":
+  version: 0.6.2
+  resolution: "ms@npm:0.6.2"
+  checksum: 2/58b15f75a33f042ce241d435c6eb218f46cd835f74db4f85b4dacf4143a92638cc0887be38423e79411da27dae103db3048a0d9aa4629607ab10c7d037b6f9e7
+  languageName: node
+  linkType: hard
+  dependencies:
+    debug: 2.0.x

test/lib/fixtures/dev-deps-only/expected-tree.json

@@ -21,4 +21,4 @@
       }
     }
   }
-}
+}

test/lib/fixtures/dev-deps-only/yarn1/expected-tree.json

@@ -0,0 +1,27 @@
+{
+  "name": "pkg-dev-deps-only",
+  "size": 3,
+  "version": "0.0.1",
+  "hasDevDependencies": true,
+  "dependencies": {
+    "debug": {
+      "name": "debug",
+      "version": "2.6.9",
+      "labels": {
+        "scope": "dev"
+      },
+      "dependencies": {
+        "ms": {
+          "name": "ms",
+          "version": "2.0.0",
+          "labels": {
+            "scope": "dev"
+          }
+        }
+      }
+    }
+  },
+  "meta": {
+    "packageManagerVersion": "1"
+  }
+}

test/lib/fixtures/dev-deps-only/yarn2/.yarnrc.yml

@@ -0,0 +1 @@
+yarnPath: .yarn/releases/yarn-rc.js