[infra] Improve srcmap support for Go projects (#3355, #2714). (#3664)

* [infra] Improve srcmap support for Go projects (#3355, #2714). * address review feedback
google · Apr 17, 2020 · 8ed2f7a · 8ed2f7a
1 parent 1dae0c2
commit 8ed2f7a
Show file tree

Hide file tree

Showing 4 changed files with 56 additions and 65 deletions.
diff --git a/infra/base-images/base-builder/srcmap b/infra/base-images/base-builder/srcmap
@@ -25,8 +25,14 @@ function jq_inplace() {
   F=$(tempfile) && cat $1 | jq "$2" > $F && mv $F $1
 }
 
+PATHS_TO_SCAN="$SRC"
+
+if [[ $FUZZING_LANGUAGE == "go" ]]; then
+  PATHS_TO_SCAN="$PATHS_TO_SCAN $GOPATH"
+fi
+
 # Git
-for DOT_GIT_DIR in $(find $SRC -name ".git" -type d); do
+for DOT_GIT_DIR in $(find $PATHS_TO_SCAN -name ".git" -type d); do
   GIT_DIR=$(dirname $DOT_GIT_DIR)
   cd $GIT_DIR
   GIT_URL=$(git config --get remote.origin.url)
@@ -35,7 +41,7 @@ for DOT_GIT_DIR in $(find $SRC -name ".git" -type d); do
 done
 
 # Subversion
-for DOT_SVN_DIR in $(find $SRC -name ".svn" -type d); do
+for DOT_SVN_DIR in $(find $PATHS_TO_SCAN -name ".svn" -type d); do
   SVN_DIR=$(dirname $DOT_SVN_DIR)
   cd $SVN_DIR
   SVN_URL=$(svn info | grep "^URL:" | sed  's/URL: //g')
@@ -44,7 +50,7 @@ for DOT_SVN_DIR in $(find $SRC -name ".svn" -type d); do
 done
 
 # Mercurial
-for DOT_HG_DIR in $(find $SRC -name ".hg" -type d); do
+for DOT_HG_DIR in $(find $PATHS_TO_SCAN -name ".hg" -type d); do
   HG_DIR=$(dirname $DOT_HG_DIR)
   cd $HG_DIR
   HG_URL=$(hg paths default)

diff --git a/infra/gcb/build_and_run_coverage.py b/infra/gcb/build_and_run_coverage.py
@@ -71,35 +71,10 @@ def get_build_steps(project_dir):
   dockerfile_path = os.path.join(project_dir, 'Dockerfile')
   name = project_yaml['name']
   image = project_yaml['image']
+  language = project_yaml['language']
   report_date = datetime.datetime.now().strftime('%Y%m%d')
 
-  build_steps = [
-      {
-          'args': [
-              'clone',
-              'https://github.com/google/oss-fuzz.git',
-          ],
-          'name': 'gcr.io/cloud-builders/git',
-      },
-      {
-          'name': 'gcr.io/cloud-builders/docker',
-          'args': [
-              'build',
-              '-t',
-              image,
-              '.',
-          ],
-          'dir': 'oss-fuzz/projects/' + name,
-      },
-      {
-          'name': image,
-          'args': [
-              'bash', '-c',
-              'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
-          ],
-          'env': ['OSSFUZZ_REVISION=$REVISION_ID'],
-      },
-  ]
+  build_steps = build_lib.project_image_steps(name, image, language)
 
   env = CONFIGURATION[:]
   out = '/workspace/out/' + SANITIZER

diff --git a/infra/gcb/build_lib.py b/infra/gcb/build_lib.py
@@ -165,3 +165,36 @@ def gsutil_rm_rf_step(url):
       ],
   }
   return step
+
+
+def project_image_steps(name, image, language):
+  """Returns GCB steps to build OSS-Fuzz project image."""
+  steps = [{
+      'args': [
+          'clone',
+          'https://github.com/google/oss-fuzz.git',
+      ],
+      'name': 'gcr.io/cloud-builders/git',
+  }, {
+      'name': 'gcr.io/cloud-builders/docker',
+      'args': [
+          'build',
+          '-t',
+          image,
+          '.',
+      ],
+      'dir': 'oss-fuzz/projects/' + name,
+  }, {
+      'name':
+          image,
+      'args': [
+          'bash', '-c',
+          'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
+      ],
+      'env': [
+          'OSSFUZZ_REVISION=$REVISION_ID',
+          'FUZZING_LANGUAGE=%s' % language,
+      ],
+  }]
+
+  return steps
diff --git a/infra/gcb/build_project.py b/infra/gcb/build_project.py
@@ -109,45 +109,22 @@ def get_build_steps(project_dir):
   dockerfile_path = os.path.join(project_dir, 'Dockerfile')
   name = project_yaml['name']
   image = project_yaml['image']
+  language = project_yaml['language']
   run_tests = project_yaml['run_tests']
 
   ts = datetime.datetime.now().strftime('%Y%m%d%H%M')
 
-  build_steps = [
-      {
-          'args': [
-              'clone',
-              'https://github.com/google/oss-fuzz.git',
-          ],
-          'name': 'gcr.io/cloud-builders/git',
-      },
-      {
-          'name': 'gcr.io/cloud-builders/docker',
-          'args': [
-              'build',
-              '-t',
-              image,
-              '.',
-          ],
-          'dir': 'oss-fuzz/projects/' + name,
-      },
-      {
-          'name': image,
-          'args': [
-              'bash', '-c',
-              'srcmap > /workspace/srcmap.json && cat /workspace/srcmap.json'
-          ],
-          'env': ['OSSFUZZ_REVISION=$REVISION_ID'],
-      },
-      {
-          'name': 'gcr.io/oss-fuzz-base/msan-builder',
-          'args': [
-              'bash',
-              '-c',
-              'cp -r /msan /workspace',
-          ],
-      },
-  ]
+  build_steps = build_lib.project_image_steps(name, image, language)
+
+  # Copy over MSan instrumented libraries.
+  build_steps.append({
+      'name': 'gcr.io/oss-fuzz-base/msan-builder',
+      'args': [
+          'bash',
+          '-c',
+          'cp -r /msan /workspace',
+      ],
+  })
 
   for fuzzing_engine in project_yaml['fuzzing_engines']:
     for sanitizer in get_sanitizers(project_yaml):