Merge pull request #177 from OpenCSPM/build

better handling of malformed JSON import files. #176

.github/ISSUE_TEMPLATE/bug_report.md

@@ -10,6 +10,9 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+**OpenCSPM version**
+What version of the OpenCSPM Docker image are you using? (e.g. 0.1.22)
+
 **To Reproduce**
 Steps to reproduce the behavior:
 1. Go to '...'

docker/app/jobs/lib/loader/asset/asset_router.rb

@@ -17,6 +17,8 @@
 
 # Comment
 class AssetRouter
+  INVALID_JSON = 'Expected NDJSON import file, found JSON import file.'
+
   def initialize(asset, import_id, db)
     @asset = asset
     @import_id = import_id
@@ -25,6 +27,12 @@ def initialize(asset, import_id, db)
   end
 
   def route
+    # raise exception if not NDJSON
+    if @asset.is_a?(Array)
+      puts "[asset_router] Error: #{INVALID_JSON}"
+      raise INVALID_JSON
+    end
+
     # Load an AWS resource
     return aws_load if (%w[account service region resource] - @asset.keys).empty?
 

docker/app/jobs/lib/loader/file_loader.rb

@@ -45,9 +45,14 @@ def loop_over_asset_lines(file_name)
       file.each_slice(batch_size) do |lines|
         line_count += lines.length
         Parallel.each(lines, in_processes: 8) do |line|
-          asset_json = FastJsonparser.parse(line, symbolize_keys: false)
-          AssetRouter.new(asset_json, @import_id, @db)
-          line_count += 1
+          begin
+            asset_json = FastJsonparser.parse(line, symbolize_keys: false)
+            AssetRouter.new(asset_json, @import_id, @db)
+            line_count += 1
+          rescue FastJsonparser::ParseError => e
+            puts "[file_loader] Error: JSON can't be parsed. Ensure you are loading NDJSON files."
+            raise e
+          end
         end
       end
     end

docker/docker-compose.yml

@@ -6,7 +6,7 @@ x-app: &app
     RAILS_ENV: ${RAILS_ENV:-production}
     RAILS_SERVE_STATIC_FILES: 1
     RAILS_LOG_TO_STDOUT: 1
-  image: gcr.io/opencspm/engine:0.1.22
+  image: gcr.io/opencspm/engine:0.1.23
   tmpfs:
     - /tmp