Change -union syntax, add fields= option

CHANGELOG.md

@@ -1,3 +1,7 @@
+v0.4.144
+* Change -union command syntax to use two or more target layers as input
+* Add -union fields= option for selecting which fields from the input layers to retain.
+
 v0.4.143
 * Fix for web UI bug that caused command line options to be ignored.
 

package.json

@@ -1,6 +1,6 @@
 {
 	"name": "mapshaper",
-	"version": "0.4.143",
+	"version": "0.4.144",
 	"description": "A tool for editing vector datasets for mapping and GIS.",
 	"keywords": [
 		"shapefile",

src/cli/mapshaper-options.js

@@ -1295,22 +1295,19 @@ internal.getOptionParser = function() {
     .option('target', targetOpt);
 
   parser.command('union')
-    .describe('create a flat mosaic from two polygon layers (A and B)')
-    .option('source', {
-      DEFAULT: true,
-      describe: 'file or layer to use as layer B'
-    })
-    // .option('field-prefixes', {
-    //   describe: 'prefixes of data fields from A- and B-layers (comma-sep.)'
+    .describe('create a flat mosaic from two or more polygon layers')
+    // .option('add-fid', {
+    //   describe: 'add FID_A, FID_B, ... fields to output layer',
+    //   type: 'flag'
     // })
-    .option('add-fid', {
-      describe: 'add FID_A and FID_B fields to output layer',
-      type: 'flag'
+    .option('fields', {
+      type: 'strings',
+      describe: 'fields to retain (comma-sep.) (default is all fields)',
     })
     .option('name', nameOpt)
     .option('no-replace', noReplaceOpt)
     .option('target', {
-      describe: 'specify layer A'
+      describe: 'specify layers to target (comma-sep. list)'
     });
 
   parser.section('Informational commands');

src/cli/mapshaper-run-command.js

@@ -98,9 +98,9 @@ api.runCommand = function(cmd, catalog, cb) {
     } else {
       targets = catalog.findCommandTargets(opts.target);
 
-      // special case to allow merge-layers to merge layers from multiple datasets
+      // special case to allow -merge-layers and -union to combine layers from multiple datasets
       // TODO: support multi-dataset targets for other commands
-      if (targets.length > 1 && name == 'merge-layers') {
+      if (targets.length > 1 && (name == 'merge-layers' || name == 'union')) {
         targets = internal.mergeCommandTargets(targets, catalog);
       }
 
@@ -357,7 +357,7 @@ api.runCommand = function(cmd, catalog, cb) {
       internal.target(catalog, opts);
 
     } else if (name == 'union') {
-      outputLayers = api.union(targetLayers, source, targetDataset, opts);
+      outputLayers = api.union(targetLayers, targetDataset, opts);
 
     } else if (name == 'uniq') {
       applyCommandToEachLayer(api.uniq, targetLayers, arcs, opts);

src/commands/mapshaper-union.js

@@ -1,66 +1,60 @@
 /* @require mapshaper-overlay-utils */
 
-api.union = function(targetLayers, src, targetDataset, opts) {
-  var sourceDataset;
-  if (!src || !src.layer || !src.dataset) {
-    error("Unexpected source layer argument");
+api.union = function(targetLayers, targetDataset, opts) {
+  // var mergedDataset = internal.mergeLayersForUnion(targetLayers, targetDataset);
+  if (targetLayers.length < 2) {
+    stop('Command requires at least two target layers');
   }
-  var mergedDataset = internal.mergeLayersForOverlay(targetLayers, src, targetDataset, opts);
-  var nodes = internal.addIntersectionCuts(mergedDataset, opts);
-  var unionLyr = mergedDataset.layers.pop();
-  var outputLayers = targetLayers.map(function(targetLyr) {
-    return internal.unionTwoLayers(targetLyr, unionLyr, nodes, opts);
+  var allFields = [];
+  var allShapes = [];
+  var layerData = [];
+  targetLayers.forEach(function(lyr, i) {
+    internal.requirePolygonLayer(lyr);
+    var fields = lyr.data ? lyr.data.getFields() : [];
+    if (opts.fields) {
+      fields = opts.fields.indexOf('*') > 1 ? fields :
+        fields.filter(function(name) {return opts.fields.indexOf(name) > -1;});
+    }
+    layerData.push({
+      layer: lyr,
+      fields: fields,
+      records: lyr.data ? lyr.data.getRecords() : null,
+      offset: allShapes.length,
+      size: lyr.shapes.length
+    });
+    allFields = allFields.concat(fields);
+    allShapes = allShapes.concat(lyr.shapes);
   });
-  targetDataset.arcs = nodes.arcs;
-  return outputLayers;
-};
-
-internal.unionTwoLayers = function(targetLyr, sourceLyr, nodes, opts) {
-  if (targetLyr.geometry_type != 'polygon' || sourceLyr.geometry_type != 'polygon') {
-    stop('Command requires two polygon layers');
-  }
-  var mergedLayer = {
+  var unionFields = utils.uniqifyNames(allFields, function(name, n) {
+    return name + '_' + n;
+  });
+  var mergedLyr = {
     geometry_type: 'polygon',
-    shapes: targetLyr.shapes.concat(sourceLyr.shapes)
+    shapes: allShapes
   };
-  // Use suffixes to disambiguate same-name fields
-  // TODO: add an option to override these defaults
-  var suffixA = '_A';
-  var suffixB = '_B';
-  var decorateRecord = opts.each ? internal.getUnionRecordDecorator(opts.each, targetLyr, sourceLyr, nodes.arcs) : null;
-  var mosaicIndex = new MosaicIndex(mergedLayer, nodes, {flat: false});
+  var nodes = internal.addIntersectionCuts(targetDataset, opts);
+  var mosaicIndex = new MosaicIndex(mergedLyr, nodes, {flat: false});
   var mosaicShapes = mosaicIndex.mosaic;
-  var targetRecords = targetLyr.data ? targetLyr.data.getRecords() : null;
-  var targetFields = targetLyr.data ? targetLyr.data.getFields() : [];
-  var targetSize = targetLyr.shapes.length;
-  var sourceRecords = sourceLyr.data ? sourceLyr.data.getRecords() : null;
-  var sourceFields = sourceLyr.data ? sourceLyr.data.getFields() : [];
-  var sourceSize = sourceLyr.shapes.length;
-  var targetMap = internal.unionGetFieldMap(targetFields, sourceFields, suffixA);
-  var sourceMap = internal.unionGetFieldMap(sourceFields, targetFields, suffixB);
-
   var mosaicRecords = mosaicShapes.map(function(shp, i) {
     var mergedIds = mosaicIndex.getSourceIdsByTileId(i);
-    var targetId = internal.unionFindOriginId(mergedIds, targetSize, sourceSize);
-    var sourceId = internal.unionFindOriginId(mergedIds, 0, targetSize);
-    var rec = {};
-    var targetRec = targetId > -1 && targetRecords ? targetRecords[targetId] : null;
-    var sourceRec = sourceId > -1 && sourceRecords ? sourceRecords[sourceId] : null;
-    internal.unionMergeDataProperties(rec, targetRec, targetFields, targetMap);
-    internal.unionMergeDataProperties(rec, sourceRec, sourceFields, sourceMap);
-    if (opts.add_fid) {
-      rec.FID_A = targetId;
-      rec.FID_B = sourceId;
+    var values = [];
+    var lyrInfo, srcId, rec;
+    for (var lyrId=0, n=layerData.length; lyrId < n; lyrId++) {
+      lyrInfo = layerData[lyrId];
+      srcId = internal.unionFindOriginId(mergedIds, lyrInfo.offset, lyrInfo.size);
+      rec = srcId == -1 || lyrInfo.records === null ? null : lyrInfo.records[srcId];
+      internal.unionAddDataValues(values, lyrInfo.fields, rec);
     }
-    return rec;
+    return internal.unionMakeDataRecord(unionFields, values);
   });
+
   var unionLyr = {
     geometry_type: 'polygon',
     shapes: mosaicShapes,
     data: new DataTable(mosaicRecords)
   };
-  if ('name' in targetLyr) unionLyr.name = targetLyr.name;
-  return unionLyr;
+  // if ('name' in targetLyr) unionLyr.name = targetLyr.name;
+  return [unionLyr];
 };
 
 internal.unionFindOriginId = function(mergedIds, offset, length) {
@@ -74,15 +68,16 @@ internal.unionFindOriginId = function(mergedIds, offset, length) {
   return -1;
 };
 
-internal.unionMergeDataProperties = function(outRec, inRec, fields, fieldMap) {
+internal.unionAddDataValues = function(arr, fields, rec) {
   for (var i=0; i<fields.length; i++) {
-    outRec[fieldMap[fields[i]]] = inRec ? inRec[fields[i]] : null;
+    arr.push(rec ? rec[fields[i]] : null);
   }
 };
 
-internal.unionGetFieldMap = function(fields, otherFields, suffix) {
-  return fields.reduce(function(memo, field) {
-    memo[field] = otherFields.indexOf(field) > -1 ? field + suffix : field;
-    return memo;
-  }, {});
+internal.unionMakeDataRecord = function(fields, values) {
+  var rec = {};
+  for (var i=0; i<fields.length; i++) {
+    rec[fields[i]] = values[i];
+  }
+  return rec;
 };

src/datatable/mapshaper-data-utils.js

@@ -143,6 +143,7 @@ internal.getUniqFieldNames = function(fields, maxLen) {
   });
 };
 
+
 internal.getUniqFieldValues = function(records, field) {
   var index = {};
   var values = [];

src/utils/mapshaper-utils.js

@@ -103,24 +103,24 @@ utils.formatVersionedName = function(name, i) {
 utils.uniqifyNames = function(names, formatter) {
   var counts = utils.countValues(names),
       format = formatter || utils.formatVersionedName,
-      blacklist = {};
+      names2 = [];
 
-  Object.keys(counts).forEach(function(name) {
-    if (counts[name] > 1) blacklist[name] = true; // uniqify all instances of a name
-  });
-  return names.map(function(name) {
-    var i = 1, // first version id
+  names.forEach(function(name) {
+    var i = 0,
         candidate = name,
         versionedName;
-    while (candidate in blacklist) {
+    while (
+        names2.indexOf(candidate) > -1 || // candidate name has already been used
+        candidate == name && counts[candidate] > 1 || // duplicate unversioned names
+        candidate != name && counts[candidate] > 0) { // versioned name is a preexisting name
+      i++;
       versionedName = format(name, i);
       if (!versionedName || versionedName == candidate) {
         throw new Error("Naming error"); // catch buggy versioning function
       }
       candidate = versionedName;
-      i++;
     }
-    blacklist[candidate] = true;
-    return candidate;
+    names2.push(candidate);
   });
+  return names2;
 };

test/test_data/issues/384/data.csv

@@ -0,0 +1,2 @@
+plz,ort,bundesland
+01067,Dresden,Sachsen

test/test_data/issues/union/polygonC.json

@@ -0,0 +1,15 @@
+{
+  "type": "FeatureCollection",
+  "features": [{
+      "type": "Feature",
+      "properties": {
+        "name": "C",
+        "value": 0
+      },
+      "geometry": {
+        "type": "Polygon",
+        "coordinates": [[[2.1, 1], [3.1, 2], [4.1, 1], [3.1, 0], [2.1, 1]]]
+      }
+    }
+  ]
+}

test/union-test.js

@@ -7,30 +7,49 @@ describe('mapshaper-union.js', function () {
   it('union of two polygons with identical fields adds disambiguating suffixes', function(done) {
     var fileA = 'test/test_data/issues/union/polygonA.json';
     var fileB = 'test/test_data/issues/union/polygonB.json';
-    var cmd = `-i ${fileA} -union ${fileB} name=merged -o`;
+    var cmd = `-i ${fileA} ${fileB} combine-files -union name=merged -o`;
     api.applyCommands(cmd, {}, function(err, out) {
       var features = JSON.parse(out['merged.json']).features;
       var records = _.pluck(features, 'properties');
       assert.deepEqual(records, [
-        { name_B: 'B', value_B: 8, name_A: null, value_A: null },
-        { name_B: 'B', value_B: 8, name_A: 'A', value_A: 4 },
-        { name_B: null, value_B: null, name_A: 'A', value_A: 4 }
+        { name_2: null, value_2: null, name_1: 'A', value_1: 4 },
+        { name_2: 'B', value_2: 8, name_1: 'A', value_1: 4 },
+        { name_2: 'B', value_2: 8, name_1: null, value_1: null }
       ]);
       done();
     });
   });
 
-  it('-union add-fid adds FID_A and FID_B fields', function(done) {
+  it('fields= option selects fields to retain', function(done) {
     var fileA = 'test/test_data/issues/union/polygonA.json';
     var fileB = 'test/test_data/issues/union/polygonB.json';
-    var cmd = `-i ${fileA} -union add-fid ${fileB} name=merged -o`;
+    var cmd = `-i ${fileA} ${fileB} combine-files -union fields=name name=merged -o`;
     api.applyCommands(cmd, {}, function(err, out) {
       var features = JSON.parse(out['merged.json']).features;
       var records = _.pluck(features, 'properties');
       assert.deepEqual(records, [
-        { FID_A: -1, FID_B: 0, name_B: 'B', value_B: 8, name_A: null, value_A: null },
-        { FID_A: 0, FID_B: 0, name_B: 'B', value_B: 8, name_A: 'A', value_A: 4 },
-        { FID_A: 0, FID_B: -1, name_B: null, value_B: null, name_A: 'A', value_A: 4 }
+        { name_2: null, name_1: 'A'},
+        { name_2: 'B', name_1: 'A'},
+        { name_2: 'B', name_1: null}
+      ]);
+      done();
+    });
+  });
+
+  it('union of three polygons with identical fields', function(done) {
+    var fileA = 'test/test_data/issues/union/polygonA.json';
+    var fileB = 'test/test_data/issues/union/polygonB.json';
+    var fileC = 'test/test_data/issues/union/polygonC.json';
+    var cmd = `-i ${fileA} ${fileB} ${fileC} combine-files -union name=merged -o`;
+    api.applyCommands(cmd, {}, function(err, out) {
+      var features = JSON.parse(out['merged.json']).features;
+      var records = _.pluck(features, 'properties');
+      assert.deepEqual(records, [
+        { name_2: null, value_2: null, name_1: 'A', value_1: 4, name_3: null, value_3: null },
+        { name_2: 'B', value_2: 8, name_1: 'A', value_1: 4, name_3: null, value_3: null  },
+        { name_2: 'B', value_2: 8, name_1: null, value_1: null, name_3: null, value_3: null  },
+        { name_2: 'B', value_2: 8, name_1: null, value_1: null, name_3: 'C', value_3: 0  },
+        { name_2: null, value_2: null, name_1: null, value_1: null, name_3: 'C', value_3: 0  }
       ]);
       done();
     });
@@ -39,14 +58,14 @@ describe('mapshaper-union.js', function () {
   it('union of two polygons with different fields preserves field names', function(done) {
     var fileA = 'test/test_data/issues/union/polygonA.json';
     var fileB = 'test/test_data/issues/union/polygonB.json';
-    var cmd = `-i ${fileA} -rename-fields nameA=name,valueA=value -union ${fileB} name=merged -o`;
+    var cmd = `-i ${fileA} -rename-fields nameA=name,valueA=value -i ${fileB} -union target=* name=merged -o`;
     api.applyCommands(cmd, {}, function(err, out) {
       var features = JSON.parse(out['merged.json']).features;
       var records = _.pluck(features, 'properties');
       assert.deepEqual(records, [
-        { name: 'B', value: 8, nameA: null, valueA: null },
+        { name: null, value: null, nameA: 'A', valueA: 4 },
         { name: 'B', value: 8, nameA: 'A', valueA: 4 },
-        { name: null, value: null, nameA: 'A', valueA: 4 }
+        { name: 'B', value: 8, nameA: null, valueA: null }
       ]);
       done();
     });
@@ -58,14 +77,14 @@ describe('mapshaper-union.js', function () {
       type: 'Polygon',
       coordinates: [[[1, 1], [2, 2], [3, 1], [2, 0], [1, 1]]]
     }
-    var cmd = `-i polygonB.json -i ${fileA} -union polygonB name=merged -o`;
+    var cmd = `-i polygonB.json -i ${fileA} -union target=polygonA,polygonB name=merged -o`;
     api.applyCommands(cmd, {'polygonB.json': geomB}, function(err, out) {
       var features = JSON.parse(out['merged.json']).features;
       var records = _.pluck(features, 'properties');
       assert.deepEqual(records, [
-        { name: null, value: null },
         { name: 'A', value: 4 },
-        { name: 'A', value: 4 }
+        { name: 'A', value: 4 },
+        { name: null, value: null }
       ]);
       done();
     });