Skip to content

Commit

Permalink
Add support for importing zipped datasets
Browse files Browse the repository at this point in the history
  • Loading branch information
mthh committed Nov 12, 2024
1 parent 5b1ccd4 commit 2848dad
Show file tree
Hide file tree
Showing 3 changed files with 59 additions and 6 deletions.
46 changes: 41 additions & 5 deletions src/components/ImportWindow.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,9 @@ import {
SupportedGeoFileTypes,
SupportedTabularFileTypes,
} from '../helpers/supportedFormats';
import { findCsvDelimiter, getDatasetInfo, removeEmptyLines } from '../helpers/formatConversion';
import {
extractZipContent, findCsvDelimiter, getDatasetInfo, removeEmptyLines,
} from '../helpers/formatConversion';
import { removeNadGrids } from '../helpers/projection';

// Stores
Expand Down Expand Up @@ -455,7 +457,6 @@ const analyzeDataset = async (
|| file.ext === 'gpkg'
|| file.ext === 'gml'
|| file.ext === 'kml'
// TODO: handle zip files...
) {
result = await analyseGeospatialDatasetGDAL(file);
} else if (
Expand Down Expand Up @@ -492,14 +493,49 @@ const extTransform = (ext) => {
return ext;
};

const groupFiles = (
const groupFiles = async (
files: CustomFileList,
): { [key: string]: { name: string, files: FileEntry[] } } => {
// We want to group the files by their name (because shapefiles have multiple files)
// but other files have only one file (and we want to avoid grouping
// other files than shapefiles).
const groupedFiles: { [key: string]: { name: string, files: FileEntry[] } } = {};
files.forEach((file) => {

// We want to process Zip files first
// to put the content of the zip file in the groupedFiles object
if (files.some((file) => file.ext === 'zip')) {
// eslint-disable-next-line no-restricted-syntax
for (const file of files.filter((f) => f.ext === 'zip')) {
// eslint-disable-next-line no-await-in-loop
const uzfiles = (await extractZipContent(file))
.filter((f) => allowedFileExtensions.includes(f.ext) && f.ext !== 'zip');
// eslint-disable-next-line no-restricted-syntax, no-await-in-loop
for await (const uzfile of uzfiles) {
const key = `${uzfile.name}.${extTransform(uzfile.ext)}`;
if (groupedFiles[key] === undefined) {
groupedFiles[key] = {
name: uzfile.name,
files: [uzfile],
};
} else {
if (
groupedFiles[key].files.some((f) => (
f.name === uzfile.name
&& f.ext === uzfile.ext
&& f.file.size === uzfile.file.size
&& f.file.lastModified === uzfile.file.lastModified))
) {
// eslint-disable-next-line no-continue
continue;
}
groupedFiles[key].files.push(uzfile);
}
}
}
}

// Process the other files
files.filter((file) => file.ext !== 'zip').forEach((file) => {
const key = `${file.name}.${extTransform(file.ext)}`;
if (groupedFiles[key] === undefined) {
groupedFiles[key] = {
Expand Down Expand Up @@ -560,7 +596,7 @@ export default function ImportWindow(): JSX.Element {
const [fileDescriptions] = createResource<any, any>(
droppedFiles,
async () => {
const groupedFiles = groupFiles(droppedFiles());
const groupedFiles = await groupFiles(droppedFiles());
const invalidToBeRemoved: string[] = [];
const resultValue = createMutable((await Promise.all(
Object.keys(groupedFiles)
Expand Down
17 changes: 17 additions & 0 deletions src/helpers/formatConversion.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import { SupportedTabularFileTypes } from './supportedFormats';

// Types
import type { GeoJSONFeatureCollection, GeoJSONFeature } from '../global';
import type { FileEntry } from './fileUpload';

/**
* Convert the given file(s) to a GeoJSON feature collection.
Expand Down Expand Up @@ -53,6 +54,22 @@ export async function convertBinaryTabularDatasetToJSON(
});
}

export const extractZipContent = async (
file: FileEntry,
): Promise<FileEntry[]> => {
const zip = new JSZip();
const content = await file.file.arrayBuffer();
const zipFile = await zip.loadAsync(content);
return Promise.all(Object.keys(zipFile.files)
.map((fileName) => zipFile.files[fileName])
.filter((f) => !f.dir)
.map(async (f) => ({
name: f.name.substring(0, f.name.lastIndexOf('.')),
ext: f.name.substring(f.name.lastIndexOf('.') + 1, f.name.length).toLowerCase(),
file: new File([await f.async('blob')], f.name),
})));
};

export const removeFeaturesWithEmptyGeometry = (layer: GeoJSONFeatureCollection) => {
// We want features with non-empty geometries
// (i.e each geometry is not null nor undefined and there is a non-empty coordinates array).
Expand Down
2 changes: 1 addition & 1 deletion src/helpers/supportedFormats.ts
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ export const allowedFileExtensions: string[] = [
'topojson',
'gml',
'kml',
'flatgeobuf',
// 'flatgeobuf',
'csv',
'tsv',
'xls',
Expand Down

0 comments on commit 2848dad

Please sign in to comment.