From 2848dadc12ca89a999745217bd624cb8b62cea76 Mon Sep 17 00:00:00 2001
From: Matthieu Viry <matthieu.viry@cnrs.fr>
Date: Tue, 12 Nov 2024 18:11:54 +0100
Subject: [PATCH] Add support for importing zipped datasets

---
 src/components/ImportWindow.tsx | 46 +++++++++++++++++++++++++++++----
 src/helpers/formatConversion.ts | 17 ++++++++++++
 src/helpers/supportedFormats.ts |  2 +-
 3 files changed, 59 insertions(+), 6 deletions(-)

diff --git a/src/components/ImportWindow.tsx b/src/components/ImportWindow.tsx
index 0ed2efae9..a91e604c3 100644
--- a/src/components/ImportWindow.tsx
+++ b/src/components/ImportWindow.tsx
@@ -35,7 +35,9 @@ import {
   SupportedGeoFileTypes,
   SupportedTabularFileTypes,
 } from '../helpers/supportedFormats';
-import { findCsvDelimiter, getDatasetInfo, removeEmptyLines } from '../helpers/formatConversion';
+import {
+  extractZipContent, findCsvDelimiter, getDatasetInfo, removeEmptyLines,
+} from '../helpers/formatConversion';
 import { removeNadGrids } from '../helpers/projection';
 
 // Stores
@@ -455,7 +457,6 @@ const analyzeDataset = async (
       || file.ext === 'gpkg'
       || file.ext === 'gml'
       || file.ext === 'kml'
-      // TODO: handle zip files...
     ) {
       result = await analyseGeospatialDatasetGDAL(file);
     } else if (
@@ -492,14 +493,49 @@ const extTransform = (ext) => {
   return ext;
 };
 
-const groupFiles = (
+const groupFiles = async (
   files: CustomFileList,
 ): { [key: string]: { name: string, files: FileEntry[] } } => {
   // We want to group the files by their name (because shapefiles have multiple files)
   // but other files have only one file (and we want to avoid grouping
   // other files than shapefiles).
   const groupedFiles: { [key: string]: { name: string, files: FileEntry[] } } = {};
-  files.forEach((file) => {
+
+  // We want to process Zip files first
+  // to put the content of the zip file in the groupedFiles object
+  if (files.some((file) => file.ext === 'zip')) {
+    // eslint-disable-next-line no-restricted-syntax
+    for (const file of files.filter((f) => f.ext === 'zip')) {
+      // eslint-disable-next-line no-await-in-loop
+      const uzfiles = (await extractZipContent(file))
+        .filter((f) => allowedFileExtensions.includes(f.ext) && f.ext !== 'zip');
+      // eslint-disable-next-line no-restricted-syntax, no-await-in-loop
+      for await (const uzfile of uzfiles) {
+        const key = `${uzfile.name}.${extTransform(uzfile.ext)}`;
+        if (groupedFiles[key] === undefined) {
+          groupedFiles[key] = {
+            name: uzfile.name,
+            files: [uzfile],
+          };
+        } else {
+          if (
+            groupedFiles[key].files.some((f) => (
+              f.name === uzfile.name
+              && f.ext === uzfile.ext
+              && f.file.size === uzfile.file.size
+              && f.file.lastModified === uzfile.file.lastModified))
+          ) {
+            // eslint-disable-next-line no-continue
+            continue;
+          }
+          groupedFiles[key].files.push(uzfile);
+        }
+      }
+    }
+  }
+
+  // Process the other files
+  files.filter((file) => file.ext !== 'zip').forEach((file) => {
     const key = `${file.name}.${extTransform(file.ext)}`;
     if (groupedFiles[key] === undefined) {
       groupedFiles[key] = {
@@ -560,7 +596,7 @@ export default function ImportWindow(): JSX.Element {
   const [fileDescriptions] = createResource<any, any>(
     droppedFiles,
     async () => {
-      const groupedFiles = groupFiles(droppedFiles());
+      const groupedFiles = await groupFiles(droppedFiles());
       const invalidToBeRemoved: string[] = [];
       const resultValue = createMutable((await Promise.all(
         Object.keys(groupedFiles)
diff --git a/src/helpers/formatConversion.ts b/src/helpers/formatConversion.ts
index 14580f46a..816ee92bd 100644
--- a/src/helpers/formatConversion.ts
+++ b/src/helpers/formatConversion.ts
@@ -8,6 +8,7 @@ import { SupportedTabularFileTypes } from './supportedFormats';
 
 // Types
 import type { GeoJSONFeatureCollection, GeoJSONFeature } from '../global';
+import type { FileEntry } from './fileUpload';
 
 /**
  * Convert the given file(s) to a GeoJSON feature collection.
@@ -53,6 +54,22 @@ export async function convertBinaryTabularDatasetToJSON(
   });
 }
 
+export const extractZipContent = async (
+  file: FileEntry,
+): Promise<FileEntry[]> => {
+  const zip = new JSZip();
+  const content = await file.file.arrayBuffer();
+  const zipFile = await zip.loadAsync(content);
+  return Promise.all(Object.keys(zipFile.files)
+    .map((fileName) => zipFile.files[fileName])
+    .filter((f) => !f.dir)
+    .map(async (f) => ({
+      name: f.name.substring(0, f.name.lastIndexOf('.')),
+      ext: f.name.substring(f.name.lastIndexOf('.') + 1, f.name.length).toLowerCase(),
+      file: new File([await f.async('blob')], f.name),
+    })));
+};
+
 export const removeFeaturesWithEmptyGeometry = (layer: GeoJSONFeatureCollection) => {
   // We want features with non-empty geometries
   // (i.e each geometry is not null nor undefined and there is a non-empty coordinates array).
diff --git a/src/helpers/supportedFormats.ts b/src/helpers/supportedFormats.ts
index bc22afbd9..201ef86f2 100644
--- a/src/helpers/supportedFormats.ts
+++ b/src/helpers/supportedFormats.ts
@@ -95,7 +95,7 @@ export const allowedFileExtensions: string[] = [
   'topojson',
   'gml',
   'kml',
-  'flatgeobuf',
+  // 'flatgeobuf',
   'csv',
   'tsv',
   'xls',