added handle upload file, updated dependencies

JeMPI_Apps/JeMPI_UI/package.json

@@ -49,6 +49,7 @@
     "leader-line": "^1.0.7",
     "leader-line-types": "^1.0.5",
     "notistack": "^2.0.8",
+    "papaparse": "^5.4.1",
     "react": "^18.2.0",
     "react-dom": "^18.2.0",
     "react-draggable": "^4.4.6",
@@ -64,6 +65,7 @@
     "@types/dockerode": "^3.3.20",
     "@types/jest": "^29.5.5",
     "@types/node": "^16.7.13",
+    "@types/papaparse": "^5.3.14",
     "@types/react": "^18.0.0",
     "@types/react-dom": "^18.0.0",
     "@typescript-eslint/eslint-plugin": "^5.51.0",

JeMPI_Apps/JeMPI_UI/src/components/import/DropZone.tsx

@@ -27,6 +27,7 @@ import UploadFileListItem from './UploadFileListItem'
 import { formatBytesSize, megabytesToBytes } from 'utils/formatters'
 import { useConfig } from 'hooks/useConfig'
 import { useFormik } from 'formik'
+import Papa, { ParseResult } from 'papaparse'
 
 const DropZone: FC = () => {
   const { enqueueSnackbar } = useSnackbar()
@@ -87,46 +88,62 @@ const DropZone: FC = () => {
     maxSize: MAX_UPLOAD_FILE_SIZE_IN_BYTES
   })
 
-  const uploadFile = async (
+  const handleUpload = async (
     fileObj: FileObj,
     importQueries: importQueriesType
   ) => {
-    return await apiClient.uploadFile(
-      createFileUploadAxiosConfig(fileObj, importQueries)
-    )
+    if (!fileObj.file) return
+
+    await Papa.parse(fileObj.file, {
+      header: true,
+      skipEmptyLines: true,
+      complete: async function (results: ParseResult<Record<string, unknown>>) {
+        const rows = results.data
+        const headers = results.meta.fields
+        const chunkSize = 10000
+
+        for (let i = 0; i < rows.length; i += chunkSize) {
+          const chunk = rows.slice(i, i + chunkSize)
+
+          const csvContent = Papa.unparse(
+            { fields: headers!, data: chunk },
+            {
+              delimiter: ',',
+              header: true,
+              newline: '\r\n'
+            }
+          )
+
+          await uploadChunk(csvContent, importQueries, i)
+        }
+        console.log('Successfully uploaded data')
+      }
+    })
   }
+  const uploadChunk = async (
+    csvContent: string,
+    importQueries: importQueriesType,
+    index: number
+  ) => {
+    const blob = new Blob([csvContent], { type: 'text/csv' })
+    const file = new File([blob], `chunk-${index}.csv`, { type: 'text/csv' })
 
-  const createFileUploadAxiosConfig = (
-    fileObj: FileObj,
-    importQueries: importQueriesType
-  ): AxiosRequestConfig<FormData> => {
     const formData = new FormData()
-    formData.set('csv', fileObj.file)
-    formData.set('queries', JSON.stringify(importQueries))
-    return {
+    formData.append('csv', file)
+    formData.append('queries', JSON.stringify(importQueries))
+
+    const config = {
       signal: abortControllerRef.current.signal,
       headers: {
-        'content-type': 'multipart/form-data'
+        'Content-Type': 'multipart/form-data'
       },
-      data: formData,
-      onUploadProgress: (progressEvent: AxiosProgressEvent) => {
-        setFilesObj((prev: FileObj | undefined) => {
-          if (prev?.file.name === fileObj.file.name && progressEvent.total) {
-            return {
-              ...prev,
-              progress: Math.round(
-                (progressEvent.loaded * 100) / progressEvent.total
-              ),
-              status: UploadStatus.Loading
-            }
-          }
-        })
-      }
+      data: formData
     }
+    return await apiClient.uploadFile(config)
   }
 
   const uploadFileMutation = useMutation({
-    mutationFn: (fileObjs: FileObj) => uploadFile(fileObjs, FormValues),
+    mutationFn: (fileObjs: FileObj) => handleUpload(fileObjs, FormValues),
     onSuccess: (_, fileObj) => {
       setFilesObj((prev: FileObj | undefined) =>
         prev ? { ...prev, status: UploadStatus.Complete } : undefined
@@ -210,13 +227,11 @@ const DropZone: FC = () => {
                     }
                     label={
                       <Typography fontSize={'0.9rem'}>
-                        {
-                          " Send to the linker and use the current M & U values"
-                        }
+                        {' Send to the linker and use the current M & U values'}
                       </Typography>
                     }
                   />
-                  <br/>
+                  <br />
                   <FormControlLabel
                     control={
                       <Radio
@@ -232,9 +247,7 @@ const DropZone: FC = () => {
                     }
                     label={
                       <Typography fontSize={'0.9rem'}>
-                        {
-                          ' Send to EM task to compute new M & U values'
-                        }
+                        {' Send to EM task to compute new M & U values'}
                       </Typography>
                     }
                   />

JeMPI_Apps/JeMPI_UI/yarn.lock

@@ -2653,6 +2653,13 @@
   dependencies:
     "undici-types" "~5.26.4"
 
+"@types/papaparse@^5.3.14":
+  "integrity" "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g=="
+  "resolved" "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz"
+  "version" "5.3.14"
+  dependencies:
+    "@types/node" "*"
+
 "@types/parse-json@^4.0.0":
   "integrity" "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw=="
   "resolved" "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz"
@@ -8631,6 +8638,11 @@
   "resolved" "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz"
   "version" "2.2.0"
 
+"papaparse@^5.4.1":
+  "integrity" "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw=="
+  "resolved" "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz"
+  "version" "5.4.1"
+
 "param-case@^3.0.4":
   "integrity" "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A=="
   "resolved" "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz"