diff --git a/JeMPI_Apps/JeMPI_UI/package-lock.json b/JeMPI_Apps/JeMPI_UI/package-lock.json index 2a2ba4a1c..993948af7 100644 --- a/JeMPI_Apps/JeMPI_UI/package-lock.json +++ b/JeMPI_Apps/JeMPI_UI/package-lock.json @@ -40,6 +40,7 @@ "leader-line": "^1.0.7", "leader-line-types": "^1.0.5", "notistack": "^2.0.8", + "papaparse": "^5.4.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-draggable": "^4.4.6", @@ -55,6 +56,7 @@ "@types/dockerode": "^3.3.20", "@types/jest": "^29.5.5", "@types/node": "^16.7.13", + "@types/papaparse": "^5.3.14", "@types/react": "^18.0.0", "@types/react-dom": "^18.0.0", "@typescript-eslint/eslint-plugin": "^5.51.0", @@ -5172,6 +5174,15 @@ "@types/node": "*" } }, + "node_modules/@types/papaparse": { + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", + "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -16432,6 +16443,11 @@ "node": ">=6" } }, + "node_modules/papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, "node_modules/param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", @@ -27577,6 +27593,15 @@ "@types/node": "*" } }, + "@types/papaparse": { + "version": "5.3.14", + "resolved": "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz", + "integrity": "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==", + "dev": true, + "requires": { + "@types/node": "*" + } + }, "@types/parse-json": { "version": "4.0.2", "resolved": "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz", @@ -35093,6 +35118,11 @@ "resolved": "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz", "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==" }, + "papaparse": { + "version": "5.4.1", + "resolved": "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz", + "integrity": "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + }, "param-case": { "version": "3.0.4", "resolved": "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz", diff --git a/JeMPI_Apps/JeMPI_UI/package.json b/JeMPI_Apps/JeMPI_UI/package.json index 6ca10f463..f428d2796 100644 --- a/JeMPI_Apps/JeMPI_UI/package.json +++ b/JeMPI_Apps/JeMPI_UI/package.json @@ -49,6 +49,7 @@ "leader-line": "^1.0.7", "leader-line-types": "^1.0.5", "notistack": "^2.0.8", + "papaparse": "^5.4.1", "react": "^18.2.0", "react-dom": "^18.2.0", "react-draggable": "^4.4.6", @@ -64,6 +65,7 @@ "@types/dockerode": "^3.3.20", "@types/jest": "^29.5.5", "@types/node": "^16.7.13", + "@types/papaparse": "^5.3.14", "@types/react": "^18.0.0", "@types/react-dom": "^18.0.0", "@typescript-eslint/eslint-plugin": "^5.51.0", diff --git a/JeMPI_Apps/JeMPI_UI/src/components/import/DropZone.tsx b/JeMPI_Apps/JeMPI_UI/src/components/import/DropZone.tsx index 5b3ff762c..69a35634d 100644 --- a/JeMPI_Apps/JeMPI_UI/src/components/import/DropZone.tsx +++ b/JeMPI_Apps/JeMPI_UI/src/components/import/DropZone.tsx @@ -27,6 +27,7 @@ import UploadFileListItem from './UploadFileListItem' import { formatBytesSize, megabytesToBytes } from 'utils/formatters' import { useConfig } from 'hooks/useConfig' import { useFormik } from 'formik' +import Papa, { ParseResult } from 'papaparse' const DropZone: FC = () => { const { enqueueSnackbar } = useSnackbar() @@ -87,46 +88,62 @@ const DropZone: FC = () => { maxSize: MAX_UPLOAD_FILE_SIZE_IN_BYTES }) - const uploadFile = async ( + const handleUpload = async ( fileObj: FileObj, importQueries: importQueriesType ) => { - return await apiClient.uploadFile( - createFileUploadAxiosConfig(fileObj, importQueries) - ) + if (!fileObj.file) return + + await Papa.parse(fileObj.file, { + header: true, + skipEmptyLines: true, + complete: async function (results: ParseResult>) { + const rows = results.data + const headers = results.meta.fields + const chunkSize = 10000 + + for (let i = 0; i < rows.length; i += chunkSize) { + const chunk = rows.slice(i, i + chunkSize) + + const csvContent = Papa.unparse( + { fields: headers!, data: chunk }, + { + delimiter: ',', + header: true, + newline: '\r\n' + } + ) + + await uploadChunk(csvContent, importQueries, i) + } + console.log('Successfully uploaded data') + } + }) } + const uploadChunk = async ( + csvContent: string, + importQueries: importQueriesType, + index: number + ) => { + const blob = new Blob([csvContent], { type: 'text/csv' }) + const file = new File([blob], `chunk-${index}.csv`, { type: 'text/csv' }) - const createFileUploadAxiosConfig = ( - fileObj: FileObj, - importQueries: importQueriesType - ): AxiosRequestConfig => { const formData = new FormData() - formData.set('csv', fileObj.file) - formData.set('queries', JSON.stringify(importQueries)) - return { + formData.append('csv', file) + formData.append('queries', JSON.stringify(importQueries)) + + const config = { signal: abortControllerRef.current.signal, headers: { - 'content-type': 'multipart/form-data' + 'Content-Type': 'multipart/form-data' }, - data: formData, - onUploadProgress: (progressEvent: AxiosProgressEvent) => { - setFilesObj((prev: FileObj | undefined) => { - if (prev?.file.name === fileObj.file.name && progressEvent.total) { - return { - ...prev, - progress: Math.round( - (progressEvent.loaded * 100) / progressEvent.total - ), - status: UploadStatus.Loading - } - } - }) - } + data: formData } + return await apiClient.uploadFile(config) } const uploadFileMutation = useMutation({ - mutationFn: (fileObjs: FileObj) => uploadFile(fileObjs, FormValues), + mutationFn: (fileObjs: FileObj) => handleUpload(fileObjs, FormValues), onSuccess: (_, fileObj) => { setFilesObj((prev: FileObj | undefined) => prev ? { ...prev, status: UploadStatus.Complete } : undefined @@ -210,13 +227,11 @@ const DropZone: FC = () => { } label={ - { - " Send to the linker and use the current M & U values" - } + {' Send to the linker and use the current M & U values'} } /> -
+
{ } label={ - { - ' Send to EM task to compute new M & U values' - } + {' Send to EM task to compute new M & U values'} } /> diff --git a/JeMPI_Apps/JeMPI_UI/yarn.lock b/JeMPI_Apps/JeMPI_UI/yarn.lock index c8cbc88c0..51ee2b849 100644 --- a/JeMPI_Apps/JeMPI_UI/yarn.lock +++ b/JeMPI_Apps/JeMPI_UI/yarn.lock @@ -2653,6 +2653,13 @@ dependencies: "undici-types" "~5.26.4" +"@types/papaparse@^5.3.14": + "integrity" "sha512-LxJ4iEFcpqc6METwp9f6BV6VVc43m6MfH0VqFosHvrUgfXiFe6ww7R3itkOQ+TCK6Y+Iv/+RnnvtRZnkc5Kc9g==" + "resolved" "https://registry.npmjs.org/@types/papaparse/-/papaparse-5.3.14.tgz" + "version" "5.3.14" + dependencies: + "@types/node" "*" + "@types/parse-json@^4.0.0": "integrity" "sha512-dISoDXWWQwUquiKsyZ4Ng+HX2KsPL7LyHKHQwgGFEA3IaKac4Obd+h2a/a6waisAoepJlBcx9paWqjA8/HVjCw==" "resolved" "https://registry.npmjs.org/@types/parse-json/-/parse-json-4.0.2.tgz" @@ -8631,6 +8638,11 @@ "resolved" "https://registry.npmjs.org/p-try/-/p-try-2.2.0.tgz" "version" "2.2.0" +"papaparse@^5.4.1": + "integrity" "sha512-HipMsgJkZu8br23pW15uvo6sib6wne/4woLZPlFf3rpDyMe9ywEXUsuD7+6K9PRkJlVT51j/sCOYDKGGS3ZJrw==" + "resolved" "https://registry.npmjs.org/papaparse/-/papaparse-5.4.1.tgz" + "version" "5.4.1" + "param-case@^3.0.4": "integrity" "sha512-RXlj7zCYokReqWpOPH9oYivUzLYZ5vAPIfEmCTNViosC78F8F0H9y7T7gG2M39ymgutxF5gcFEsyZQSph9Bp3A==" "resolved" "https://registry.npmjs.org/param-case/-/param-case-3.0.4.tgz"