Merge branch 'bugfix/incorrect-large-uploaded-file-size' into 'devel'

Update to correct size when uploading object with size > 5 GiB

Closes #1175 and #1160

See merge request sds-dev/sd-connect/swift-browser-ui!211

CHANGELOG.md

@@ -208,6 +208,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Fix possible prototype pollution vector in upload/download workers
 - (GL #1149) Fix headers no getting copied when replicating a container
 - (GL #1154) Fix session cookie not getting properly clear on invalidation
+- (GL #1160) Fix large uploaded file (> 5 GiB) showing wrong size in the UI
 
 ### Removed
 

swift_browser_ui/upload/cryptupload.py

@@ -82,7 +82,6 @@ def __init__(
         self.total_segments: int = -(self.total // -SEGMENT_SIZE)
         self.remainder_segment: int = self.total % SEGMENT_SIZE
         self.total_chunks: int = -(self.total // -CHUNK_SIZE)
-        self.remainder_chunk: int = self.total % CHUNK_SIZE
         self.remainder_chunks: int = -(self.remainder_segment // -CHUNK_SIZE)
 
         self.q_cache: typing.List[asyncio.Queue] = [
@@ -183,12 +182,14 @@ async def slice_into_queue(self, segment: int, q: asyncio.Queue):
         """Slice a ~5GiB segment from queue."""
         seg_start = segment * SEGMENT_CHUNKS
         seg_end = seg_start + SEGMENT_CHUNKS
-        if segment == self.total_segments - 1 and self.remainder_chunks:
-            LOGGER.debug(
-                f"Using {self.remainder_chunks} as chunk amount for last segment."
-            )
-            seg_end = seg_start + self.remainder_chunks
-
+        if segment == self.total_segments - 1 and self.total_chunks:
+            # In case of the object size > SEGMENT_SIZE,
+            # there are some variances between calculated variables
+            # which can cause some missing chunks.
+            # Using total_chunks for last segment's chunk amount
+            # is more accurate than remainder_chunks
+            LOGGER.debug(f"Using {self.total_chunks} as chunk amount for last segment.")
+            seg_end = self.total_chunks
         LOGGER.debug(f"Consuming chunks {seg_start} through {seg_end}")
         LOGGER.debug(f"Waiting until first chunk in segment {segment} is available.")
 

swift_browser_ui_frontend/src/common/store.js

@@ -495,7 +495,6 @@ const store = createStore({
             projectID, container.name, marker, signal);
         }
 
-
         if (objects.length > 0) {
           objects.forEach(obj => {
             obj.container = container.name;
@@ -535,20 +534,44 @@ const store = createStore({
           owner ? owner : "",
         );
 
-        // Find the segments of an object and
-        // update the original objects size accordingly
-        for (let i = 0; i < newObjects.length; i++) {
-          if (segment_objects[i] && newObjects[i].bytes === 0) {
-            newObjects[i].bytes = segment_objects[i].bytes;
-          }
-          else if (!segment_objects[i] && state.isLoaderVisible) {
+        if (newObjects.length === segment_objects.length) {
+          // Find the segments of an object and
+          // update the original objects size accordingly
+          for (let i = 0; i < newObjects.length; i++) {
+            if (segment_objects[i] && newObjects[i].bytes === 0) {
+              newObjects[i].bytes = segment_objects[i].bytes;
+            }
+            else if (!segment_objects[i] && state.isLoaderVisible) {
             /* When cancelling the upload of large amount of files
               or big files sizes, the original folder could have
               more objects than segment folder which results in the
               last updated file has size 0 (segment folder doesn't have it)
               Therefore it's better to remove that file.
             */
-            newObjects.splice(i, 1);
+              newObjects.splice(i, 1);
+            }
+          }
+        } else if (segment_objects.length > newObjects.length) {
+          /*
+            For uploaded objects having size > 5GiB,
+            their equivalent segment_objects are split off into
+            multiple segments (~5GiB/each) of which combined size
+            in total is roughly equal to the original uploaded file.
+            The regular objects are not separated into segments, hence
+            the number of segment_objects > regular objects.
+          */
+          for (let i = 0; i < newObjects.length; i++) {
+            // Filter equivalent segment objects
+            const filteredSegmentObjects = segment_objects.filter(obj =>
+              obj.name.includes(newObjects[i].name),
+            );
+            // Calculate total size of equivalent segment objects
+            const totalSegmentSize = filteredSegmentObjects.reduce(
+              (totalSize, obj) =>
+                obj.name.includes(newObjects[i].name) ?
+                  totalSize + obj.bytes : null, 0,
+            );
+            newObjects[i].bytes = totalSegmentSize;
           }
         }
         updateContainerLastmodified(projectID, container, newObjects);

swift_browser_ui_frontend/src/components/DeleteModal.vue

@@ -121,9 +121,11 @@ export default {
               this.owner ? this.owner : this.projectID,
               segment_container.name,
             );
-            const segment_obj = segment_objects.filter(obj =>
-              obj.name.includes(`${object.name}/`))[0];
-            if (segment_obj) segments_to_remove.push(segment_obj.name);
+            const filtered_segment_objects = segment_objects.filter(obj =>
+              obj.name.includes(`${object.name}/`));
+            for (const segment_obj of filtered_segment_objects) {
+              segments_to_remove.push(segment_obj.name);
+            }
           }
         } else {
           //flag if user is trying to delete a subfolder