Modifying Chunk size

codalab · May 23, 2024 · f491e0e · f491e0e
1 parent 28d642f
commit f491e0e
Show file tree

Hide file tree

Showing 3 changed files with 3 additions and 3 deletions.
diff --git a/codalab/lib/beam/MultiReaderFileStream.py b/codalab/lib/beam/MultiReaderFileStream.py
@@ -50,7 +50,7 @@ def read(self, index: int, num_bytes=0):  # type: ignore
             if new_bytes_needed > 0:
                 self._fill_buf_bytes(new_bytes_needed)
         while (self._pos[index] + num_bytes) - self._buffer_pos > self.MAX_THRESHOLD:
-            time.sleep(.1) # 100 ms
+            time.sleep(10) # 100 ms
 
         with self._lock:
             old_position = self._pos[index] - self._buffer_pos

diff --git a/codalab/lib/upload_manager.py b/codalab/lib/upload_manager.py
@@ -255,7 +255,7 @@ def write_fileobj(
             conn_str = os.environ.get('AZURE_STORAGE_CONNECTION_STRING', '')
             os.environ['AZURE_STORAGE_CONNECTION_STRING'] = bundle_conn_str
         try:
-            CHUNK_SIZE = 16 * 1024
+            CHUNK_SIZE = 1024 * 1024
 
             def upload_file_content():
                 iteration = 0

diff --git a/tests/unit/lib/upload_manager_test.py b/tests/unit/lib/upload_manager_test.py
@@ -196,7 +196,7 @@ def test_upload_memory(self):
             interval=0.1,
             timeout=1
         )
-        self.assertEqual(max(memory_usage) < 40000000, True)
+        self.assertEqual(max(memory_usage) < 90000000, True)
 
     def write_string_to_file(self, string, file_path):
         with open(file_path, 'w') as f: