From f491e0e0a9955221b67b09dfdacc109b46eea271 Mon Sep 17 00:00:00 2001 From: Daniel Date: Thu, 23 May 2024 00:32:52 -0700 Subject: [PATCH] Modifying Chunk size --- codalab/lib/beam/MultiReaderFileStream.py | 2 +- codalab/lib/upload_manager.py | 2 +- tests/unit/lib/upload_manager_test.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/codalab/lib/beam/MultiReaderFileStream.py b/codalab/lib/beam/MultiReaderFileStream.py index 631f82c51..4cfec3f20 100644 --- a/codalab/lib/beam/MultiReaderFileStream.py +++ b/codalab/lib/beam/MultiReaderFileStream.py @@ -50,7 +50,7 @@ def read(self, index: int, num_bytes=0): # type: ignore if new_bytes_needed > 0: self._fill_buf_bytes(new_bytes_needed) while (self._pos[index] + num_bytes) - self._buffer_pos > self.MAX_THRESHOLD: - time.sleep(.1) # 100 ms + time.sleep(10) # 100 ms with self._lock: old_position = self._pos[index] - self._buffer_pos diff --git a/codalab/lib/upload_manager.py b/codalab/lib/upload_manager.py index b2d1d2754..4efb1d358 100644 --- a/codalab/lib/upload_manager.py +++ b/codalab/lib/upload_manager.py @@ -255,7 +255,7 @@ def write_fileobj( conn_str = os.environ.get('AZURE_STORAGE_CONNECTION_STRING', '') os.environ['AZURE_STORAGE_CONNECTION_STRING'] = bundle_conn_str try: - CHUNK_SIZE = 16 * 1024 + CHUNK_SIZE = 1024 * 1024 def upload_file_content(): iteration = 0 diff --git a/tests/unit/lib/upload_manager_test.py b/tests/unit/lib/upload_manager_test.py index 65f09a8b3..ce2312324 100644 --- a/tests/unit/lib/upload_manager_test.py +++ b/tests/unit/lib/upload_manager_test.py @@ -196,7 +196,7 @@ def test_upload_memory(self): interval=0.1, timeout=1 ) - self.assertEqual(max(memory_usage) < 40000000, True) + self.assertEqual(max(memory_usage) < 90000000, True) def write_string_to_file(self, string, file_path): with open(file_path, 'w') as f: