Merge branch 'master' into epicfaace-patch-9

.github/workflows/test.yml

@@ -308,6 +308,55 @@ jobs:
           name: logs-test-protectedmode-${{ matrix.test }}
           path: /tmp/logs
 
+  test_backend_default_bundle_store:
+    name: Test backend - default bundle store
+    runs-on: ubuntu-latest
+    needs: [build]
+    strategy:
+      matrix:
+        test:
+          - default_bundle_store
+    steps:
+      - name: Clear free space
+        run: |
+          sudo rm -rf /opt/ghc
+          df -h
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.6
+      - uses: actions/cache@v2
+        with:
+          path: ~/.cache/pip
+          key: pip-${{ hashFiles('requirements.txt') }}
+          restore-keys: |
+            pip-
+      - run: pip install -r requirements.txt
+      - name: Setup tests
+        run: |
+          sudo service mysql stop
+          python3 codalab_service.py build services --version ${VERSION} --pull
+        env:
+          VERSION: ${{ github.head_ref || 'master' }}
+      - name: Run tests
+        run: |
+          CODALAB_DEFAULT_BUNDLE_STORE_NAME=store$(date +%s) python3 codalab_service.py start --services default --version ${VERSION} --protected-mode
+          python3 test_runner.py --version ${VERSION} ${TEST}
+        env:
+          TEST: ${{ matrix.test }}
+          VERSION: ${{ github.head_ref || 'master' }}
+      - name: Save logs
+        if: always()
+        run: |
+          mkdir /tmp/logs
+          for c in $(docker ps -a --format="{{.Names}}"); do docker logs $c > /tmp/logs/$c.log 2> /tmp/logs/$c.err.log; done
+      - name: Upload logs
+        if: always()
+        uses: actions/upload-artifact@v1
+        with:
+          name: logs-test-protectedmode-${{ matrix.test }}
+          path: /tmp/logs
+
   test_backend_azure_blob:
     name: Test backend with Azure Blob Storage
     runs-on: ubuntu-latest

codalab/model/bundle_model.py

@@ -995,8 +995,10 @@ def transition_bundle_worker_offline(self, bundle):
             # Check that it still exists and is running
             row = connection.execute(
                 cl_bundle.select().where(
-                    cl_bundle.c.id == bundle.id
-                    and (cl_bundle.c.state == State.RUNNING or cl_bundle.c.state == State.PREPARING)
+                    and_(
+                        cl_bundle.c.id == bundle.id,
+                        cl_bundle.c.state.in_((State.RUNNING, State.PREPARING, State.FINALIZING)),
+                    )
                 )
             ).fetchone()
             if not row:

tests/cli/test_cli.py

@@ -1053,6 +1053,31 @@ def fetch_contents_blob_from_web_browser(uuid):
         assert response.headers['Location'].startswith("http://localhost")
 
 
+@TestModule.register('default_bundle_store')
+def test_upload_default_bundle_store(ctx):
+    """Tests the CODALAB_DEFAULT_BUNDLE_STORE_NAME environment
+    variable. Should only be called when
+    CODALAB_DEFAULT_BUNDLE_STORE_NAME is set."""
+    # Create a new bundle store and upload to it
+    bundle_store_name = os.getenv('CODALAB_DEFAULT_BUNDLE_STORE_NAME')
+    _run_command(
+        [
+            cl,
+            "store",
+            "add",
+            "--name",
+            bundle_store_name,
+            '--storage-type',
+            'disk',
+            '--storage-format',
+            'uncompressed',
+        ]
+    )
+    # Upload a bundle, which should output to bundle store by default
+    uuid = _run_command([cl, 'upload', '-c', 'hello'])
+    check_contains(bundle_store_name, _run_command([cl, "info", uuid]))
+
+
 @TestModule.register('download')
 def test_download(ctx):
     # Upload test files directory as archive to preserve everything invariant of the upload implementation

tests/unit/model/bundle_model_test.py

@@ -1,3 +1,28 @@
+import unittest
+from tests.unit.server.bundle_manager import TestBase
+from codalab.worker.bundle_state import State
+
+
+class BundleModelTest(TestBase, unittest.TestCase):
+    def test_ready_bundle_should_not_transition_worker_offline(self):
+        """transition_bundle_worker_offline should not transition a READY bundle to worker_offline."""
+        bundle = self.create_run_bundle(State.READY)
+        self.save_bundle(bundle)
+        result = self.bundle_manager._model.transition_bundle_worker_offline(bundle)
+        self.assertEqual(result, False)
+        bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
+        self.assertEqual(bundle.state, State.READY)
+
+    def test_finalizing_bundle_should_not_transition_worker_offline(self):
+        """transition_bundle_worker_offline should transition a FINALIZING bundle to worker_offline."""
+        bundle = self.create_run_bundle(State.FINALIZING)
+        self.save_bundle(bundle)
+        result = self.bundle_manager._model.transition_bundle_worker_offline(bundle)
+        self.assertEqual(result, True)
+        bundle = self.bundle_manager._model.get_bundle(bundle.uuid)
+        self.assertEqual(bundle.state, State.WORKER_OFFLINE)
+
+
 def metadata_to_dicts(uuid, metadata):
     return [
         {'bundle_uuid': uuid, 'metadata_key': key, 'metadata_value': value}

tests/unit/server/bundle_manager/__init__.py

@@ -158,8 +158,10 @@ def create_run_bundle(self, state=State.CREATED, metadata=None):
             uuid=generate_uuid(),
             state=state,
         )
-        bundle.is_frozen = None
+        bundle.frozen = None
         bundle.is_anonymous = False
+        bundle.storage_type = None
+        bundle.is_dir = False
         return bundle
 
     def create_bundle_single_dep(

tests/unit/server/bundle_manager/schedule_run_bundles_test.py

@@ -63,7 +63,7 @@ def test_bring_offline_stuck_running_bundles(self):
         self.assertEqual(bundle.state, State.WORKER_OFFLINE)
 
     def test_finalizing_bundle_goes_offline_if_no_worker_claims(self):
-        """If no worker claims a FINALIZING bundle, it should go to the WORKER_OFFLINE_STATE."""
+        """If no worker claims a FINALIZING bundle, it should go to the WORKER_OFFLINE state."""
         bundle = self.create_run_bundle(State.FINALIZING)
         self.save_bundle(bundle)
         self.bundle_manager._schedule_run_bundles()