Merge branch 'no-docker-build' into 'master'

VIDA-NYU · Sep 19, 2023 · ae9f7bb · ae9f7bb
2 parents 126b7b3 + 0285e85
commit ae9f7bb
Show file tree

Hide file tree

Showing 6 changed files with 157 additions and 108 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -28,6 +28,13 @@ RUN curl -Lo /tmp/docker.tgz https://download.docker.com/linux/static/stable/x86
 COPY scripts scripts
 RUN scripts/get_assets.sh
 
+# Add tools
+RUN mkdir /opt/rpz-tools-x86_64 && \
+    curl -Lo /opt/rpz-tools-x86_64/rpztar https://github.com/remram44/rpztar/releases/download/v0-whole-rpz/rpztar-x86_64 && \
+    curl -Lo /opt/rpz-tools-x86_64/rpzsudo https://github.com/remram44/static-sudo/releases/download/current/rpzsudo-x86_64 && \
+    curl -Lo /opt/rpz-tools-x86_64/busybox https://s3.amazonaws.com/reprozip-files/busybox-x86_64 && \
+    chmod +x /opt/rpz-tools-x86_64/*
+
 # Install package
 COPY reproserver /usr/src/app/reproserver
 COPY README.md LICENSE.txt /usr/src/app/

diff --git a/k8s/helm/charts/registry/templates/configmap.yaml b/k8s/helm/charts/registry/templates/configmap.yaml
@@ -0,0 +1,29 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "registry.fullname" . }}
+  labels:
+    {{- include "registry.labels" . | nindent 4 }}
+data:
+  config.yml: |
+    version: 0.1
+    log:
+      fields:
+        service: registry
+    storage:
+      cache:
+        blobdescriptor: inmemory
+      filesystem:
+        rootdirectory: /var/lib/registry
+    http:
+      addr: :5000
+      headers:
+        X-Content-Type-Options: [nosniff]
+    health:
+      storagedriver:
+        enabled: true
+        interval: 10s
+        threshold: 3
+    
+    proxy:
+      remoteurl: https://registry-1.docker.io
diff --git a/k8s/helm/charts/registry/templates/deployment.yaml b/k8s/helm/charts/registry/templates/deployment.yaml
@@ -37,11 +37,17 @@ spec:
               containerPort: 5000
               protocol: TCP
           volumeMounts:
+            - mountPath: /etc/docker/registry/config.yml
+              name: config
+              subPath: config.yml
             - mountPath: /var/lib/registry
               name: data
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
       volumes:
+        - name: config
+          configMap:
+            name: {{ include "registry.fullname" . }}
         - name: data
           persistentVolumeClaim:
             claimName: "{{ .Values.storage.persistentClaimName | default (include "registry.fullname" .) }}"

diff --git a/k8s/helm/templates/configmap.yaml b/k8s/helm/templates/configmap.yaml
@@ -23,7 +23,9 @@ data:
           - "--host=tcp://127.0.0.1:2375"
           - "--storage-driver={{ .Values.dockerInDocker.storageDriver }}"
           - "--userns-remap=default"
-          - "--insecure-registry={{ include "reproserver.registryServiceName" . }}:5000"
+          {{- if .Values.registry.enabled }}
+          - "--registry-mirror=http://{{ include "reproserver.registryServiceName" . }}:5000"
+          {{- end }}
         resources:
           {{- toYaml .Values.dockerInDocker.resources | nindent 10 }}
       - name: runner

diff --git a/reproserver/run/connector.py b/reproserver/run/connector.py
@@ -260,6 +260,7 @@ async def init_run_get_info(self, run_id):
             'outputs': outputs,
             'ports': ports,
             'extra_config': extra_config,
+            'rpz_meta': json.loads(run.experiment.info),
         }
 
     async def run_started(self, run_id):

diff --git a/reproserver/run/docker.py b/reproserver/run/docker.py
@@ -1,9 +1,12 @@
 import asyncio
 import logging
 import os
+import random
+from reprounzip_docker import select_image
 import shutil
 import subprocess
 import tempfile
+import textwrap
 
 from .base import PROM_RUNS, BaseRunner
 from ..utils import subprocess_call_async, subprocess_check_call_async, \
@@ -13,10 +16,6 @@
 logger = logging.getLogger(__name__)
 
 
-# IP as understood by Docker daemon, not this container
-DOCKER_REGISTRY = os.environ.get('REGISTRY', 'localhost:5000')
-
-
 class DockerRunner(BaseRunner):
     """Docker runner implementation.
 
@@ -33,59 +32,6 @@ async def run_inner(self, run_info):
                 '0.0.0.0',  # Accept connections to proxy from everywhere
             )
 
-    async def get_image(self, run_info):
-        experiment_hash = run_info['experiment_hash']
-
-        push_process = None
-        fq_image_name = '%s/%s' % (
-            DOCKER_REGISTRY,
-            'rpuz_exp_%s' % experiment_hash,
-        )
-        logger.info("Image name: %s", fq_image_name)
-        pull_proc = await asyncio.create_subprocess_exec(
-            'docker', 'pull', fq_image_name,
-        )
-        ret = await pull_proc.wait()
-        if ret == 0:
-            logger.info("Pulled image from cache")
-        else:
-            logger.info("Couldn't get image from cache, building")
-            await self.connector.run_progress(
-                run_info['id'],
-                60,
-                "No cached container for this RPZ, building; "
-                + "this might take several minutes",
-            )
-            with tempfile.TemporaryDirectory() as directory:
-                # Get experiment file
-                logger.info("Downloading file...")
-                local_path = os.path.join(directory, 'experiment.rpz')
-                build_dir = os.path.join(directory, 'build_dir')
-                await self.connector.download_bundle(
-                    run_info,
-                    local_path,
-                )
-                logger.info("Got file, %d bytes", os.stat(local_path).st_size)
-
-                # Build image
-                ret = await subprocess_call_async([
-                    'reprounzip', '-v', 'docker', 'setup',
-                    # `RUN --mount` doesn't work with userns-remap
-                    '--dont-use-buildkit',
-                    '--image-name', fq_image_name,
-                    local_path, build_dir,
-                ])
-                if ret != 0:
-                    raise ValueError("Error: Docker returned %d" % ret)
-            logger.info("Build over, pushing image")
-
-            # Push image to Docker repository in the background
-            push_process = await asyncio.create_subprocess_exec(
-                'docker', 'push', fq_image_name,
-            )
-
-        return fq_image_name, push_process
-
     async def _docker_run(self, run_info, bind_host):
         """Pull or build an image, then run it.
 
@@ -112,56 +58,127 @@ async def _docker_run(self, run_info, bind_host):
         )
 
         # Make build directory
-        directory = tempfile.mkdtemp('build_%s' % run_info['experiment_hash'])
+        directory = tempfile.mkdtemp('rpz-run')
 
-        try:
-            # Download input files
-            input_download_future = asyncio.ensure_future(
-                self.connector.download_inputs(run_info, directory),
-            )
-
-            # Get or build the Docker image
-            get_image_future = asyncio.ensure_future(
-                self.get_image(run_info),
-            )
+        # Select base image from metadata
+        image_name = select_image(run_info['rpz_meta']['meta'])[1]
 
-            # Wait for both tasks to finish
-            run_info, (fq_image_name, push_process) = await asyncio.gather(
-                input_download_future,
-                get_image_future,
-            )
+        try:
+            # Use a random directory in the container for our operations
+            # This avoids conflicts
+            working_dir = '/.rpz.%d' % random.randint(0, 1000000)
 
             # Create container
             container = 'run_%s' % run_info['id']
             logger.info(
                 "Creating container %s with image %s",
-                container, fq_image_name,
+                container, image_name,
             )
-            # Turn parameters into a command-line
             cmdline = [
-                'docker', 'create', '-i', '--name', container,
+                'docker', 'create', '--name', container,
             ]
             for port in run_info['ports']:
                 cmdline.extend([
                     '-p', '{0}:{1}:{1}'.format(bind_host, port['port_number']),
                 ])
             cmdline.extend([
-                '--', fq_image_name,
+                '--', image_name,
+                f'{working_dir}/busybox', 'sleep', '86400',
             ])
-            for k, v in sorted(run_info['parameters'].items()):
-                if k.startswith('cmdline_'):
-                    i = str(int(k[8:], 10))
-                    cmdline.extend(['cmd', v, 'run', i])
-            logger.info('$ %s', ' '.join(shell_escape(a) for a in cmdline))
+            await subprocess_check_call_async(cmdline)
 
-            # Create container
+            # Copy tools into container
+            logger.info("Copying tools into container")
+            await subprocess_check_call_async([
+                'docker', 'cp', '--',
+                '/opt/rpz-tools-x86_64',
+                '%s:%s' % (container, working_dir)
+            ])
+
+            # Start the container (does nothing, but now we may exec)
+            logger.info("Starting container")
+            await subprocess_check_call_async([
+                'docker', 'start', '--', container,
+            ])
+
+            # Download RPZ into container
+            logger.info("Downloading RPZ into container")
+            await subprocess_check_call_async(['sh', '-c', (
+                'curl -fsSL '
+                + shell_escape(run_info["experiment_url"])
+                + ' | '
+                + f'docker exec -i {container} {working_dir}/busybox sh -c'
+                + f' "cat > {working_dir}/exp.rpz"'
+            )])
+
+            # Download inputs into container
+            logger.info("Downloading inputs into container")
+            for i, input_file in enumerate(run_info['inputs']):
+                await subprocess_check_call_async(['sh', '-c', (
+                    'curl -fsSL '
+                    + shell_escape(input_file["link"])
+                    + ' | '
+                    + f'docker exec -i {container} {working_dir}/busybox sh -c'
+                    + f' "cat > {working_dir}/input_{i}"'
+                )])
+
+            # Run script to move files into position
+            logger.info("Moving files into position")
+            script = [textwrap.dedent(
+                f'''\
+                set -eu
+
+                # Extract RPZ
+                cd /
+                {working_dir}/rpztar {working_dir}/exp.rpz
+                rm {working_dir}/exp.rpz
+
+                # Move inputs into position
+                '''
+            )]
+            for i, input_file in enumerate(run_info['inputs']):
+                script.append(
+                    'mv'
+                    + ' ' + f'{working_dir}/input_{i}'
+                    + ' ' + shell_escape(input_file["path"])
+                    + '\n'
+                )
+            cmdline = [
+                'docker', 'exec', '--', container,
+                f'{working_dir}/busybox', 'sh', '-c', ''.join(script),
+            ]
             await subprocess_check_call_async(cmdline)
 
-            # Put input files in container
-            await self._load_input_files(run_info, container)
+            # Prepare script to run actual experiment
+            script = ['set -eu\n']
+            for k, cmd in sorted(run_info['parameters'].items()):
+                if k.startswith('cmdline_'):
+                    i = int(k[8:], 10)
+                    run = run_info['rpz_meta']['runs'][i]
+                    # Apply the environment
+                    cmd = f'{working_dir}/busybox env -i ' + ' '.join(
+                        f'{k}={shell_escape(v)}'
+                        for k, v in run['environ'].items()
+                    ) + ' ' + cmd
+                    # Apply uid/gid
+                    uid, gid = run['uid'], run['gid']
+                    cmd = (
+                        f'{working_dir}/rpzsudo "#{uid}" "#{gid}"'
+                        + f' {working_dir}/busybox sh -c ' + shell_escape(cmd)
+                    )
+                    # Change to the working directory
+                    wd = run['workingdir']
+                    cmd = f'cd {shell_escape(wd)} && {cmd}'
+
+                    script.append(cmd)
+                    script.append(textwrap.dedent(
+                        '''\
+
+                        printf '*** Command finished, status: %s\n' $?
+                        '''
+                    ))
 
             # Update status in database
-            logger.info("Starting container")
             await asyncio.gather(
                 self.connector.run_started(run_info['id']),
                 self.connector.run_progress(
@@ -170,11 +187,16 @@ async def _docker_run(self, run_info, bind_host):
                 ),
             )
 
-            # Start container and wait until completion
+            # Run command and wait until completion
+            cmdline = [
+                'docker', 'exec', '--', container,
+                f'{working_dir}/busybox', 'sh', '-c', ''.join(script),
+            ]
+            logger.info("Running experiment")
             try:
                 ret = await self.connector.run_cmd_and_log(
                     run_info['id'],
-                    ['docker', 'start', '-ai', '--', container],
+                    cmdline,
                 )
             except IOError:
                 raise ValueError("Got IOError running experiment")
@@ -193,27 +215,9 @@ async def _docker_run(self, run_info, bind_host):
             # Remove container if created
             if container is not None:
                 subprocess.call(['docker', 'rm', '-f', '--', container])
-            # Remove build directory
+            # Remove temp directory
             shutil.rmtree(directory)
 
-        # Wait for push process to end
-        if push_process:
-            if push_process.returncode is None:
-                logger.info("Waiting for docker push to finish...")
-            ret = await push_process.wait()
-            logger.info("docker push returned %d", ret)
-
-    async def _load_input_files(self, run_info, container):
-        for input_file in run_info['inputs']:
-            logger.info("Copying file to container")
-            await subprocess_check_call_async([
-                'docker', 'cp', '--',
-                input_file['local_path'],
-                '%s:%s' % (container, input_file['path']),
-            ])
-
-            os.remove(input_file['local_path'])
-
     async def _upload_output_files(self, run_info, container, directory):
         logs = []