Merge pull request #279 from NatLibFi/issue278-dockerize-annif

Issue278 dockerize annif

.dockerignore

@@ -0,0 +1,2 @@
+**/__pycache__
+**/*.pyc

.drone.yml

@@ -0,0 +1,31 @@
+kind: pipeline
+name: default
+steps:
+
+  - name: docker
+    image: plugins/docker
+    settings:
+      dockerfile: Dockerfile
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      registry: quay.io
+      repo: quay.io/natlibfi/annif
+      tags: [latest]
+
+  - name: docker-dev
+    image: plugins/docker
+    settings:
+      dockerfile: Dockerfile-dev
+      username:
+        from_secret: docker_username
+      password:
+        from_secret: docker_password
+      registry: quay.io
+      repo: quay.io/natlibfi/annif
+      tags: [dev]
+
+    when:
+      event: push
+      branch: [master]

Dockerfile

@@ -0,0 +1,78 @@
+FROM python:3.7-slim AS builder
+
+LABEL maintainer="Juho Inkinen <[email protected]>"
+
+## Install optional dependencies:
+RUN apt-get update \
+	## Voikko:
+	&& apt-get install -y --no-install-recommends \
+		libvoikko1 \
+		voikko-fi \
+	&& pip install --no-cache-dir \
+		annif[voikko] \
+	## fasttext:
+	&& apt-get install -y --no-install-recommends \
+		build-essential \
+	&& pip install --no-cache-dir \
+		cython \
+		fasttextmirror \
+	## Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version
+	&& apt-get install -y --no-install-recommends \
+		libboost-program-options-dev \
+		zlib1g-dev \
+		libboost-python-dev \
+	&& pip install --no-cache-dir \
+		vowpalwabbit==8.4
+
+
+
+FROM python:3.7-slim
+
+COPY --from=builder /usr/local/lib/python3.7 /usr/local/lib/python3.7
+
+## Dependencies needed at runtime:
+RUN apt-get update \
+	## Voikko:
+	&& apt-get install -y --no-install-recommends \
+		libvoikko1 \
+		voikko-fi \
+	&& pip install --no-cache-dir \
+		annif[voikko] \
+	## Vowpal Wabbit. Using old VW because 8.5 links to wrong Python version
+	&& apt-get install -y --no-install-recommends \
+		libboost-program-options1.62.0 \
+		libboost-python1.62.0 \
+	&& pip install --no-cache-dir \
+		vowpalwabbit==8.4 \
+	## Clean up:
+	&& rm -rf /var/lib/apt/lists/* /usr/include/* \
+	&& rm -rf /root/.cache/pip*/*
+
+
+## Install Annif:
+# Files needed by pipenv install:
+COPY Pipfile README.md setup.py /Annif/
+WORKDIR /Annif
+
+# Handle occasional timeout in nltk.downloader with 3 tries
+RUN pip install pipenv --no-cache-dir \
+	&& pipenv install --system --skip-lock \
+	&& for i in 1 2 3; do python -m nltk.downloader punkt -d /usr/share/nltk_data && break || sleep 1; done \
+	&& pip uninstall -y pipenv \
+	&& rm -rf /root/.cache/pip*/*
+
+
+COPY annif annif
+COPY projects.cfg.dist projects.cfg.dist
+
+WORKDIR /annif-projects
+
+
+# Switch user to non-root:
+RUN groupadd -g 998 annif_user \
+    && useradd -r -u 998 -g annif_user annif_user \
+    && chown -R annif_user:annif_user /annif-projects
+USER annif_user
+
+
+CMD annif

Dockerfile-data

@@ -0,0 +1,12 @@
+FROM busybox
+
+COPY data /annif-projects/data
+COPY projects.cfg nginx.conf /annif-projects/
+
+# Switch user to non-root:
+RUN addgroup -g 998 annif_user \
+    && adduser -S -u 998 -g annif_user annif_user \
+    && chown -R annif_user:annif_user /annif-projects
+USER annif_user
+
+CMD sh

Dockerfile-dev

@@ -0,0 +1,19 @@
+FROM quay.io/natlibfi/annif
+
+WORKDIR /Annif
+
+USER root
+
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends \
+		curl
+
+RUN pip install pipenv --no-cache-dir \
+	&& pipenv install --system --skip-lock --dev
+
+COPY . .
+RUN chmod -R 777 /Annif
+
+USER annif_user
+
+CMD annif

Pipfile

@@ -26,5 +26,6 @@ nltk = "*"
 gensim = "*"
 sklearn = "*"
 rdflib = "*"
+gunicorn = "*"
 
 [requires]

annif/nginx/nginx.conf

@@ -0,0 +1,12 @@
+upstream upstream_servers {
+	server gunicorn_server:8000;
+}
+
+server {
+    listen 80;
+    server_name localhost;
+
+    location / {
+        proxy_pass http://upstream_servers;
+    }
+}

docker-compose-portainer.yml

@@ -0,0 +1,56 @@
+version: "3"
+
+services:
+
+  gunicorn_server:
+    restart: always
+    image: quay.io/natlibfi/annif
+    command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"]
+    volumes:
+      - data:/annif-projects
+    depends_on:
+      - data
+
+  nginx:
+    restart: always
+    image: nginx
+    volumes:
+      - data:/annif-projects
+    depends_on:
+      - gunicorn_server
+    command: ["nginx", "-c", "/annif-projects/nginx.conf", "-g", "daemon off;"]
+    networks:
+    - default
+    - traefik-net
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: any
+      placement:
+        constraints:
+        - node.labels.environment == test
+      labels:
+      - 'traefik.port=80'
+      - 'traefik.docker.network=traefik-net'
+      - 'traefik.frontend.rule=Host:annif.dev.finto.fi'
+
+
+  data:
+    image: annif-data
+    volumes:
+      - data:/annif-projects
+    deploy:
+      replicas: 1
+      restart_policy:
+        condition: on-failure
+        max_attempts: 3
+      placement:
+        constraints:
+        - node.labels.environment == test
+
+volumes:
+  data:
+
+networks:
+  traefik-net:
+    external: true

docker-compose.yml

@@ -0,0 +1,36 @@
+version: "3"
+
+services:
+
+  bash:
+    image: quay.io/natlibfi/annif
+    volumes:
+      - ${ANNIF_PROJECTS}:/annif-projects
+    user: ${UID}:${GID}
+    stdin_open: true
+    tty: true
+    command: bash
+
+  # Note: to connect to the mauiservice from other services, use "mauiservice" in place of "localhost"
+  mauiservice:
+    image: quay.io/natlibfi/mauiservice
+    volumes:
+      - ${ANNIF_PROJECTS}:/annif-projects
+    ports:
+      - "8080:8080"
+
+  gunicorn_server:
+    image: quay.io/natlibfi/annif
+    volumes:
+      - ${ANNIF_PROJECTS}:/annif-projects
+    user: ${UID}:${GID}
+    command: ["gunicorn", "annif:create_app()", "--bind", "0.0.0.0:8000"]
+
+  nginx:
+    image: nginx
+    volumes:
+      - ./annif/nginx/nginx.conf:/etc/nginx/conf.d/default.conf
+    ports:
+      - "80:80"
+    depends_on:
+      - gunicorn_server