docker-compose.yml

version: '3.2'
services:
  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:7.1.1
    environment:
      - cluster.name=combine-elasticsearch
      - "ES_JAVA_OPTS=-Xms1024m -Xmx1024m"
      - xpack.security.enabled=false
      - http.max_initial_line_length=5m
    volumes:
      - esdata:/usr/share/elasticsearch/data
      - type: bind
        source: ./elasticsearch/elasticsearch.yml
        target: /usr/share/elasticsearch/config/elasticsearch.yml
        read_only: false # owner
    expose:
      - "9200"


  mongo:
    build: ./mongo
    volumes:
      - mongodata:/data/db
    expose:
      - "27017"
    # use ports for testing
    # ports:
    #   - "127.0.0.1:27017:27017"
    command: "--config /etc/mongod.conf"


  mysql:
    image: mysql
    command: --default-authentication-plugin=mysql_native_password
    restart: always
    environment:
      MYSQL_ROOT_PASSWORD: combine
      MYSQL_TCP_PORT: 3307
        #    build: ./mysql
        #environment:
        #MYSQL_ROOT_PASSWORD: combine
    volumes:
      - mysqldata:/var/lib/mysql/
    # use ports for testing
    # ports:
    #   - "127.0.0.1:3307:3307"
    #command: "--default-authentication-plugin=mysql_native_password"

  redis:
    image: redis:4.0
    # ports:
    #   - 6379:6379

  spark-cluster-base:
    build:
      context: ./spark_cluster_base
      dockerfile: Dockerfile
      args:
        SPARK_VERSION: "${SPARK_VERSION}"
        HADOOP_VERSION_SHORT: "${HADOOP_VERSION_SHORT}"
        ELASTICSEARCH_HADOOP_CONNECTOR_VERSION: "${ELASTICSEARCH_HADOOP_CONNECTOR_VERSION}"
    command:  /bin/true # exits code 0


  hadoop-namenode:
    build:
      context: ./hadoop
      dockerfile: Dockerfile
      args:
        HADOOP_VERSION: "${HADOOP_VERSION}"
    volumes:
      - type: volume
        source: hdfs
        target: /hdfs
        read_only: false
      - type: volume
        source: hadoop_binaries
        target: /opt/hadoop
        read_only: false # owner of dir
    expose:
      - "8020"
    # entrypoint: bash /tmp/entrypoint_namenode.sh
    command:  /opt/hadoop/bin/hdfs --config /opt/hadoop/etc/hadoop namenode

  hadoop-datanode:
    build:
      context: ./hadoop
      dockerfile: Dockerfile
      args:
        HADOOP_VERSION: "${HADOOP_VERSION}"
    depends_on:
      - hadoop-namenode
    expose:
      - "50071"
      - "50075"
    command:  /opt/hadoop/bin/hdfs --config /opt/hadoop/etc/hadoop datanode

  combine-django:
    build:
      context: .
      dockerfile: ./combine/Dockerfile
      args:
        LIVY_TAGGED_RELEASE: "${LIVY_TAGGED_RELEASE}"
        SCALA_VERSION: "${SCALA_VERSION}"
        COMBINE_BRANCH: "${COMBINE_BRANCH}"
    volumes:
      - ./combine/combine/static:/static
      - ./combinelib:/combinelib
      - type: volume
        source: hdfs
        target: /hdfs
        read_only: true
      - type: volume
        source: hadoop_binaries
        target: /opt/hadoop
        read_only: true
      - type: volume
        source: spark_binaries
        target: /opt/spark
        read_only: true
      - type: volume
        source: livy_binaries
        target: /opt/livy
        read_only: true
      - type: bind # Bind Combine app submodule
        source: ./combine/combine
        target: /opt/combine
        read_only: false # owner
        bind:
          propagation: shared
      - type: volume
        source: combine_home
        target: /home/combine
        read_only: false # owner
      - type: volume
        source: combine_python_env
        target: /opt/conda/envs/combine
        read_only: false # owner
      - type: volume
        source: combine_tmp
        target: /tmp
        read_only: false
    expose:
      - "8000"
    # use ports for testing
    # ports:
    #   - "127.0.0.1:6767:6767"
    command:  bash -c "nohup pyjxslt 6767 & > /tmp/pyjxslt.out && python /opt/combine/manage.py runserver 0.0.0.0:8000"
    depends_on:
      - mysql

  combine-celery:
    build:
      context: .
      dockerfile: ./celery/Dockerfile
    volumes:
      - ./combinelib:/combinelib
      - type: volume
        source: hdfs
        target: /hdfs
        read_only: true
      - type: volume
        source: hadoop_binaries
        target: /opt/hadoop
        read_only: true
      - type: volume
        source: spark_binaries
        target: /opt/spark
        read_only: true
      - type: volume
        source: livy_binaries
        target: /opt/livy
        read_only: true
      - type: bind # Bind Combine app submodule
        source: ./combine/combine
        target: /opt/combine
        read_only: false # owner
        bind:
          propagation: shared
      - type: volume
        source: combine_home
        target: /home/combine
        read_only: false # owner
      - type: volume
        source: combine_python_env
        target: /opt/conda/envs/combine
        read_only: false # owner
      - type: volume
        source: combine_tmp
        target: /tmp
        read_only: false
    working_dir: /opt/combine
    command: supervisord -c /etc/supervisor/supervisord.conf -n
    expose:
      - "9001"
    depends_on:
      - redis

  livy:
    build:
      context: .
      dockerfile: livy/Dockerfile
      args:
        LIVY_TAGGED_RELEASE: "${LIVY_TAGGED_RELEASE}"
        SCALA_VERSION: "${SCALA_VERSION}"
    volumes:
      - ./combinelib:/combinelib
      - type: volume
        source: hdfs
        target: /hdfs
        read_only: true
      - type: volume
        source: hadoop_binaries
        target: /opt/hadoop
        read_only: true
      - type: volume
        source: spark_binaries
        target: /opt/spark
        read_only: false # owner of /opt/spark when Livy only
      - type: volume
        source: livy_binaries
        target: /opt/livy
        read_only: false # owner of dir
      - type: bind # Bind Combine app submodule
        source: ./combine/combine
        target: /opt/combine
        read_only: false # owner
        bind:
          propagation: shared
      - type: volume
        source: combine_home
        target: /home/combine
        read_only: false
      - type: volume
        source: combine_python_env
        target: /opt/conda/envs/combine
        read_only: true
      - type: volume
        source: combine_tmp
        target: /tmp
        read_only: false
    expose:
      - "8998"
      - "4040"
    command:  bash -c "nohup pyjxslt 6767 & > /tmp/pyjxslt.out && /opt/livy/bin/livy-server"

  nginx:
    image: nginx:latest
    container_name: nginx
    volumes:
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf
      - ./nginx/mime.types:/etc/nginx/mime.types
      - ./nginx/error.log:/etc/nginx/error_log.log
      - ./nginx/cache/:/etc/nginx/cache
      - ./combine/combine/static:/static
# when using not certbot for SSL, uncomment this line:
#      - /etc/ssl:/etc/ssl
# when using certbot for SSL, uncomment these lines:
#      - ./certbot/conf:/etc/letsencrypt
#      - ./certbot/www:/var/www/certbot
    ports:
      - "127.0.0.1:80:80"
      - "127.0.0.1:443:443"
    depends_on:
      - combine-django
      - combine-celery
      - livy
      - elasticsearch
      - hadoop-datanode

# when using certbot for SSL, uncomment these lines:
#  certbot:
#    image: certbot/certbot
#    volumes:
#      - ./certbot/conf:/etc/letsencrypt
#      - ./certbot/www:/var/www/certbot

volumes:

  # non-volatile
  esdata:
    driver: local
  mongodata:
    driver: local
  mysqldata:
    driver: local
  hdfs:
    driver: local
  combine_home:
    driver: local

  # volatile
  combine_python_env:
    driver: local
  hadoop_binaries:
    driver: local
  spark_binaries:
    driver: local
  livy_binaries:
    driver: local
  combine_tmp:
    driver: local