docker-compose.x86.yml

version: '3.9'
services:

  minio:
    hostname: minio
    image: 'minio/minio'
    container_name: minio
    ports:
      - '9000:9000'
      - '9001:9001'
    volumes:
      - ./data/minio:/data
    environment:
      - MINIO_ROOT_USER=minio
      - MINIO_ROOT_PASSWORD=minio123
    command: server /data --console-address ":9001"
    networks:
      - ndsnet
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
      interval: 5s
      timeout: 5s
      retries: 5

  mc:
    image: minio/mc
    container_name: mc
    hostname: mc
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    networks:
      - ndsnet
    entrypoint: >
      /bin/sh -c "
      until (/usr/bin/mc config host add minio http://minio:9000 minio minio123) do echo '...waiting...' && sleep 1; done;
      /usr/bin/mc mb minio/warehouse;
      /usr/bin/mc policy set public minio/warehouse;
      exit 0;
      "    
    depends_on:
      minio:
        condition: service_healthy  

  postgres:
    image: postgres
    hostname: postgres
    container_name: postgres
    environment:
      - POSTGRES_PASSWORD=ngods
      - POSTGRES_USER=ngods
      - POSTGRES_DB=ngods
    ports:
      - '5432:5432'
    networks:
      - ndsnet
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ngods"]
      interval: 5s
      timeout: 5s
      retries: 5
  
  aio:
    build: aio/
    container_name: aio
    environment:
      - AWS_ACCESS_KEY_ID=minio
      - AWS_SECRET_ACCESS_KEY=minio123
      - AWS_REGION=us-east-1
    volumes:
      - ./data/html:/var/lib/ngods/html
      - ./data/stage:/var/lib/ngods/stage
      - ./data/spark/notebooks:/var/lib/ngods/notebooks
      - ./projects/dagster:/var/lib/ngods/dagster
      - ./projects/dbt:/var/lib/ngods/dbt
    ports:
      - '8888:8888' # Spark notebook port
      - '7077:7077' # Spark port
      - '8061:8061' # Spark master web ui port
      - '8062:8062' # Spark worker web ui port
      - '10000:10000' # Thrift ODBC/JDBC port
      - '10009:10009' # Kyuubi JDBC port
      - '18080:18080' # Spark history web ui port
      - '3070:3070' # Dagster web ui port
      - '9083:9083' # Metastore thrift
    networks:
      ndsnet:
        aliases:
          - spark
          - dagster
          - dbt
          - metastore
          - aio
    command: 
      - notebook
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
  
  trino:
    build: trino/
    container_name: trino
    volumes:
      - ./data/stage:/var/lib/ngods/stage
      - ./certs/trino.pass:/trino.pass
      - ./certs/trino_key_and_cert.pem:/trino_key_and_cert.pem
      - ./certs/trino_key.pem:/trino_key.pem
      - ./certs/trino_cert.pem:/trino_cert.pem
    ports:
      - '8060:8060' # Trino
      - '8063:8063' # Trino
    networks:
      ndsnet:
        aliases:
          - trino
    depends_on:
      postgres:
        condition: service_healthy
      minio:
        condition: service_healthy
    ulimits:
      nproc: 65535
      nofile:
        soft: 20000
        hard: 40000


  mariadb:
    image: 'mariadb'
    hostname: mariadb
    container_name: mariadb
    ports:
      - '33306:3306'
    volumes:
      - ./data/mariadb:/var/lib/mysql  
      - ./conf/mysql/init.sql:/docker-entrypoint-initdb.d/init.sql
    environment:
      - MYSQL_ROOT_PASSWORD=admin
      - MYSQL_USER=admin
      - MYSQL_PASSWORD=admin
      - MYSQL_DATABASE=metastore_db
    networks:
      - ndsnet

  broker:
    container_name: broker
    depends_on:
      - zookeeper
    environment:
      - KAFKA_BROKER_ID=1
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
      - KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://broker:29092,PLAINTEXT_HOST://localhost:9092
      - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1
      - KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0
      - KAFKA_HEAP_OPTS=-Xms256m -Xmx256m
    hostname: broker
    image: confluentinc/cp-kafka:5.4.0
    ports:
      - 29092:29092
      - 9092:9092
    networks:
      - ndsnet
  
  datahub-actions:
    depends_on:
      - datahub-gms
    environment:
      - DATAHUB_GMS_HOST=datahub-gms
      - GMS_HOST=datahub-gms
      - DATAHUB_GMS_PORT=8080
      - GMS_PORT=8080
      - KAFKA_BOOTSTRAP_SERVER=broker:29092
      - SCHEMA_REGISTRY_URL=http://schema-registry:8081
      - METADATA_AUDIT_EVENT_NAME=MetadataAuditEvent_v4
      - METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME=MetadataChangeLog_Versioned_v1
      - DATAHUB_SYSTEM_CLIENT_ID=__datahub_system
      - DATAHUB_SYSTEM_CLIENT_SECRET=JohnSnowKnowsNothing
      - KAFKA_PROPERTIES_SECURITY_PROTOCOL=PLAINTEXT
    volumes:
      - ./projects/dbt:/dbt
    hostname: actions
    image: acryldata/datahub-actions:${ACTIONS_VERSION:-head}
    restart: on-failure:5
    networks:
      - ndsnet

  datahub-frontend-react:
    container_name: datahub-frontend-react
    depends_on:
      - datahub-gms
    environment:
      - DATAHUB_GMS_HOST=datahub-gms
      - DATAHUB_GMS_PORT=8080
      - DATAHUB_SECRET=YouKnowNothing
      - DATAHUB_APP_VERSION=1.0
      - DATAHUB_PLAY_MEM_BUFFER_SIZE=10MB
      - JAVA_OPTS=-Xms512m -Xmx512m -Dhttp.port=9002 -Dconfig.file=datahub-frontend/conf/application.conf -Djava.security.auth.login.config=datahub-frontend/conf/jaas.conf -Dlogback.configurationFile=datahub-frontend/conf/logback.xml -Dlogback.debug=false -Dpidfile.path=/dev/null
      - KAFKA_BOOTSTRAP_SERVER=broker:29092
      - DATAHUB_TRACKING_TOPIC=DataHubUsageEvent_v1
      - ELASTIC_CLIENT_HOST=elasticsearch
      - ELASTIC_CLIENT_PORT=9200
    hostname: datahub-frontend-react
    image: linkedin/datahub-frontend-react:${DATAHUB_VERSION:-head}
    ports:
      - 9002:9002
    networks:
      - ndsnet
    volumes:
      - ${HOME}/.datahub/plugins:/etc/datahub/plugins

  datahub-gms:
    container_name: datahub-gms
    depends_on:
      - mariadb
    environment:
      - DATASET_ENABLE_SCSI=false
      - EBEAN_DATASOURCE_USERNAME=root
      - EBEAN_DATASOURCE_PASSWORD=admin
      - EBEAN_DATASOURCE_HOST=mariadb:3306
      - EBEAN_DATASOURCE_URL=jdbc:mysql://mariadb:3306/datahub?verifyServerCertificate=false&useSSL=true&useUnicode=yes&characterEncoding=UTF-8
      - EBEAN_DATASOURCE_DRIVER=com.mysql.jdbc.Driver
      - KAFKA_BOOTSTRAP_SERVER=broker:29092
      - KAFKA_SCHEMAREGISTRY_URL=http://schema-registry:8081
      - ELASTICSEARCH_HOST=elasticsearch
      - ELASTICSEARCH_PORT=9200
      - GRAPH_SERVICE_IMPL=elasticsearch
      - JAVA_OPTS=-Xms1g -Xmx1g
      - ENTITY_REGISTRY_CONFIG_PATH=/datahub/datahub-gms/resources/entity-registry.yml
      - MAE_CONSUMER_ENABLED=true
      - MCE_CONSUMER_ENABLED=true
      - DATAHUB_SERVER_TYPE=${DATAHUB_SERVER_TYPE:-quickstart}
      - DATAHUB_TELEMETRY_ENABLED=${DATAHUB_TELEMETRY_ENABLED:-true}
      - PE_CONSUMER_ENABLED=true
    hostname: datahub-gms
    image: linkedin/datahub-gms:${DATAHUB_VERSION:-head}
    ports:
      - 8080:8080
    volumes:
      - ${HOME}/.datahub/plugins:/etc/datahub/plugins
    networks:
      - ndsnet

  elasticsearch:
    container_name: elasticsearch
    environment:
      - discovery.type=single-node
      - xpack.security.enabled=false
      - ES_JAVA_OPTS=-Xms256m -Xmx256m -Dlog4j2.formatMsgNoLookups=true
    healthcheck:
      retries: 4
      start_period: 2m
      test:
        - CMD-SHELL
        - curl -sS --fail 'http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=0s' || exit 1
    hostname: elasticsearch
    image: elasticsearch:7.9.3
    mem_limit: 1g
    ports:
      - 9200:9200
    volumes:
      - ./data/elasticsearch:/usr/share/elasticsearch/data
    networks:
      - ndsnet
  
  elasticsearch-setup:
    container_name: elasticsearch-setup
    depends_on:
      - elasticsearch
    environment:
      - ELASTICSEARCH_HOST=elasticsearch
      - ELASTICSEARCH_PORT=9200
      - ELASTICSEARCH_PROTOCOL=http
    hostname: elasticsearch-setup
    image: linkedin/datahub-elasticsearch-setup:${DATAHUB_VERSION:-head}
    networks:
      - ndsnet

  kafka-setup:
    container_name: kafka-setup
    depends_on:
      - broker
      - schema-registry
    environment:
      - KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181
      - KAFKA_BOOTSTRAP_SERVER=broker:29092
    hostname: kafka-setup
    image: linkedin/datahub-kafka-setup:${DATAHUB_VERSION:-head}
    networks:
      - ndsnet
  
  mysql-setup:
    container_name: mysql-setup
    depends_on:
      - mariadb
    environment:
      - MYSQL_HOST=mariadb
      - MYSQL_PORT=3306
      - MYSQL_USERNAME=root
      - MYSQL_PASSWORD=admin
      - DATAHUB_DB_NAME=datahub
    hostname: mysql-setup
    image: acryldata/datahub-mysql-setup:${DATAHUB_VERSION:-head}
    networks:
      - ndsnet
  
  schema-registry:
    container_name: schema-registry
    depends_on:
      - zookeeper
      - broker
    environment:
      - SCHEMA_REGISTRY_HOST_NAME=schemaregistry
      - SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL=zookeeper:2181
    hostname: schema-registry
    image: confluentinc/cp-schema-registry:5.4.0
    ports:
      - 8081:8081
    networks:
      - ndsnet
  
  zookeeper:
    container_name: zookeeper
    environment:
      - ZOOKEEPER_CLIENT_PORT=2181
      - ZOOKEEPER_TICK_TIME=2000
    hostname: zookeeper
    image: confluentinc/cp-zookeeper:5.4.0
    ports:
      - 2181:2181
    volumes:
      - ./data/zookeeper:/var/opt/zookeeper
    networks:
      - ndsnet

  cube:
    image: cubejs/cube
    container_name: cube
    hostname: cube
    volumes:
      - ./conf/cube:/cube/conf
    ports:
      - 3000:3000 
      - 4000:4000
      - 3245:3245
    networks:
      - ndsnet
    depends_on:
      - aio

  metabase:
    container_name: metabase
    depends_on:
      - cube
    hostname: metabase
    build: metabase/
    volumes:
      - ./conf/metabase:/conf
    ports:
      - 3030:3030
    networks:
      - ndsnet
  
networks:
  ndsnet:
    driver: bridge