docker-compose.yml

version: '3' 
x-airflow-common:
  &airflow-common
  image: fjardim/mds-airflow
  environment:
    &airflow-common-env
    AIRFLOW__CORE__EXECUTOR: CeleryExecutor
    AIRFLOW__DATABASE__SQL_ALCHEMY_CONN: postgresql+psycopg2://admin:admin@postgres/airflow
    AIRFLOW__CORE__SQL_ALCHEMY_CONN: postgresql+psycopg2://admin:admin@postgres/airflow
    AIRFLOW__CELERY__RESULT_BACKEND: db+postgresql://admin:admin@postgres/airflow
    AIRFLOW__CELERY__BROKER_URL: redis://:@redis:6379/0
    AIRFLOW__CORE__FERNET_KEY: ''
    AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true'
    AIRFLOW__CORE__LOAD_EXAMPLES: 'false'
    AIRFLOW__API__AUTH_BACKENDS: 'airflow.api.auth.backend.basic_auth'
    #_PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:- apache-airflow-providers-apache-spark}
  volumes:
    - ./airflow/dags:/opt/airflow/dags
    - ./airflow/logs:/opt/airflow/logs
    - ./airflow/plugins:/opt/airflow/plugins
    - ./scripts:/scripts
  user: "${AIRFLOW_UID:-50000}:0"
  depends_on:
    &airflow-common-depends-on
    redis:
      condition: service_healthy
    postgres:
      condition: service_healthy

services:
  zookeeper:
    image: fjardim/mds-kafka-zookeeper
    hostname: zookeeper
    container_name: zookeeper
    ports:
       - "2181:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  kafka-broker:
    image: fjardim/mds-kafka-broker
    container_name: kafka-broker
    hostname: kafka-broker
    ports:
      - "9092:9092"
    depends_on:
      - zookeeper
    environment:
      KAFKA_BROKER_ID: 1
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka-broker:9092
      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
      CONFLUENT_METRICS_REPORTER_BOOTSTRAP_SERVERS: kafka-broker:9092
      CONFLUENT_METRICS_REPORTER_TOPIC_REPLICAS: 1
      KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS: 0
      KAFKA_CONFLUENT_LICENSE_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_CONFLUENT_BALANCER_TOPIC_REPLICATION_FACTOR: 1
      KAFKA_TRANSACTION_STATE_LOG_MIN_ISR: 1
      KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR: 1
      KAFKA_JMX_PORT: 9101
      KAFKA_JMX_HOSTNAME: kafka-broker
      CONFLUENT_METRICS_ENABLE: 'false'
      CONFLUENT_SUPPORT_CUSTOMER_ID: 'anonymous'
      # AUTO_REGISTER_SCHEMAS : 'false'

  kafka-schema-registry:
    image: fjardim/mds-kafka-schema-registry
    hostname: kafka-schema-registry
    container_name: kafka-schema-registry
    depends_on:
      - kafka-broker
    ports:
      - "8071:8081"
    environment:
      SCHEMA_REGISTRY_HOST_NAME: schema-registry
      SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
      SCHEMA_REGISTRY_LISTENERS: http://0.0.0.0:8081

  kafka-connect:
    image: fjardim/mds-kafka-connect
    hostname: kafka-connect
    container_name: kafka-connect
    depends_on:
      - kafka-broker
      - kafka-schema-registry
      - postgres
    ports:
      - "8083:8083"
    environment:
      CONNECT_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
      CONNECT_REST_ADVERTISED_HOST_NAME: connect
      CONNECT_REST_PORT: 8083
      CONNECT_GROUP_ID: connect-group
      CONNECT_CONFIG_STORAGE_TOPIC: connect-configs
      CONNECT_CONFIG_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_OFFSET_FLUSH_INTERVAL_MS: 10000
      CONNECT_OFFSET_STORAGE_TOPIC: connect-offsets
      CONNECT_OFFSET_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_STATUS_STORAGE_TOPIC: connect-status
      CONNECT_STATUS_STORAGE_REPLICATION_FACTOR: 1
      CONNECT_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_KEY_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_INTERNAL_VALUE_CONVERTER: org.apache.kafka.connect.json.JsonConverter
      CONNECT_VALUE_CONVERTER_SCHEMA_REGISTRY_URL: http://kafka-schema-registry:8081
      # CLASSPATH required due to CC-2422
      CLASSPATH: /usr/share/java/monitoring-interceptors/monitoring-interceptors-6.1.0.jar
      CONNECT_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      CONNECT_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
      CONNECT_PLUGIN_PATH: "/usr/share/java,/usr/share/confluent-hub-components"
      CONNECT_LOG4J_LOGGERS: org.apache.zookeeper=ERROR,org.I0Itec.zkclient=ERROR,org.reflections=ERROR
      
         
  kafka-control-center:
    image: fjardim/mds-kafka-control-center
    hostname: kafka-control-center
    container_name: kafka-control-center
    depends_on:
      - kafka-broker
      - kafka-schema-registry
      - kafka-connect
      - kafka-ksqldb-server
      - kafka-ksqldb-cli
      - postgres
    ports:
      - "9021:9021"
    environment:
      CONTROL_CENTER_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
      CONTROL_CENTER_CONNECT_CLUSTER: 'kafka-connect:8083'
      CONTROL_CENTER_CONNECT_CONNECT-DEFAULT_CLUSTER: 'kafka-connect:8083'
      CONTROL_CENTER_KSQL_KSQLDB1_URL: "http://kafka-ksqldb-server:8088"
      CONTROL_CENTER_KSQL_KSQLDB1_ADVERTISED_URL: "http://kafka-ksqldb-server:8088"
      CONTROL_CENTER_SCHEMA_REGISTRY_URL: "http://kafka-schema-registry:8081"
      CONTROL_CENTER_REPLICATION_FACTOR: 1
      CONTROL_CENTER_INTERNAL_TOPICS_PARTITIONS: 1
      CONTROL_CENTER_MONITORING_INTERCEPTOR_TOPIC_PARTITIONS: 1
      CONFLUENT_METRICS_TOPIC_REPLICATION: 1
      CONTROL_CENTER_CONNECT_HEALTHCHECK_ENDPOINT: '/connectors'
      PORT: 9021

  kafka-ksqldb-server:
    image: fjardim/mds-kafka-ksqldb-server
    hostname: kafka-ksqldb-server
    container_name: kafka-ksqldb-server
    depends_on:
      - kafka-broker
      - kafka-connect
    ports:
      - "8068:8088"
    environment:
      KSQL_CONFIG_DIR: "/etc/ksql"
      KSQL_BOOTSTRAP_SERVERS: "kafka-broker:9092"
      KSQL_HOST_NAME: ksqldb-server
      KSQL_LISTENERS: "http://0.0.0.0:8088"
      KSQL_CACHE_MAX_BYTES_BUFFERING: 0
      KSQL_KSQL_SCHEMA_REGISTRY_URL: "http://kafka-schema-registry:8081"
      KSQL_PRODUCER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringProducerInterceptor"
      KSQL_CONSUMER_INTERCEPTOR_CLASSES: "io.confluent.monitoring.clients.interceptor.MonitoringConsumerInterceptor"
      KSQL_KSQL_CONNECT_URL: "http://kafka-connect:8083"
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_REPLICATION_FACTOR: 1
      KSQL_KSQL_LOGGING_PROCESSING_TOPIC_AUTO_CREATE: 'true'
      KSQL_KSQL_LOGGING_PROCESSING_STREAM_AUTO_CREATE: 'true'

  kafka-ksqldb-cli:
    image: fjardim/mds-kafka-sqldb-cli
    container_name: kafka-ksqldb-cli
    hostname: kafka-ksqldb-cli
    depends_on:
      - kafka-broker
      - kafka-connect
      - kafka-ksqldb-server
    entrypoint: /bin/sh
    tty: true

  
  kafka-rest-proxy:
   image: fjardim/mds-kafka-rest
   hostname: kafka-rest-proxy
   depends_on:
     - kafka-broker
     - kafka-schema-registry
   ports:
     - 8082:8082
   container_name: kafka-rest-proxy
   environment:
     KAFKA_REST_HOST_NAME: rest-proxy
     KAFKA_REST_BOOTSTRAP_SERVERS: 'kafka-broker:9092'
     KAFKA_REST_LISTENERS: "http://0.0.0.0:8082"
     KAFKA_REST_SCHEMA_REGISTRY_URL: 'http://schema-registry:8081'


  nifi:
    image: fjardim/mds-nifi
    container_name: nifi
    hostname: nifi
    volumes:
      - ./nifi/flow:/opt/nifi/nifi-current/flowfile_repository
      - ./nifi/util:/util
      - ./scripts:/scripts
    environment:
      NIFI_WEB_HTTP_PORT: "9090"
      SINGLE_USER_CREDENTIALS_USERNAME: "admin"
      SINGLE_USER_CREDENTIALS_PASSWORD: "admin"
    ports:
      - "9090:9090"
      - "8443:8443"
    deploy:
      resources:
        limits:
          memory: 2048m

  postgres:
    image: fjardim/mds-postgres
    container_name: postgres
    hostname: postgres
    environment:
      POSTGRES_USER: admin
      POSTGRES_PASSWORD: admin
      POSTGRES_DB: admin 
    ports:
      - 5442:5432
    volumes:
      - ./postgres/postgres-db-volume:/var/lib/postgresql/data
      - ./postgres:/util
      - ./scripts:/scripts
    #healthcheck:
    #  test: ["CMD", "pg_isready"]
    #  interval: 5s
    #  retries: 5
    
  redis:
    image: fjardim/mds-redis
    container_name: redis
    hostname: redis
    expose:
      - 6379
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 30s
      retries: 50
    
  airflow-webserver:
    <<: *airflow-common
    container_name: airflow-webserver
    hostname: airflow-webserver
    command: webserver
    user: root
    ports:
      - 8180:8080
    healthcheck:
      test: ["CMD", "curl", "--fail", "http://localhost:8080/health"]
      interval: 10s
      timeout: 10s
      retries: 5
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-scheduler:
    <<: *airflow-common
    container_name: airflow-scheduler
    hostname: airflow-scheduler
    command: scheduler
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type SchedulerJob --hostname "$${HOSTNAME}"']
      interval: 10s
      timeout: 10s
      retries: 5
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-worker:
    <<: *airflow-common
    container_name: airflow-worker
    hostname: airflow-worker
    command: celery worker
    healthcheck:
      test:
        - "CMD-SHELL"
        - 'celery --app airflow.executors.celery_executor.app inspect ping -d "celery@$${HOSTNAME}"'
      interval: 10s
      timeout: 10s
      retries: 5
    environment:
      <<: *airflow-common-env
      # Required to handle warm shutdown of the celery workers properly
      # See https://airflow.apache.org/docs/docker-stack/entrypoint.html#signal-propagation
      DUMB_INIT_SETSID: "0"
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-triggerer:
    <<: *airflow-common
    container_name: airflow-triggerer
    hostname: airflow-triggerer
    command: triggerer
    healthcheck:
      test: ["CMD-SHELL", 'airflow jobs check --job-type TriggererJob --hostname "$${HOSTNAME}"']
      interval: 10s
      timeout: 10s
      retries: 5
    depends_on:
      <<: *airflow-common-depends-on
      airflow-init:
        condition: service_completed_successfully

  airflow-init:
    <<: *airflow-common
    entrypoint: /bin/bash
    # yamllint disable rule:line-length
    command:
      - -c
      - |
        function ver() {
          printf "%04d%04d%04d%04d" $${1//./ }
        }
        airflow_version=$$(AIRFLOW__LOGGING__LOGGING_LEVEL=INFO && gosu airflow airflow version)
        airflow_version_comparable=$$(ver $${airflow_version})
        min_airflow_version=2.2.0
        min_airflow_version_comparable=$$(ver $${min_airflow_version})
        if (( airflow_version_comparable < min_airflow_version_comparable )); then
          echo
          echo -e "\033[1;31mERROR!!!: Too old Airflow version $${airflow_version}!\e[0m"
          echo "The minimum Airflow version supported: $${min_airflow_version}. Only use this or higher!"
          echo
          exit 1
        fi
        if [[ -z "${AIRFLOW_UID}" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: AIRFLOW_UID not set!\e[0m"
          echo "If you are on Linux, you SHOULD follow the instructions below to set "
          echo "AIRFLOW_UID environment variable, otherwise files will be owned by root."
          echo "For other operating systems you can get rid of the warning with manually created .env file:"
          echo "    See: https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#setting-the-right-airflow-user"
          echo
        fi
        one_meg=1048576
        mem_available=$$(($$(getconf _PHYS_PAGES) * $$(getconf PAGE_SIZE) / one_meg))
        cpus_available=$$(grep -cE 'cpu[0-9]+' /proc/stat)
        disk_available=$$(df / | tail -1 | awk '{print $$4}')
        warning_resources="false"
        if (( mem_available < 4000 )) ; then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough memory available for Docker.\e[0m"
          echo "At least 4GB of memory required. You have $$(numfmt --to iec $$((mem_available * one_meg)))"
          echo
          warning_resources="true"
        fi
        if (( cpus_available < 2 )); then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough CPUS available for Docker.\e[0m"
          echo "At least 2 CPUs recommended. You have $${cpus_available}"
          echo
          warning_resources="true"
        fi
        if (( disk_available < one_meg * 10 )); then
          echo
          echo -e "\033[1;33mWARNING!!!: Not enough Disk space available for Docker.\e[0m"
          echo "At least 10 GBs recommended. You have $$(numfmt --to iec $$((disk_available * 1024 )))"
          echo
          warning_resources="true"
        fi
        if [[ $${warning_resources} == "true" ]]; then
          echo
          echo -e "\033[1;33mWARNING!!!: You have not enough resources to run Airflow (see above)!\e[0m"
          echo "Please follow the instructions to increase amount of resources available:"
          echo "   https://airflow.apache.org/docs/apache-airflow/stable/howto/docker-compose/index.html#before-you-begin"
          echo
        fi
        mkdir -p /sources/logs /sources/dags /sources/plugins
        chown -R "${AIRFLOW_UID}:0" /sources/{logs,dags,plugins}
        exec /entrypoint airflow version
    # yamllint enable rule:line-length
    environment:
      <<: *airflow-common-env
      _AIRFLOW_DB_UPGRADE: 'true'
      _AIRFLOW_WWW_USER_CREATE: 'true'
      _AIRFLOW_WWW_USER_USERNAME: ${_AIRFLOW_WWW_USER_USERNAME:-airflow}
      _AIRFLOW_WWW_USER_PASSWORD: ${_AIRFLOW_WWW_USER_PASSWORD:-airflow}
      _PIP_ADDITIONAL_REQUIREMENTS: ''
    user: "0:0"
    volumes:
      - ./airflow:/sources

  airflow-cli:
    <<: *airflow-common
    container_name: airflow-cli
    hostname: airflow-cli
    profiles:
      - debug
    environment:
      <<: *airflow-common-env
      CONNECTION_CHECK_MAX_COUNT: "0"
    # Workaround for entrypoint issue. See: https://github.com/apache/airflow/issues/16252
    command:
      - bash
      - -c
      - airflow

  spark-master:
    image: fjardim/mds-spark
    hostname: spark-master
    container_name: spark-master
    command: 
      - /bin/sh
      - -c
      - |
        /usr/local/spark/sbin/start-master.sh
        start-notebook.sh --NotebookApp.token=''
    #environment:
    #  PYSPARK_SUBMIT_ARGS: "--packages io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.392"
    #env_file:
    #  - ./data/jupyter/jupyter.env
    #working_dir: /home/root
    #entrypoint: ["chmod", "+x", "/env/start-master.sh","/env","./start-master.sh"]
    ports:
      - 8889:8888
      - 4040:4040
      - 4041:4041
      - 4042:4042
      - 4043:4043
      - 8180:8080
      - 7077:7077
    volumes:
      - ./spark/util:/util/
      - ./spark/work:/home/user 
      - ./spark/env:/env 
      - ./scripts:/scripts
      #- ./jupyter/conf:/usr/local/spark/conf
    #environment:
    #  - GRANT_SUDO=yes
    #  - CHOWN_HOME=yes
    #  - NB_USER=spark
    #  - NB_GID=100
    #  - CHOWN_HOME_OPTS='-R'
      #- GEN_CERT=yes

    #   SPARK_MASTER: local[*]
    #   JUPYTER_PORT: 8889
    #deploy:
    #  resources:
    #    limits:
    #      memory: 500m

  spark-worker:
    image: fjardim/mds-spark
    hostname: spark-worker
    container_name: spark-worker
    #command: start-notebook.sh --NotebookApp.token='' 
    command: 
      - /bin/sh
      - -c
      - |
        /usr/local/spark/sbin/start-worker.sh spark-master:7077
        start-notebook.sh --NotebookApp.token='' 
    #command: /usr/local/spark/sbin/start-worker.sh jupyter-spark:7077
    #environment:
    #  PYSPARK_SUBMIT_ARGS: "--packages io.delta:delta-core_2.12:2.2.0,org.apache.hadoop:hadoop-aws:3.3.1,com.amazonaws:aws-java-sdk-bundle:1.12.392"
    env_file:
      - ./spark/env/jupyter.env
    #working_dir: /home/root
    ports:
      - 5040:4040
      - 5041:4041
      - 5042:4042
      - 5043:4043
      - 8881:8081
      - 36533:36533
    volumes:
      - ./spark/util:/util/
      - ./spark/work:/home/user 
      - ./scripts:/scripts
    environment:
      SPARK_MASTER: spark-master
    #   JUPYTER_PORT: 8889
    depends_on:
        - spark-master
    #deploy:
    #  resources:
    #    limits:
    #      memory: 1g

  metabase:
    image: fjardim/mds-metabase
    container_name: metabase
    hostname: metabase
    ports:
      - "3000:3000"
    depends_on:
      - postgres
    environment:
      MB_DB_TYPE: postgres
      MB_DB_DBNAME: metabase
      MB_DB_PORT: 5432
      MB_DB_USER: admin
      MB_DB_PASS: admin
      MB_DB_HOST: postgres
      MB_PASSWORD_COMPLEXITY: "weak"
      MB_PASSWORD_LENGTH: 4

  elasticsearch:
    image: fjardim/mds-elasticsearch
    ports:
      - "9200:9200"
      - "9300:9300"
    environment:
      discovery.type: "single-node"
      ES_JAVA_OPTS: "-Xms2g -Xmx2g"
      xpack.monitoring.enabled: "true"
    volumes:
      - ./elastic/esdata:/usr/share/elasticsearch/data
      - ./scripts:/scripts
      
  kibana:
    image: fjardim/mds-kibana
    ports:
      - "5601:5601"
    environment:
      ELASTICSEARCH_URL: http://elasticsearch:9200
    depends_on:
      - elasticsearch


  dbeaver:
    image: fjardim/mds-dbeaver
    #image: dbeaver/cloudbeaver:latest
    container_name: dbeaver
    hostname: dbeaver
    #restart: unless-stopped 
    depends_on:
      - postgres
    #command: ./run-server.sh
    #environment:
    #    CB_ADMIN_NAME: cbadmin 
    #    CB_ADMIN_PASSWORD: cbadmin123
    ports: 
      - "8010:8978" 
    volumes:
      - ./scripts:/scripts 
      #- ./dbeaver:/opt/cloudbeaver/workspace 
      #- ./dbeaver/workspace/cloudbeaver.conf:/opt/cloudbeaver/conf/cloudbeaver.conf
      #- ./dbeaver/workspace/initial-data.conf:/opt/cloudbeaver/conf/initial-data.conf 

  trino:
    image: fjardim/mds-trino 
    #image: trinodb/trino:latest
    container_name: trino
    hostname: trino
    ports: 
      - "8080:8080" 
    depends_on:
      - postgres
      #- metastore
    volumes: 
      - ./trino/catalog:/etc/trino/catalog  
      - ./trino/conf:/conf
      - ./scripts:/scripts

  hive:
    hostname: hive
    #image: apache/hive:4.0.0-alpha-2
    image: fjardim/mds-hive
    container_name: hive
    environment:
      AWS_ACCESS_KEY_ID: datalake
      AWS_SECRET_ACCESS_KEY: datalake
      HIVE_CUSTOM_CONF_DIR: "/hive_custom_conf"
      SERVICE_NAME: hiveserver2
      SERVICE_OPTS: "-Dhive.metastore.uris=thrift://metastore:9083" 
      IS_RESUME: "true"
      #HIVE_VERSION: "3.1.3"
    ports:
       - "10000:10000"
       - "10002:10002"
    depends_on:
        - hive-metastore
    user: root
    volumes:
       - ./hive/conf:/hive_custom_conf
       - ./hive:/util

  hive-metastore:
    hostname: metastore
    image: fjardim/mds-hive-metastore
    #image: apache/hive:4.0.0-alpha-2
    container_name: metastore
    environment:
      AWS_ACCESS_KEY_ID: datalake
      AWS_SECRET_ACCESS_KEY: datalake
      HIVE_CUSTOM_CONF_DIR: "/hive_custom_conf"
      SERVICE_NAME: metastore
      #SERVICE_OPTS: "-Dhive.metastore.uris=thrift://metastore:9083" 
      IS_RESUME: "true"
      DB_DRIVER: postgres 
      SERVICE_OPTS: "-Djavax.jdo.option.ConnectionDriverName=org.postgresql.Driver -Djavax.jdo.option.ConnectionURL=jdbc:postgresql://postgres:5432/metastore_db -Djavax.jdo.option.ConnectionUserName=admin -Djavax.jdo.option.ConnectionPassword=admin" 
    ports:
       - "9083:9083"
    depends_on: 
      - postgres
    user: root
    volumes:
       - ./hive/meta:/opt/hive/data/warehouse 
       - ./hive/conf:/hive_custom_conf
       - ./hive:/util

  minio:
    image: fjardim/mds-minio
    container_name: minio
    command:  server --console-address ":9001" /data
    ports:
      - "9050:9000"
      - "9051:9001"
    hostname: minio
    environment:
      MINIO_ROOT_USER: admin
      MINIO_ROOT_PASSWORD: minioadmin
      MINIO_ACCESS_KEY: datalake
      MINIO_SECRET_KEY: datalake
    volumes:
      - ./minio/data1:/data
      - ./scripts:/scripts

  superset:
    env_file: ./superset/docker/env-non-dev
    image: fjardim/mds-superset
    container_name: superset_app
    hostname: superset_app
    command: ["/app/docker/docker-bootstrap.sh", "app-gunicorn"]
    user: "root"
    restart: unless-stopped
    ports:
      - 8088:8088
    depends_on: 
      - postgres
      - redis
      - trino
    volumes: 
      - ./superset/docker:/app/docker
      - ./superset/superset_home:/app/superset_home
      - ./scripts:/scripts


  pinot-zookeeper:
    image: fjardim/mds-pinot-zookeeper
    container_name: pinot-zookeeper
    ports:
      - "2182:2181"
    environment:
      ZOOKEEPER_CLIENT_PORT: 2181
      ZOOKEEPER_TICK_TIME: 2000

  pinot-controller:
    image: fjardim/mds-pinot
    command: "StartController -zkAddress pinot-zookeeper:2181"
    container_name: pinot-controller
    restart: unless-stopped
    ports:
      - "9000:9000"
    environment:
      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms1G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-controller.log"
    depends_on:
      - pinot-zookeeper

  pinot-broker:
    image: fjardim/mds-pinot
    command: "StartBroker -zkAddress pinot-zookeeper:2181"
    restart: unless-stopped
    container_name: "pinot-broker"
    ports:
      - "8099:8099"
    environment:
      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx4G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-broker.log"
    depends_on:
      - pinot-controller

  pinot-server:
    image: fjardim/mds-pinot
    command: "StartServer -zkAddress pinot-zookeeper:2181"
    restart: unless-stopped
    container_name: "pinot-server"
    ports:
      - "8098:8098"
    environment:
      JAVA_OPTS: "-Dplugins.dir=/opt/pinot/plugins -Xms4G -Xmx16G -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -Xloggc:gc-pinot-server.log"
    depends_on:
      - pinot-broker
      - kafka-broker

  namenode:
    image: fjardim/mds-namenode
    container_name: namenode
    hostname: namenode
    volumes:
      - ./hadoop/hdfs/namenode:/hadoop/dfs/name
      - ./hadoop/util:/util
    env_file:
      - ./hadoop/hadoop.env
    ports:
      - "9870:9870"
    deploy:
      resources:
        limits:
          memory: 500m
  
  datanode:
    image: fjardim/mds-datanode
    container_name: datanode
    hostname: datanode
    volumes:
      - ./hadoop/hdfs/datanode:/hadoop/dfs/data
      - ./hadoop/util:/util
    env_file:
      - ./hadoop/hadoop.env
    environment:
      SERVICE_PRECONDITION: "namenode:9870"
    depends_on:
      - namenode
    ports:
      - "9864:9864"
    deploy:
      resources:
        limits:
          memory: 500m

  hue:
    image: fjardim/mds-hue
    hostname: hue
    container_name: hue
    dns: 8.8.8.8
    ports:
    - "8888:8888"
    volumes:
      - ./hue/hue.ini:/usr/share/hue/desktop/conf/z-hue.ini
    depends_on:
      - "postgres"
    deploy:
      resources:
        limits:
          memory: 500m