-
Notifications
You must be signed in to change notification settings - Fork 78
/
docker-compose-with-celery-executor-2-workers.yml
160 lines (159 loc) · 3.72 KB
/
docker-compose-with-celery-executor-2-workers.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
version: '3.9'
networks:
airflow:
services:
postgres:
image: postgres:13.1
environment:
- POSTGRES_USER=airflow
- POSTGRES_DB=airflow
- POSTGRES_PASSWORD=airflow
- PGDATA=/var/lib/postgresql/data/pgdata
ports:
- 5432:5432
volumes:
- /var/run/docker.sock:/var/run/docker.sock
- ./database/data:/var/lib/postgresql/data/pgdata
- ./database/logs:/var/lib/postgresql/data/log
command: >
postgres
-c listen_addresses=*
-c logging_collector=on
-c log_destination=stderr
-c max_connections=200
networks:
- airflow
redis:
image: redis:5.0.5
environment:
REDIS_HOST: redis
REDIS_PORT: 6379
ports:
- 6379:6379
networks:
- airflow
webserver:
env_file:
- .env
image: apache/airflow:2.5.0-python3.8
ports:
- 8080:8080
volumes:
- ./airflow_files/dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./files:/opt/airflow/files
- /var/run/docker.sock:/var/run/docker.sock
deploy:
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
depends_on:
- postgres
- redis
- initdb
command: webserver
healthcheck:
test: ["CMD-SHELL", "[ -f /opt/airflow/airflow-webserver.pid ]"]
interval: 30s
timeout: 30s
retries: 3
networks:
- airflow
flower:
image: apache/airflow:2.5.0-python3.8
env_file:
- .env
ports:
- 5555:5555
depends_on:
- redis
deploy:
restart_policy:
condition: on-failure
delay: 8s
max_attempts: 3
volumes:
- ./logs:/opt/airflow/logs
command: celery flower
networks:
- airflow
scheduler:
image: apache/airflow:2.5.0-python3.8
env_file:
- .env
volumes:
- ./airflow_files/dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./files:/opt/airflow/files
- /var/run/docker.sock:/var/run/docker.sock
command: scheduler
depends_on:
- initdb
deploy:
restart_policy:
condition: always
delay: 5s
window: 120s
networks:
- airflow
worker_1:
image: apache/airflow:2.5.0-python3.8
env_file:
- .env
volumes:
- ./airflow_files/dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./files:/opt/airflow/files
- /var/run/docker.sock:/var/run/docker.sock
command: celery worker -H worker_1_name
depends_on:
- scheduler
deploy:
restart_policy:
condition: on-failure
delay: 8s
max_attempts: 3
networks:
- airflow
worker_2:
image: apache/airflow:2.5.0-python3.8
env_file:
- .env
volumes:
- ./airflow_files/dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./files:/opt/airflow/files
- /var/run/docker.sock:/var/run/docker.sock
command: celery worker -H worker_2_name
depends_on:
- scheduler
deploy:
restart_policy:
condition: on-failure
delay: 8s
max_attempts: 3
networks:
- airflow
initdb:
image: apache/airflow:2.5.0-python3.8
env_file:
- .env
volumes:
- ./airflow_files/dags:/opt/airflow/dags
- ./logs:/opt/airflow/logs
- ./files:/opt/airflow/files
- /var/run/docker.sock:/var/run/docker.sock
entrypoint: /bin/bash
deploy:
restart_policy:
condition: on-failure
delay: 8s
max_attempts: 5
command: -c "airflow db init && airflow users create --firstname admin --lastname admin --email admin --password admin --username admin --role Admin"
depends_on:
- redis
- postgres
networks:
- airflow