-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
95 lines (88 loc) · 1.9 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
version: '2'
services:
namenode:
image: bde2020/hadoop-namenode:1.1.0-hadoop2.8-java8
container_name: namenode
volumes:
- ./data/namenode:/hadoop/dfs/name
environment:
- CLUSTER_NAME=test
env_file:
- ./hadoop.env
ports:
- 50070:50070
- 8020:8020
datanode:
image: bde2020/hadoop-datanode:1.1.0-hadoop2.8-java8
container_name: datanode
depends_on:
- namenode
volumes:
- ./data/datanode:/hadoop/dfs/data
env_file:
- ./hadoop.env
ports:
- 50075:50075
spark-master:
build: ./master
image: spark-master
container_name: spark-master
ports:
- "18080:18080"
- "17077:17077"
env_file:
- ./hadoop.env
environment:
- "NAMENODE_HOSTNAME=namenode"
- "INIT_DAEMON_STEP=setup_spark"
- "constraint:node==spark-master"
spark-worker-1:
build: ./worker
image: spark-worker
container_name: spark-worker-1
depends_on:
- datanode
- namenode
- spark-master
ports:
- "18081:18081"
env_file:
- ./hadoop.env
environment:
- "NAMENODE_HOSTNAME=namenode"
- "SPARK_MASTER=spark://spark-master:17077"
- "constraint:node==spark-worker-1"
livy:
build: ./livy
image: livy
container_name: livy
depends_on:
- datanode
- namenode
- spark-master
ports:
- "8998:8998"
env_file:
- ./hadoop.env
environment:
- "NAMENODE_HOSTNAME=namenode"
spark-submit:
build: ./submit
image: spark-submit
container_name: spark-submit
volumes:
- ./submit/app:/app
env_file:
- ./hadoop.env
depends_on:
- datanode
- namenode
- spark-worker-1
- spark-master
postgres:
image: postgres:10.4-alpine
container_name: postgres
ports:
- "5432:5432"
volumes:
- ./data/pg/data:/var/lib/postgresql/data