-
Notifications
You must be signed in to change notification settings - Fork 0
/
docker-compose.yml
105 lines (96 loc) · 3.05 KB
/
docker-compose.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
version: "3.7"
services:
db:
image: postgres:12.3-alpine
container_name: films-db-dev
networks:
- default_net
volumes:
- postgres_data:/var/lib/postgresql/data
- ./pg-init-scripts:/docker-entrypoint-initdb.d
env_file: postgres.env
ports:
- "5455:5432"
spark:
image: docker.io/bitnami/spark:3-debian-10
# https://docs.bitnami.com/tutorials/work-with-non-root-containers/
user: root
hostname: spark
networks:
- default_net
environment:
- SPARK_MODE=master
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
volumes:
- .:/app
- ./spark/app:/usr/local/spark/app
- ./spark/resources:/usr/local/spark/resources
ports:
- "8181:8080"
spark-worker-1:
image: docker.io/bitnami/spark:3-debian-10
user: root
networks:
- default_net
environment:
- SPARK_MODE=worker
- SPARK_MASTER_URL=spark://spark:7077
- SPARK_WORKER_MEMORY=1G
- SPARK_WORKER_CORES=1
- SPARK_RPC_AUTHENTICATION_ENABLED=no
- SPARK_RPC_ENCRYPTION_ENABLED=no
- SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
- SPARK_SSL_ENABLED=no
volumes:
- .:/app
- ./spark/app:/usr/local/spark/app
- ./spark/resources:/usr/local/spark/resources
# spark-worker-2:
# image: docker.io/bitnami/spark:3-debian-10
# user: root
# networks:
# - default_net
# environment:
# - SPARK_MODE=worker
# - SPARK_MASTER_URL=spark://spark:7077
# - SPARK_WORKER_MEMORY=1G
# - SPARK_WORKER_CORES=1
# - SPARK_RPC_AUTHENTICATION_ENABLED=no
# - SPARK_RPC_ENCRYPTION_ENABLED=no
# - SPARK_LOCAL_STORAGE_ENCRYPTION_ENABLED=no
# - SPARK_SSL_ENABLED=no
# volumes:
# - .:/app
# - ./spark/app:/usr/local/spark/app
# - ./spark/resources:/usr/local/spark/resources
work-env:
image: bitnami/spark:3-debian-10
volumes:
- .:/app
command: tail -f /dev/null
user: root
working_dir: /app
environment:
PYTHONPATH: /opt/bitnami/spark/python:/opt/bitnami/spark/python/lib/py4j-0.10.9-src.zip
networks:
- default_net
volumes:
postgres_data:
networks:
default_net:
#jupyter-spark:
#image: jupyter/pyspark-notebook:spark-3.1.2
#build: .
#container_name: films-jupyter-pyspark
#networks:
#- default_net
#ports:
#- "8888:8888"
#- "4040-4080:4040-4080"
#volumes:
#- ./notebooks:/home/jovyan/work/notebooks/
#- ./spark/resources/data:/home/jovyan/work/data/
#- ./spark/resources/jars:/home/jovyan/work/jars/