Merge pull request #182 from julienrf/improve-docker

Simplify the Docker setup
scylladb · Jul 31, 2024 · 11926be · 11926be
2 parents 0b30392 + c2835e5
commit 11926be
Show file tree

Hide file tree

Showing 3 changed files with 7 additions and 78 deletions.
diff --git a/docker-compose-tests.yml b/docker-compose-tests.yml
@@ -3,9 +3,6 @@ services:
   dynamodb:
     command: "-jar DynamoDBLocal.jar -sharedDb -inMemory"
     image: "amazon/dynamodb-local:latest"
-    container_name: dynamodb
-    networks:
-      - scylla
     expose:
       - 8001
     ports:
@@ -14,8 +11,6 @@ services:
 
   cassandra:
     image: cassandra:latest
-    networks:
-      - scylla
     volumes:
       - ./tests/docker/cassandra:/var/lib/cassandra
     ports:
@@ -25,8 +20,6 @@ services:
 
   scylla-source:
     image: scylladb/scylla:latest
-    networks:
-      - scylla
     volumes:
       - "./tests/docker/scylla-source:/var/lib/scylla"
     ports:
@@ -38,8 +31,6 @@ services:
 
   scylla:
     image: scylladb/scylla:latest
-    networks:
-      - scylla
     volumes:
       - "./tests/docker/scylla:/var/lib/scylla"
     ports:
@@ -56,33 +47,17 @@ services:
     volumes:
       - "./tests/docker/s3:/var/lib/localstack"
       - "/var/run/docker.sock:/var/run/docker.sock"
-    networks:
-      - scylla
 
   spark-master:
     build: dockerfiles/spark
     command: master
-    container_name: spark-master
-    hostname: spark-master
     environment:
-      SPARK_PUBLIC_DNS: spark-master
-    networks:
-      - scylla
+      SPARK_PUBLIC_DNS: localhost
     expose:
       - 5005
-      - 7001
-      - 7002
-      - 7003
-      - 7004
-      - 7005
-      - 7006
-      - 7077
-      - 6066
     ports:
       - 4040:4040
       - 5005:5005
-      - 6066:6066
-      - 7077:7077
       - 8080:8080
     volumes:
       - ./migrator/target/scala-2.13:/jars
@@ -93,29 +68,17 @@ services:
   spark-worker:
     build: dockerfiles/spark
     command: worker
-    hostname: spark-worker
-    container_name: spark-worker-1
     environment:
       SPARK_WORKER_CORES: 3
       SPARK_WORKER_MEMORY: 1024m
       SPARK_WORKER_WEBUI_PORT: 8081
-      SPARK_PUBLIC_DNS: spark-worker
-    networks:
-      - scylla
+      SPARK_PUBLIC_DNS: localhost
     expose:
       - 5006
-      - 7012
-      - 7013
-      - 7014
-      - 7015
-      - 7016
     ports:
       - 5006:5006
       - 8081:8081
     volumes:
       - ./tests/docker/parquet:/app/parquet
     depends_on:
       - spark-master
-
-networks:
-  scylla:
diff --git a/docker-compose.yaml b/docker-compose.yaml
@@ -2,25 +2,10 @@ services:
   spark-master:
     build: dockerfiles/spark
     command: master
-    container_name: spark-master
-    hostname: spark-master
     environment:
-      SPARK_PUBLIC_DNS: spark-master
-    networks:
-      - spark
-    expose:
-      - 7001
-      - 7002
-      - 7003
-      - 7004
-      - 7005
-      - 7006
-      - 7077
-      - 6066
+      SPARK_PUBLIC_DNS: localhost
     ports:
       - 4040:4040
-      - 6066:6066
-      - 7077:7077
       - 8080:8080
     volumes:
       - ./data/spark-master:/tmp/data
@@ -31,29 +16,15 @@ services:
   spark-worker:
     build: dockerfiles/spark
     command: worker
-    hostname: spark-worker
-    container_name: spark-worker-1
     environment:
       SPARK_WORKER_CORES: 3
       SPARK_WORKER_MEMORY: 1024m
       SPARK_WORKER_WEBUI_PORT: 8081
-      SPARK_PUBLIC_DNS: spark-worker
-    networks:
-      - spark
-    expose:
-      - 7012
-      - 7013
-      - 7014
-      - 7015
-      - 7016
-      - 8881
+      SPARK_PUBLIC_DNS: localhost
     ports:
       - 8081:8081
     volumes:
       - ./data/spark-worker:/tmp/data
       - ./parquet-data:/parquet-data
     depends_on:
       - spark-master
-
-networks:
-  spark:
diff --git a/docs/source/getting-started/docker.rst b/docs/source/getting-started/docker.rst
@@ -2,7 +2,9 @@
 Set Up a Spark Cluster with Docker
 ==================================
 
-This page describes how to set up a Spark cluster locally on your machine by using Docker containers. This approach is useful if you do not need a high-level of performance, and want to quickly try out the Migrator without having to set up a real cluster of nodes. It requires Docker and Git.
+This page describes how to set up a Spark cluster locally on your machine by using Docker containers. It requires Docker and Git.
+
+.. note:: This approach is useful if you do not need a high-level of performance, and want to quickly try out the Migrator without having to set up a real cluster of nodes. We recommend using a real cluster for production, though.
 
 1. Clone the Migrator repository.
 
@@ -31,13 +33,6 @@ This page describes how to set up a Spark cluster locally on your machine by usi
 
    http://localhost:8080
 
-   Tip: add the following aliases to your ``/etc/hosts`` to make links work in the Spark UI
-
-   .. code-block:: text
-
-     127.0.0.1   spark-master
-     127.0.0.1   spark-worker
-
 5. Rename the file ``config.yaml.example`` to ``config.yaml``, and `configure <../#configure-the-migration>`_ it according to your needs.
 
 6. Finally, run the migration.