Add lessons course

.editorconfig

@@ -0,0 +1,17 @@
+; This file is for unifying the coding style for different editors and IDEs.
+; More information at http://editorconfig.org
+
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.scala]
+indent_size = 2
+indent_style = space
+
+[*.md]
+trim_trailing_whitespace = false

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/check-dependencies-updates.yml

@@ -0,0 +1,16 @@
+on:
+  schedule:
+    - cron: '0 6 * * 1-5'
+
+name: 🍄 Check dependencies updates
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  scala-steward:
+    runs-on: ubuntu-22.04
+    name: Check Scala project dependencies updates with Scala Steward
+    steps:
+      - uses: scala-steward-org/scala-steward-action@v2

.github/workflows/ci.yml

@@ -0,0 +1,25 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-java@v4
+        with:
+          distribution: 'zulu'
+          java-version: '21'
+          cache: 'sbt'
+      - name: 👌 Run "pre-push" tasks (compile and style-check)
+        run: sbt prep
+      - name: ✅ Run test
+        run: sbt test

.github/workflows/update-github-dependency-graph.yml

@@ -0,0 +1,16 @@
+name: Update GitHub Dependency Graph
+
+on:
+  push:
+    branches:
+      - main
+
+permissions:
+  contents: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: scalacenter/sbt-dependency-submission@v3

.gitignore

@@ -0,0 +1,8 @@
+target/
+boot/
+lib_managed/
+src_managed/
+project/plugins/project/
+
+#docker-compose
+volume/

.scalafmt.conf

@@ -0,0 +1,11 @@
+version = 3.8.2
+runner.dialect = scala213
+style = default
+maxColumn = 120
+continuationIndent.callSite = 2
+align.preset = more
+runner.optimizer.forceConfigStyleMinArgCount = 1
+rewrite.rules = [SortImports]
+importSelectors = singleLine
+project.excludeFilters = ["target/"]
+project.git = true # Only format files tracked by git

LICENSE

@@ -0,0 +1,21 @@
+The MIT License
+
+Copyright (c) 2024 com.codely, Inc. https://com.codely
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

README.md

@@ -0,0 +1,66 @@
+# ✨ spark-deploy-best-practices-course
+
+[![License](https://img.shields.io/github/license/com.codely/spark-deploy-best-practices-course?style=flat-square)](/LICENSE) 
+
+[![GitHub Repo stars](https://img.shields.io/github/stars/com.codely/spark-deploy-best-practices-course?style=flat-square)](https://github.com/com.codely/spark-deploy-best-practices-course/stargazers)
+
+[![Continuous Integration status](https://img.shields.io/github/actions/workflow/status/com.codely/spark-deploy-best-practices-course/ci.yml?style=flat-square)](https://github.com/com.codely/spark-deploy-best-practices-course/actions/workflows/ci.yml)
+
+## 🚀 Environment setup
+
+You only need the common tooling used for developing Scala applications:
+
+- [JDK](https://www.oracle.com/java/technologies/downloads/)
+- [SBT](https://www.scala-sbt.org/download)
+
+<details>
+<summary>Installing instructions for macOS with SDKMAN!</summary>
+
+If you use macOS, we would recommend using SDKMAN! to manage different JDK versions and tooling:
+
+1. [Install SDKMAN with homebrew](https://github.com/sdkman/homebrew-tap?tab=readme-ov-file#installation)
+2. Install the JDK you prefer. If you ask us, we will opt for:
+   1. [Check the latest Java LTS JDK version](https://endoflife.date/oracle-jdk)
+   2. Check the latest Zulu distribution version for that LTS with:
+      ```shell
+      sdk list java
+      ```
+   3. Install it:
+      ```shell
+      sdk install java XX.YY.ZZ-zulu
+      ```
+3. Install the latest SBT:
+   ```shell
+   sdk install sbt
+   ```
+</details>
+
+## ✅ Tests
+
+Just run:
+
+```shell
+sbt test
+```
+
+There is also the `sbt t` alias 😊
+
+## 🤽‍ Pre-push Git hook
+
+There's one Git hook included. It's inside the `doc/hooks` folder, and it will run the `prep` SBT task before pushing to any remote.
+
+This `prep` task is intended to run all the checks you consider before pushing.
+At this very moment, it tries to compile and check the code style rules with ScalaFmt.
+
+You can define what this task does by modifying the `prep` task in the `build.sbt` file.
+We like the approach of just having to run 1 single SBT task instead of multiple tasks because it's more efficient (the hook doesn't have to create multiple SBT sessions), and also because this way we can control the pre push tasks with the SBT alias defined at the `build.sbt` without altering the hooks.
+
+If you want to install this hook, just `cd doc/hooks` and run `./install-hooks.sh`.
+
+## ⚖️ License
+
+See [`LICENSE`](LICENSE).
+
+## 🤔 About
+
+Project developed by [com.codely, Inc.](https://com.codely) and bootstrapped with the [Codely Scala Spark Skeleton (powered by Giter 8)](https://github.com/CodelyTV/scala-spark-skeleton.g8).

build.sbt

@@ -0,0 +1,14 @@
+Settings.settings
+
+libraryDependencies := Dependencies.all
+
+SbtAliases.aliases.flatMap { case (alias, command) =>
+  addCommandAlias(alias, command)
+}
+
+assembly / assemblyMergeStrategy := {
+  case PathList("META-INF", xs @ _*) => MergeStrategy.discard
+  case PathList("org", "apache", "spark", "unused", "UnusedStubClass.class") =>
+    MergeStrategy.first
+  case _ => MergeStrategy.first
+}

doc/hooks/install-hooks.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+cd "$(dirname "$0")/../.."
+
+rm -rf .git/hooks
+
+ln -s ../doc/hooks .git/hooks
+sudo chmod -R 777 doc/hooks/*

doc/hooks/pre-push

@@ -0,0 +1,50 @@
+#!/bin/bash
+
+# Checks if locally staged changes are formatted properly ignoring non-staged changes.
+# Install it with the `install-hooks.sh` script
+# Based on: https://gist.github.com/cvogt/2676ed6c6d1abafa3d6a
+
+PATH=$PATH:/usr/local/bin:/usr/local/sbin
+
+echo ""
+echo "Running pre-push hook… (you can omit this with --no-verify, but don't)"
+
+echo "* Moving to the project directory…"
+_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+DIR=$( echo $_DIR | sed 's/\/.git\/hooks$//' )
+
+echo "* Stashing non-staged changes so we avoid checking them…"
+git diff --quiet
+hadNoNonStagedChanges=$?
+
+if ! [ $hadNoNonStagedChanges -eq 0 ]
+then
+	git stash --keep-index -u > /dev/null
+fi
+
+echo "* Checking pre push conditions ('prep' SBT task)…"
+sbt prep > /dev/null
+canPush=$?
+
+if [ $canPush -ne 0 ]
+then
+	echo "  [KO] Error :("
+fi
+
+echo "* Applying the stash with the non-staged changes…"
+if ! [ $hadNoNonStagedChanges -eq 0 ]
+then
+	sleep 1 && git stash pop --index > /dev/null & # sleep because otherwise commit fails when this leads to a merge conflict
+fi
+
+# Final result
+echo ""
+
+if [ $canPush -eq 0 ]
+then
+    echo "[OK] Your code will be pushed young Padawan"
+    exit 0
+else
+    echo "[KO] Cancelling push due to test code style error (run 'sbt prep' for more information)"
+    exit 1
+fi

docker-compose.yml

@@ -0,0 +1,75 @@
+name: "kafka-rabbitmq-sqs"
+
+networks:
+  kafka-rabbitmq-sqs:
+    name: kafka-rabbitmq-sqs
+    driver: bridge
+    ipam:
+      driver: default
+      config:
+        - subnet: "172.18.0.0/16"
+
+services:
+  kafka:
+    image: bitnami/kafka:3.7.0
+    container_name: kafka
+    ports:
+      - "9092:9092"
+    environment:
+      KAFKA_CFG_NODE_ID: 0
+      KAFKA_CFG_PROCESS_ROLES: controller,broker
+      KAFKA_CFG_LISTENERS: "PLAINTEXT://172.18.0.10:9092,CONTROLLER://0.0.0.0:9093"
+      KAFKA_CFG_LISTENER_SECURITY_PROTOCOL_MAP: "CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT"
+      KAFKA_CFG_CONTROLLER_QUORUM_VOTERS: [email protected]:9093
+      KAFKA_CFG_CONTROLLER_LISTENER_NAMES: CONTROLLER
+    networks:
+      kafka-rabbitmq-sqs:
+        ipv4_address: 172.18.0.10
+
+  init-kafka:
+    image: bitnami/kafka:3.7.0
+    container_name: init-kafka
+    depends_on:
+      kafka:
+        condition: service_started
+    entrypoint: [ '/usr/bin/bash', '-c' ]
+    command: |
+      "
+      set -ex
+
+      # blocks until kafka is reachable
+      /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server 172.18.0.10:9092 --list
+
+      echo -e 'Creating kafka topics'
+      /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server 172.18.0.10:9092 --create --if-not-exists --topic topic-events --replication-factor 1 --partitions 1
+
+      echo -e 'Successfully created the following topics:'
+      /opt/bitnami/kafka/bin/kafka-topics.sh --bootstrap-server 172.18.0.10:9092 --list
+      "
+    networks:
+      kafka-rabbitmq-sqs:
+        ipv4_address: 172.18.0.20
+
+  localstack:
+    container_name: "${LOCALSTACK_DOCKER_NAME:-localstack-main}"
+    image: localstack/localstack:3.4.0
+    ports:
+      - "127.0.0.1:4566:4566"
+      - "127.0.0.1:4510-4559:4510-4559"
+    environment:
+      - SERVICES=events,sqs
+      - DEBUG=true
+      - DEFAULT_REGION=eu-west-1
+    volumes:
+      - "${LOCALSTACK_VOLUME_DIR:-./volume}:/var/lib/localstack"
+      - "/var/run/docker.sock:/var/run/docker.sock"
+
+  rabbitmq:
+    image: 'rabbitmq:3.12-management'
+    restart: unless-stopped
+    ports:
+      - "5672:5672"
+      - "15672:15672"
+    environment:
+      - RABBITMQ_DEFAULT_USER=codely
+      - RABBITMQ_DEFAULT_PASS=codely

etc/rabbitmq/0-install.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+curl -O http://localhost:15672/cli/rabbitmqadmin
+chmod +x rabbitmqadmin

etc/rabbitmq/1-configure.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./rabbitmqadmin -u codely -p codely -H localhost -P 15672 declare queue name=spark_queue durable=false

etc/rabbitmq/2-list_queues.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./rabbitmqadmin -u codely -p codely -H localhost -P 15672 list queues

etc/rabbitmq/3-publish_message.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+./rabbitmqadmin -u codely -p codely -H localhost -P 15672 publish exchange=amq.default routing_key=spark_queue payload="Hello, World!"

etc/rabbitmq/4-consume_message.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+
+QUEUE_NAME="spark_queue"
+USER="codely"
+PASS="codely"
+HOST="localhost"
+PORT="15672"
+
+# Consume a message from the queue without auto-ack
+response=$(curl -u $USER:$PASS -H "content-type:application/json" \
+    -X POST -d '{"count":1,"ackmode":"ack_requeue_true","encoding":"auto","truncate":50000}' \
+    http://$HOST:$PORT/api/queues/%2f/$QUEUE_NAME/get)
+
+# Print the message and delivery_tag
+echo "Message: $(echo $response | jq -r '.[0].payload')"

etc/rabbitmq/5-delete_messages.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+QUEUE_NAME="spark_queue"
+USER="codely"
+PASS="codely"
+HOST="localhost"
+PORT="15672"
+
+# Purge all messages from the queue
+curl -u $USER:$PASS -H "content-type:application/json" \
+    -X DELETE http://$HOST:$PORT/api/queues/%2f/$QUEUE_NAME/contents