Dockerfile

FROM deltaio/delta-docker:latest
# latest is at 1.0.0_3.0.0, 

USER root

COPY ./requirements.txt ./
RUN pip install -r requirements.txt

# Copy SQL DDL and DML to SPARK_HOME dir
COPY ./setup.sql ./
COPY ./count.sql ./
COPY ./create_buckets.py ./

# Copy tpch data generator into the container
COPY ./tpch-dbgen ./tpch-dbgen/

RUN apt-get update && \
    apt-get install -y --no-install-recommends \
    rsync && \
    apt-get clean && \
    rm -rf /var/lib/apt/lists/*

COPY ./conf/spark-defaults.conf "$SPARK_HOME/conf/spark-defaults.conf"
COPY ./conf/metrics.properties "$SPARK_HOME/conf/metrics.properties"
ENV SPARK_CONF_DIR="$SPARK_HOME/conf"
ENV SPARK_MASTER="spark://spark-master:7077"
ENV SPARK_MASTER_HOST spark-master
ENV SPARK_MASTER_PORT 7077
ENV PYSPARK_PYTHON python3

# Create and event logging directory to store job logs
RUN mkdir /tmp/spark-events

RUN chmod u+x /opt/spark/sbin/* && \
    chmod u+x /opt/spark/bin/*

ENV PYTHONPATH=$SPARK_HOME/python/:$PYTHONPATH

COPY entrypoint.sh .

ENTRYPOINT ["./entrypoint.sh"]