Skip to content

Commit

Permalink
fix emr serverless python deps sample dockerfile (#255)
Browse files Browse the repository at this point in the history
---------

Co-authored-by: Mathieu Cloutier <[email protected]>
  • Loading branch information
HarshCasper and cloutierMat authored Oct 6, 2024
1 parent ec34fa0 commit 636c092
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 8 deletions.
1 change: 1 addition & 0 deletions emr-serverless-python-dependencies/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
pyspark_env
10 changes: 6 additions & 4 deletions emr-serverless-python-dependencies/Dockerfile-localstack
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,11 @@ COPY . .
## Build and export stages for standard Python projects
## ----------------------------------------------------------------------------
# Build stage - installs required dependencies and creates a venv package
FROM base as build-poetry
RUN poetry self add poetry-plugin-bundle && \
poetry bundle venv dist/bundle
FROM base AS build-poetry
WORKDIR /app

RUN poetry self add [email protected]
RUN poetry bundle venv /app/dist/bundle --clear

FROM scratch as export-poetry
FROM scratch AS export-poetry
COPY --from=build-poetry /app/dist/bundle /pyspark_env/
12 changes: 8 additions & 4 deletions emr-serverless-python-dependencies/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ build-aws:
docker build . --file Dockerfile-aws --output .

deploy:
docker-compose up --detach
docker compose up --detach
terraform workspace select local
AWS_ENDPOINT_URL=https://localhost.localstack.cloud:4566 terraform apply --auto-approve

Expand All @@ -31,7 +31,11 @@ run-aws:
./start_job.sh aws

stop:
docker-compose down
docker compose down

logs:
@localstack logs > logs.txt
cat logs.txt

destroy:
terraform workspace select local
Expand All @@ -44,5 +48,5 @@ destroy-aws:
terraform destroy --auto-approve

test-ci:
make init build deploy run; return_code=`echo $$?`;\
make stop; exit $$return_code;
make init build deploy logs run; return_code=`echo $$?`;\
make logs; make stop; exit $$return_code;

0 comments on commit 636c092

Please sign in to comment.