From 5a54ab807712a0fcc41f9fcaa977ee219dc37729 Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:22:56 -0500 Subject: [PATCH 1/8] clean up sdk a bit --- internal/abstractions/function/http.go | 2 +- internal/abstractions/taskqueue/taskqueue.go | 1 + sdk/src/beam/clients/bucket.py | 28 ---------- sdk/src/beam/type.py | 55 ++++---------------- 4 files changed, 11 insertions(+), 75 deletions(-) delete mode 100644 sdk/src/beam/clients/bucket.py diff --git a/internal/abstractions/function/http.go b/internal/abstractions/function/http.go index 3926a007a..159ec7c55 100644 --- a/internal/abstractions/function/http.go +++ b/internal/abstractions/function/http.go @@ -70,7 +70,7 @@ func (g *functionGroup) FunctionInvoke(ctx echo.Context) error { args, err := json.Marshal(payload) if err != nil { return ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "error": "error marshaling function payload", + "error": "error marshalling function payload", }) } diff --git a/internal/abstractions/taskqueue/taskqueue.go b/internal/abstractions/taskqueue/taskqueue.go index ca7c607a9..8fdf43f99 100644 --- a/internal/abstractions/taskqueue/taskqueue.go +++ b/internal/abstractions/taskqueue/taskqueue.go @@ -592,6 +592,7 @@ func (tq *RedisTaskQueue) handleContainerEvents() { Change: -1, } } + case <-tq.ctx.Done(): return } diff --git a/sdk/src/beam/clients/bucket.py b/sdk/src/beam/clients/bucket.py deleted file mode 100644 index 9a7d853de..000000000 --- a/sdk/src/beam/clients/bucket.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# sources: bucket.proto -# plugin: python-betterproto -from dataclasses import dataclass - -import betterproto -import grpclib - - -@dataclass -class PutObjectRequest(betterproto.Message): - pass - - -@dataclass -class PutObjectResponse(betterproto.Message): - ok: bool = betterproto.bool_field(1) - - -class BucketServiceStub(betterproto.ServiceStub): - async def put_object(self) -> PutObjectResponse: - request = PutObjectRequest() - - return await self._unary_unary( - "/bucket.BucketService/PutObject", - request, - PutObjectResponse, - ) diff --git a/sdk/src/beam/type.py b/sdk/src/beam/type.py index 6fb4d399b..634ab6e37 100644 --- a/sdk/src/beam/type.py +++ b/sdk/src/beam/type.py @@ -37,7 +37,6 @@ class PythonVersion(str, Enum): ``` """ - Python37 = "python3.7" Python38 = "python3.8" Python39 = "python3.9" Python310 = "python3.10" @@ -49,15 +48,18 @@ class GpuType(str, Enum): """ An enum that defines types of GPUs. - - GPUs L4 and A100 are coming soon. Email us at founders@beam.cloud to learn more. - - Example: ```python - from beam import Runtime, GpuType + from beam import GpuType, function + + @function(gpu=GpuType.T4) + def some_func() + print("I will run on a T4 gpu!") - r = Runtime(gpu=GpuType.T4) + # This is equivalent to the above ^ + @function(gpu="T4") + def some_other_func() + print("I will run on a T4 gpu!") ``` """ @@ -68,42 +70,3 @@ class GpuType(str, Enum): A10G = "A10G" A100_40 = "A100-40" A100_80 = "A100-80" - - -class VolumeType(str, Enum): - """ - An enum that defines types of volumes. - - Example: - ```python - from beam import Volume, VolumeType - - pv = Volume( - name='my-persistent-data', - path='./my-persistent-volume' - volume_type=VolumeType.Persistent, - ) - ``` - """ - - Persistent = "persistent" - Shared = "shared" - - -class AutoscalingType(str, Enum): - """ - An enum that defines types of autoscaling. - - - This is deprecated. Please see the [RequestLatencyAutoscaler](#requestlatencyautoscaler). - - - Example: - ```python - from beam import Autoscaling, AutoscalingType - - a = Autoscaling(autoscaling_type=AutoscalingType.MaxRequestLatency) - ``` - """ - - MaxRequestLatency = "max_request_latency" From 06bba2bcf59f877a2cd71a26f824a6ff6206c6fc Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:26:28 -0500 Subject: [PATCH 2/8] wip --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index ca1b2f513..50cfe3734 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@

- The distributed container runtime + The distributed Python container runtime

--- From 35561fab966a9333259bac194731e26c9db89f9c Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:30:12 -0500 Subject: [PATCH 3/8] add go tests to ci --- .github/workflows/ci.yml | 24 +++++++++++++++++++++++- Makefile | 3 +++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aeee0351..99ac3d182 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -7,7 +7,7 @@ defaults: working-directory: sdk jobs: - lint_and_test: + lint_and_test_python_sdk: runs-on: ubuntu-latest strategy: max-parallel: 4 @@ -53,3 +53,25 @@ jobs: env: CI: true run: make tests + + lint_and_test_go_pkg: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Run Tests + run: | + make test-internal \ No newline at end of file diff --git a/Makefile b/Makefile index bc718f79f..00c887fe1 100644 --- a/Makefile +++ b/Makefile @@ -39,3 +39,6 @@ stop: protocol: cd proto && ./gen.sh + +test-internal: + go test -v ./internal/... -bench=./internal/.. From a13cb310ad6145e183d127c7b0f2e480780efa9b Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:32:15 -0500 Subject: [PATCH 4/8] add tests badge --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 50cfe3734..aa862cd26 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,9 @@ Twitter + + Tests Passing + License

From 18221b1017c7edd59478136c6820984404f71241 Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:35:55 -0500 Subject: [PATCH 5/8] wip --- .github/workflows/ci.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 99ac3d182..9134db904 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,12 +2,10 @@ name: CI on: [push] -defaults: - run: - working-directory: sdk jobs: lint_and_test_python_sdk: + working-directory: sdk runs-on: ubuntu-latest strategy: max-parallel: 4 From ca1e368db39a0e75a834dba777a614929aec8160 Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:38:34 -0500 Subject: [PATCH 6/8] move default dir --- .github/workflows/ci.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 9134db904..441e7475a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,7 +5,10 @@ on: [push] jobs: lint_and_test_python_sdk: - working-directory: sdk + defaults: + run: + working-directory: sdk + runs-on: ubuntu-latest strategy: max-parallel: 4 From 186d44bfae689fdb6f609369e34343c6f5e25edf Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:51:16 -0500 Subject: [PATCH 7/8] remove github tokens --- Makefile | 6 +++--- docker/Dockerfile.beam | 4 +--- docker/Dockerfile.worker | 1 - 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 00c887fe1..a78bc3b5f 100644 --- a/Makefile +++ b/Makefile @@ -17,17 +17,17 @@ k3d-down: k3d cluster delete --config hack/k3d.yaml beam: - docker build . --target build --secret id=github-token,src=<(echo -n ${GITHUB_TOKEN}) -f ./docker/Dockerfile.beam -t localhost:5000/beam:$(imageVersion) + docker build . --target build -f ./docker/Dockerfile.beam -t localhost:5000/beam:$(imageVersion) docker push localhost:5000/beam:$(imageVersion) beam-worker: - docker build . --target final --build-arg BASE_STAGE=dev --secret id=github-token,src=<(echo -n ${GITHUB_TOKEN}) -f ./docker/Dockerfile.worker -t localhost:5000/beam-worker:$(imageVersion) + docker build . --target final --build-arg BASE_STAGE=dev -f ./docker/Dockerfile.worker -t localhost:5000/beam-worker:$(imageVersion) docker push localhost:5000/beam-worker:latest bin/delete_workers.sh beam-runner: for target in py312 py311 py310 py39 py38; do \ - docker build . --target $$target --platform=linux/amd64 --secret id=github-token,src=<(echo -n ${GITHUB_TOKEN}) -f ./docker/Dockerfile.runner -t localhost:5000/beam-runner:$$target-latest; \ + docker build . --target $$target --platform=linux/amd64 -f ./docker/Dockerfile.runner -t localhost:5000/beam-runner:$$target-latest; \ docker push localhost:5000/beam-runner:$$target-latest; \ done diff --git a/docker/Dockerfile.beam b/docker/Dockerfile.beam index 6c2126d47..014726fe5 100644 --- a/docker/Dockerfile.beam +++ b/docker/Dockerfile.beam @@ -15,9 +15,7 @@ RUN apt install -y libfuse3-dev && \ ENV GOPRIVATE=github.com/beam-cloud/* COPY go.mod go.sum ./ -RUN --mount=type=secret,id=github-token,required=true \ - echo "machine github.com login beam-cloud password $(cat /run/secrets/github-token)" > /root/.netrc && \ - go mod download && go mod verify +RUN go mod download && go mod verify COPY . . diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 9b4592bbb..33642729b 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -7,7 +7,6 @@ RUN --mount=type=secret,id=github-token,required=true < /root/.netrc EOT From 4853aa57f5f9e411d0f946a7887cad1215b7e757 Mon Sep 17 00:00:00 2001 From: Luke Lombardi Date: Fri, 12 Jan 2024 09:53:31 -0500 Subject: [PATCH 8/8] remove github tokens --- docker/Dockerfile.beam | 1 - docker/Dockerfile.worker | 5 ++--- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.beam b/docker/Dockerfile.beam index 014726fe5..ce5e4eac6 100644 --- a/docker/Dockerfile.beam +++ b/docker/Dockerfile.beam @@ -13,7 +13,6 @@ WORKDIR /workspace RUN apt install -y libfuse3-dev && \ go install github.com/cosmtrek/air@latest -ENV GOPRIVATE=github.com/beam-cloud/* COPY go.mod go.sum ./ RUN go mod download && go mod verify diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 33642729b..823377903 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -3,7 +3,7 @@ ARG BASE_STAGE=dev FROM golang:1.21-bullseye AS golang -RUN --mount=type=secret,id=github-token,required=true <