diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1aeee0351..441e7475a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,12 +2,13 @@ name: CI on: [push] -defaults: - run: - working-directory: sdk jobs: - lint_and_test: + lint_and_test_python_sdk: + defaults: + run: + working-directory: sdk + runs-on: ubuntu-latest strategy: max-parallel: 4 @@ -53,3 +54,25 @@ jobs: env: CI: true run: make tests + + lint_and_test_go_pkg: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Set up Go + uses: actions/setup-go@v3 + with: + go-version: "1.20" + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + + - name: Run Tests + run: | + make test-internal \ No newline at end of file diff --git a/Makefile b/Makefile index bc718f79f..a78bc3b5f 100644 --- a/Makefile +++ b/Makefile @@ -17,17 +17,17 @@ k3d-down: k3d cluster delete --config hack/k3d.yaml beam: - docker build . --target build --secret id=github-token,src=<(echo -n ${GITHUB_TOKEN}) -f ./docker/Dockerfile.beam -t localhost:5000/beam:$(imageVersion) + docker build . --target build -f ./docker/Dockerfile.beam -t localhost:5000/beam:$(imageVersion) docker push localhost:5000/beam:$(imageVersion) beam-worker: - docker build . --target final --build-arg BASE_STAGE=dev --secret id=github-token,src=<(echo -n ${GITHUB_TOKEN}) -f ./docker/Dockerfile.worker -t localhost:5000/beam-worker:$(imageVersion) + docker build . --target final --build-arg BASE_STAGE=dev -f ./docker/Dockerfile.worker -t localhost:5000/beam-worker:$(imageVersion) docker push localhost:5000/beam-worker:latest bin/delete_workers.sh beam-runner: for target in py312 py311 py310 py39 py38; do \ - docker build . --target $$target --platform=linux/amd64 --secret id=github-token,src=<(echo -n ${GITHUB_TOKEN}) -f ./docker/Dockerfile.runner -t localhost:5000/beam-runner:$$target-latest; \ + docker build . --target $$target --platform=linux/amd64 -f ./docker/Dockerfile.runner -t localhost:5000/beam-runner:$$target-latest; \ docker push localhost:5000/beam-runner:$$target-latest; \ done @@ -39,3 +39,6 @@ stop: protocol: cd proto && ./gen.sh + +test-internal: + go test -v ./internal/... -bench=./internal/.. diff --git a/README.md b/README.md index ca1b2f513..aa862cd26 100644 --- a/README.md +++ b/README.md @@ -12,12 +12,15 @@ Twitter + + Tests Passing + License

- The distributed container runtime + The distributed Python container runtime

--- diff --git a/docker/Dockerfile.beam b/docker/Dockerfile.beam index 6c2126d47..ce5e4eac6 100644 --- a/docker/Dockerfile.beam +++ b/docker/Dockerfile.beam @@ -13,11 +13,8 @@ WORKDIR /workspace RUN apt install -y libfuse3-dev && \ go install github.com/cosmtrek/air@latest -ENV GOPRIVATE=github.com/beam-cloud/* COPY go.mod go.sum ./ -RUN --mount=type=secret,id=github-token,required=true \ - echo "machine github.com login beam-cloud password $(cat /run/secrets/github-token)" > /root/.netrc && \ - go mod download && go mod verify +RUN go mod download && go mod verify COPY . . diff --git a/docker/Dockerfile.worker b/docker/Dockerfile.worker index 9b4592bbb..823377903 100644 --- a/docker/Dockerfile.worker +++ b/docker/Dockerfile.worker @@ -3,11 +3,10 @@ ARG BASE_STAGE=dev FROM golang:1.21-bullseye AS golang -RUN --mount=type=secret,id=github-token,required=true < /root/.netrc EOT @@ -58,8 +57,7 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN GOPRIVATE=github.com/beam-cloud/* \ - go build -o /usr/local/bin/worker ./cmd/worker/main.go +RUN go build -o /usr/local/bin/worker ./cmd/worker/main.go # NVIDIA CUDA - Final Stage diff --git a/internal/abstractions/function/http.go b/internal/abstractions/function/http.go index 3926a007a..159ec7c55 100644 --- a/internal/abstractions/function/http.go +++ b/internal/abstractions/function/http.go @@ -70,7 +70,7 @@ func (g *functionGroup) FunctionInvoke(ctx echo.Context) error { args, err := json.Marshal(payload) if err != nil { return ctx.JSON(http.StatusInternalServerError, map[string]interface{}{ - "error": "error marshaling function payload", + "error": "error marshalling function payload", }) } diff --git a/internal/abstractions/taskqueue/taskqueue.go b/internal/abstractions/taskqueue/taskqueue.go index ca7c607a9..8fdf43f99 100644 --- a/internal/abstractions/taskqueue/taskqueue.go +++ b/internal/abstractions/taskqueue/taskqueue.go @@ -592,6 +592,7 @@ func (tq *RedisTaskQueue) handleContainerEvents() { Change: -1, } } + case <-tq.ctx.Done(): return } diff --git a/sdk/src/beam/clients/bucket.py b/sdk/src/beam/clients/bucket.py deleted file mode 100644 index 9a7d853de..000000000 --- a/sdk/src/beam/clients/bucket.py +++ /dev/null @@ -1,28 +0,0 @@ -# Generated by the protocol buffer compiler. DO NOT EDIT! -# sources: bucket.proto -# plugin: python-betterproto -from dataclasses import dataclass - -import betterproto -import grpclib - - -@dataclass -class PutObjectRequest(betterproto.Message): - pass - - -@dataclass -class PutObjectResponse(betterproto.Message): - ok: bool = betterproto.bool_field(1) - - -class BucketServiceStub(betterproto.ServiceStub): - async def put_object(self) -> PutObjectResponse: - request = PutObjectRequest() - - return await self._unary_unary( - "/bucket.BucketService/PutObject", - request, - PutObjectResponse, - ) diff --git a/sdk/src/beam/type.py b/sdk/src/beam/type.py index 6fb4d399b..634ab6e37 100644 --- a/sdk/src/beam/type.py +++ b/sdk/src/beam/type.py @@ -37,7 +37,6 @@ class PythonVersion(str, Enum): ``` """ - Python37 = "python3.7" Python38 = "python3.8" Python39 = "python3.9" Python310 = "python3.10" @@ -49,15 +48,18 @@ class GpuType(str, Enum): """ An enum that defines types of GPUs. - - GPUs L4 and A100 are coming soon. Email us at founders@beam.cloud to learn more. - - Example: ```python - from beam import Runtime, GpuType + from beam import GpuType, function + + @function(gpu=GpuType.T4) + def some_func() + print("I will run on a T4 gpu!") - r = Runtime(gpu=GpuType.T4) + # This is equivalent to the above ^ + @function(gpu="T4") + def some_other_func() + print("I will run on a T4 gpu!") ``` """ @@ -68,42 +70,3 @@ class GpuType(str, Enum): A10G = "A10G" A100_40 = "A100-40" A100_80 = "A100-80" - - -class VolumeType(str, Enum): - """ - An enum that defines types of volumes. - - Example: - ```python - from beam import Volume, VolumeType - - pv = Volume( - name='my-persistent-data', - path='./my-persistent-volume' - volume_type=VolumeType.Persistent, - ) - ``` - """ - - Persistent = "persistent" - Shared = "shared" - - -class AutoscalingType(str, Enum): - """ - An enum that defines types of autoscaling. - - - This is deprecated. Please see the [RequestLatencyAutoscaler](#requestlatencyautoscaler). - - - Example: - ```python - from beam import Autoscaling, AutoscalingType - - a = Autoscaling(autoscaling_type=AutoscalingType.MaxRequestLatency) - ``` - """ - - MaxRequestLatency = "max_request_latency"