feat: integrate operator into workspace

Prior to this change, the operator and runner projects were separate
cargo projects. Now they are a singular workspace and the yaml is
updated to deploy the operator into k8s.

Cargo.toml

@@ -0,0 +1,26 @@
+[workspace]
+members = ["operator", "runner", "common"]
+
+[workspace.dependencies]
+anyhow = "1"
+clap = { version = "4", features = ["derive", "env"] }
+env_logger = "0.10.0"
+keramik-common = { path = "./common/" }
+opentelemetry = { version = "0.18", features = [
+    "metrics",
+    "trace",
+    "rt-tokio",
+] }
+opentelemetry-otlp = { version = "0.11", features = [
+    "metrics",
+    "trace",
+    "tokio",
+] }
+tokio = { version = "1", features = ["full"] }
+tonic = { version = "0.8" }
+tracing = "0.1.37"
+tracing-opentelemetry = "0.18"
+tracing-subscriber = { version = "0.3", features = ["json", "env-filter"] }
+
+[patch.crates-io]
+goose = { git = "https://github.com/nathanielc/goose.git", branch = "fixes-0.16.4" }

Dockerfile_operator

@@ -0,0 +1,24 @@
+FROM public.ecr.aws/r5b3e0r5/3box/rust-builder:latest as chef
+
+RUN mkdir -p /home/builder/keramik/
+WORKDIR /home/builder/keramik/
+
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --bin operator --recipe-path recipe.json
+
+FROM chef AS builder
+COPY --from=planner /home/builder/keramik/recipe.json recipe.json
+
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Build application
+COPY . .
+RUN cargo build -p keramik-operator --release --locked
+
+FROM ubuntu:latest
+
+COPY --from=builder /home/builder/keramik/target/release/keramik-operator /usr/bin
+
+ENTRYPOINT ["/usr/bin/keramik-operator"]

Dockerfile_runner

@@ -0,0 +1,24 @@
+FROM public.ecr.aws/r5b3e0r5/3box/rust-builder:latest as chef
+
+RUN mkdir -p /home/builder/keramik/
+WORKDIR /home/builder/keramik/
+
+FROM chef AS planner
+COPY . .
+RUN cargo chef prepare --bin runner --recipe-path recipe.json
+
+FROM chef AS builder
+COPY --from=planner /home/builder/keramik/recipe.json recipe.json
+
+# Build dependencies - this is the caching Docker layer!
+RUN cargo chef cook --release --recipe-path recipe.json
+
+# Build application
+COPY . .
+RUN cargo build -p keramik-runner --release --locked
+
+FROM ubuntu:latest
+
+COPY --from=builder /home/builder/keramik/target/release/keramik-runner /usr/bin
+
+ENTRYPOINT ["/usr/bin/keramik-runner"]

README.md

@@ -22,30 +22,30 @@ Requires
 # Create a new kind cluster (i.e. local k8s)
 kind create cluster
 kubectl create ns keramik-0
-# Build the runner image and load it into kind
-docker build -t keramik/runner:dev runner/
-kind load docker-image keramik/runner:dev
+
+# Create CRDs
+cargo run --bin crdgen | kubectl apply -f -
+
 # Create new random secrets
 ./k8s/ceramic/create-secrets.sh
+
 # Start up the network
 kubectl apply -k ./k8s/ceramic
 ```
 
 View logs
 
-```
-kubectl logs ceramic-0 -c ceramic
-```
+    kubectl logs ceramic-0 -c ceramic
 
 ## AWS EKS
 
 Keramik can also be deployed against an AWS EKS cluster.
 This process is much the same, however the container images must be accessible to the EKS cluster.
 
-    $ kubectl create namespace keramik-0
-    $ ./k8s/ceramic/create-secrets.sh
-    $ kubectl apply -k ./k8s/ceramic/        # Start up ceramic cluster
-    $ kubectl apply -k ./k8s/opentelemetry/  # Start up monitoring infra
+    kubectl create namespace keramik-0
+    ./k8s/ceramic/create-secrets.sh
+    kubectl apply -k ./k8s/ceramic/        # Start up ceramic cluster
+    kubectl apply -k ./k8s/opentelemetry/  # Start up monitoring infra
 
 
 ## Change network size
@@ -58,9 +58,34 @@ The network size can be increase by changing the number of replicas for the cera
 The `runner` is a utility for running various jobs to initialize the network and run workloads against it.
 Any changes to the runner require that you rebuild it and load it into kind again.
 
-    docker build -t keramik/runner:dev runner/
+    docker buildx build -t keramik/runner:dev -f Dockerfile_runner .
     kind load docker-image keramik/runner:dev
 
+Now edit `./k8s/ceramic/kustomization.yaml` to use the `dev` tag
+
+```yaml
+images:
+  - name: keramik/runner
+    newTag: dev
+```
+
+
+## Operator
+
+The `operator` automates creating and manipulating networks via custom resource definition.
+Any changes to the operator require that you rebuild it and load it into kind again.
+
+    docker buildx build -t keramik/operator:dev -f Dockerfile_operator .
+    kind load docker-image keramik/operator:dev
+
+Now edit `./k8s/ceramic/kustomization.yaml` to use the `dev` tag
+
+```yaml
+images:
+  - name: keramik/operator
+    newTag: dev
+```
+
 ## Opentelemetry
 
 Add opentelemetry collector to the k8s cluster

ci-scripts/publish.sh

@@ -1,15 +1,20 @@
 #!/bin/bash
 
-# Build and publish a docker image run running ceramic-one
+# Build and publish a docker images
 #
-# DOCKER_PASSWORD must be set
 # Use:
 #
 #   export DOCKER_PASSWORD=$(aws ecr-public get-login-password --region us-east-1)
 #   echo "${DOCKER_PASSWORD}" | docker login --username AWS --password-stdin public.ecr.aws/r5b3e0r5
 #
-# to get a docker login password.
+# to setup docker login.
 
-docker buildx build -t 3box/keramik-runner runner
+# Build runner image
+docker buildx build -t 3box/keramik-runner -f Dockerfile_runner .
 docker tag 3box/keramik-runner:latest public.ecr.aws/r5b3e0r5/3box/keramik-runner:latest
 docker push public.ecr.aws/r5b3e0r5/3box/keramik-runner:latest
+
+# Build operator image
+docker buildx build -t 3box/keramik-operator -f Dockerfile_operator .
+docker tag 3box/keramik-operator:latest public.ecr.aws/r5b3e0r5/3box/keramik-operator:latest
+docker push public.ecr.aws/r5b3e0r5/3box/keramik-operator:latest

common/Cargo.toml

@@ -0,0 +1,18 @@
+[package]
+name = "keramik-common"
+version = "0.1.0"
+edition = "2021"
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+actix-web = "4.3.1"
+anyhow.workspace = true
+gethostname = "0.4.2"
+tokio.workspace = true
+tonic.workspace = true
+tracing.workspace = true
+tracing-opentelemetry.workspace = true
+tracing-subscriber.workspace = true
+opentelemetry.workspace = true
+opentelemetry-otlp.workspace = true

common/src/lib.rs

@@ -0,0 +1 @@
+pub mod telemetry;

runner/src/telemetry.rs → common/src/telemetry.rs

@@ -21,7 +21,14 @@ pub async fn init(otlp_endpoint: String) -> Result<BasicController> {
                 .with_endpoint(otlp_endpoint.clone()),
         )
         .with_trace_config(opentelemetry::sdk::trace::config().with_resource(
-            opentelemetry::sdk::Resource::new(vec![opentelemetry::KeyValue::new(
+            opentelemetry::sdk::Resource::new(vec![
+                opentelemetry::KeyValue::new(
+                    "hostname",
+                    gethostname::gethostname()
+                        .into_string()
+                        .expect("hostname should be valid utf-8"),
+                ),
+                opentelemetry::KeyValue::new(
                 "service.name",
                 "keramik",
             )]),

k8s/ceramic/kustomization.yaml

@@ -5,7 +5,7 @@ resources:
   - ./manifests/cas.yaml
   - ./manifests/ceramic.yaml
   - ./manifests/bootstrap.yaml
-  - ./manifests/network.yaml
+  - ./manifests/operator.yaml
   - ./manifests/simulate.yaml
 
 
@@ -37,3 +37,6 @@ images:
   - name: keramik/runner
     newName: public.ecr.aws/r5b3e0r5/3box/keramik-runner
     newTag: latest
+  - name: keramik/operator
+    newName: public.ecr.aws/r5b3e0r5/3box/keramik-operator
+    newTag: latest

k8s/ceramic/manifests/operator.yaml

@@ -0,0 +1,117 @@
+---
+# Scoped service account
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: keramik-operator
+  labels:
+    app: keramik-operator
+    app.kubernetes.io/name: keramik-operator
+    app.kubernetes.io/version: "0.12.5"
+  namespace: default
+automountServiceAccountToken: true
+---
+# Access for the service account
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: keramik-operator
+rules:
+  - apiGroups: ["keramik.3box.io"]
+    resources: ["networks", "networks/status"]
+    verbs: ["get", "list", "watch", "patch"]
+  - apiGroups: ["events.k8s.io"]
+    resources: ["events"]
+    verbs: ["create"]
+---
+# Binding the role to the account
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+  name: keramik-operator
+subjects:
+- kind: ServiceAccount
+  namespace: default
+  name: keramik-operator
+roleRef:
+  kind: ClusterRole
+  name: keramik-operator
+  apiGroup: rbac.authorization.k8s.io
+---
+# Expose the http port of the service
+apiVersion: v1
+kind: Service
+metadata:
+  name: keramik-operator
+  namespace: default
+  labels:
+    app: keramik-operator
+    app.kubernetes.io/name: keramik-operator
+    app.kubernetes.io/version: "0.12.5"
+spec:
+  type: ClusterIP
+  ports:
+  - port: 80
+    targetPort: 8080
+    protocol: TCP
+    name: http
+  selector:
+    app: keramik-operator
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: keramik-operator
+  namespace: default
+  labels:
+    app: keramik-operator
+    app.kubernetes.io/name: keramik-operator
+    app.kubernetes.io/version: "0.12.5"
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: keramik-operator
+  template:
+    metadata:
+      labels:
+        app: keramik-operator
+      annotations:
+        kubectl.kubernetes.io/default-container: keramik-operator
+    spec:
+      serviceAccountName: keramik-operator
+      securityContext:
+        {}
+      containers:
+      - name: keramik-operator
+        image: "keramik/operator"
+        imagePullPolicy: Always
+        command:
+          - "/usr/bin/keramik-operator"
+          - "daemon"
+        securityContext:
+          {}
+        resources:
+          limits:
+            cpu: 200m
+            memory: 256Mi
+          requests:
+            cpu: 50m
+            memory: 100Mi
+        ports:
+        - name: http
+          containerPort: 8080
+          protocol: TCP
+        env:
+        # We are pointing to tempo or grafana tracing agent's otlp grpc receiver port
+        - name: OPERATOR_OTLP_ENDPOINT
+          value: "https://otel:4317"
+        - name: RUST_LOG
+          value: "info,kube=debug,operator=debug"
+        #readinessProbe:
+        #  httpGet:
+        #    path: /health
+        #    port: http
+        #  initialDelaySeconds: 5
+        #  periodSeconds: 5
+