This repository has been archived by the owner on Aug 28, 2024. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 14
Support for local kind cluster #190
Merged
Changes from all commits
Commits
Show all changes
10 commits
Select commit
Hold shift + click to select a range
fa759af
Checkpoint
nstogner 9362028
Checkpoint
nstogner c19491b
Add local registry to kind
nstogner 522cd38
Install manifests
nstogner 2162536
Add comment
nstogner 8d3ac9b
Cleanup
nstogner 4053dcb
Add Kind arch diagram
nstogner 3481452
Fix diagram
nstogner d7e9445
Fixy
nstogner ec95b7b
Cleanup
nstogner File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,27 @@ | ||
# Start from the latest go base image | ||
FROM golang:1.19 AS builder | ||
ARG TARGETOS=linux | ||
ARG TARGETARCH=amd64 | ||
|
||
WORKDIR /workspace | ||
COPY go.mod go.sum ./ | ||
RUN go mod download | ||
|
||
COPY cmd/sci-kind/main.go cmd/sci-kind/main.go | ||
COPY internal/ internal/ | ||
|
||
# Build the app | ||
RUN CGO_ENABLED=0 GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ | ||
go build -a -o main cmd/sci-kind/main.go | ||
|
||
FROM gcr.io/distroless/static:nonroot | ||
WORKDIR / | ||
|
||
# Copy the Pre-built binary file from the previous stage | ||
COPY --from=builder /workspace/main . | ||
USER root | ||
EXPOSE 10080 | ||
EXPOSE 8080 | ||
|
||
# run the executable | ||
CMD ["/main"] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,59 @@ | ||
package main | ||
|
||
import ( | ||
"flag" | ||
"fmt" | ||
"log" | ||
"net" | ||
"net/http" | ||
|
||
"github.com/substratusai/substratus/internal/sci" | ||
scikind "github.com/substratusai/substratus/internal/sci/kind" | ||
"google.golang.org/grpc" | ||
"google.golang.org/grpc/health" | ||
hv1 "google.golang.org/grpc/health/grpc_health_v1" | ||
) | ||
|
||
func main() { | ||
var cfg struct { | ||
port int | ||
signedURLPort int | ||
hostSignedURLAddress string | ||
} | ||
flag.IntVar(&cfg.port, "port", 10080, "port number to listen on") | ||
flag.IntVar(&cfg.signedURLPort, "signed-url-port", 8080, "port to listen for signed url traffic") | ||
flag.StringVar(&cfg.hostSignedURLAddress, "host-signed-url-address", "http://localhost:30080", | ||
"host address that port forwards to the signed url port within the cluster. this should be set in kind config.yaml.") | ||
flag.Parse() | ||
|
||
s := &scikind.Server{ | ||
SignedURLAddress: cfg.hostSignedURLAddress, | ||
} | ||
signedURLServer := &http.Server{ | ||
Addr: fmt.Sprintf(":%v", cfg.signedURLPort), | ||
Handler: s, | ||
} | ||
go func() { | ||
log.Printf("Listening for signed URL traffic on address: %v", cfg.signedURLPort) | ||
log.Fatal(signedURLServer.ListenAndServe()) | ||
}() | ||
|
||
gs := grpc.NewServer() | ||
sci.RegisterControllerServer(gs, s) | ||
|
||
// Setup Health Check | ||
hs := health.NewServer() | ||
hs.SetServingStatus("", hv1.HealthCheckResponse_SERVING) | ||
hv1.RegisterHealthServer(gs, hs) | ||
|
||
addr := fmt.Sprintf(":%v", cfg.port) | ||
log.Printf("Listening for gRPC traffic on address: %v", addr) | ||
lis, err := net.Listen("tcp", addr) | ||
if err != nil { | ||
log.Fatalf("failed to listen: %v", err) | ||
} | ||
|
||
if err := gs.Serve(lis); err != nil { | ||
log.Fatalf("failed to serve: %v", err) | ||
} | ||
} |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,58 @@ | ||
# This patch inject a sidecar container which is a HTTP proxy for the | ||
# controller manager, it performs RBAC authorization against the Kubernetes API using SubjectAccessReviews. | ||
apiVersion: apps/v1 | ||
kind: Deployment | ||
metadata: | ||
name: controller-manager | ||
namespace: system | ||
spec: | ||
template: | ||
spec: | ||
affinity: | ||
nodeAffinity: | ||
requiredDuringSchedulingIgnoredDuringExecution: | ||
nodeSelectorTerms: | ||
- matchExpressions: | ||
- key: kubernetes.io/arch | ||
operator: In | ||
values: | ||
- amd64 | ||
- arm64 | ||
- ppc64le | ||
- s390x | ||
- key: kubernetes.io/os | ||
operator: In | ||
values: | ||
- linux | ||
containers: | ||
- name: kube-rbac-proxy | ||
securityContext: | ||
allowPrivilegeEscalation: false | ||
capabilities: | ||
drop: | ||
- "ALL" | ||
image: gcr.io/kubebuilder/kube-rbac-proxy:v0.13.1 | ||
args: | ||
- "--secure-listen-address=0.0.0.0:8443" | ||
- "--upstream=http://127.0.0.1:8080/" | ||
- "--logtostderr=true" | ||
- "--v=0" | ||
ports: | ||
- containerPort: 8443 | ||
protocol: TCP | ||
name: https | ||
resources: | ||
limits: | ||
cpu: 500m | ||
memory: 128Mi | ||
requests: | ||
cpu: 5m | ||
memory: 64Mi | ||
- name: manager | ||
envFrom: | ||
- configMapRef: | ||
name: system | ||
args: | ||
- "--health-probe-bind-address=:8081" | ||
- "--metrics-bind-address=127.0.0.1:8080" | ||
- "--leader-elect" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,9 @@ | ||
apiVersion: v1 | ||
kind: ConfigMap | ||
metadata: | ||
name: system | ||
namespace: substratus | ||
data: | ||
CLOUD: kind | ||
CLUSTER_NAME: substratus | ||
PRINCIPAL: unused |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,32 @@ | ||
# Adds namespace to all resources. | ||
namespace: substratus | ||
|
||
# Labels to add to all resources and selectors. | ||
#labels: | ||
#- includeSelectors: true | ||
# pairs: | ||
# someName: someValue | ||
|
||
resources: | ||
- ./namespace.yaml | ||
- ./config.yaml | ||
- ../crd | ||
- ../rbac | ||
- ../manager | ||
- ../registry-kind | ||
- ../sci-kind | ||
# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in | ||
# crd/kustomization.yaml | ||
#- ../webhook | ||
# [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required. | ||
#- ../certmanager | ||
# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. | ||
#- ../prometheus | ||
|
||
# Protect the /metrics endpoint by putting it behind auth. | ||
# If you want your controller-manager to expose the /metrics | ||
# endpoint w/o any authn/z, please comment the following line. | ||
patches: | ||
- path: manager_patch.yaml | ||
apiVersion: kustomize.config.k8s.io/v1beta1 | ||
kind: Kustomization |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,12 @@ | ||
apiVersion: v1 | ||
kind: Namespace | ||
metadata: | ||
name: substratus | ||
labels: | ||
control-plane: controller-manager | ||
app.kubernetes.io/name: namespace | ||
app.kubernetes.io/instance: system | ||
app.kubernetes.io/component: manager | ||
app.kubernetes.io/created-by: substratus | ||
app.kubernetes.io/part-of: substratus | ||
app.kubernetes.io/managed-by: kustomize |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,34 @@ | ||
apiVersion: v1 | ||
nstogner marked this conversation as resolved.
Show resolved
Hide resolved
|
||
kind: ConfigMap | ||
metadata: | ||
name: configure-cri | ||
data: | ||
configure-cri.sh: | | ||
#!/usr/bin/env bash | ||
|
||
set -x | ||
# Exit on non-existant variable. | ||
set -u | ||
# Exit on error. | ||
set -e | ||
|
||
export IMAGE_REGISTRY=$REGISTRY_PORT_5000_TCP_ADDR:5000 | ||
|
||
if ! grep -q $IMAGE_REGISTRY /mnt/etc/containerd/config.toml; then | ||
containerd_version=$(nsenter --target 1 --mount bash -c "containerd --version | awk '{ print substr(\$3,0,4) }'") | ||
if [ "$containerd_version" = "1.3." ] || [ "$containerd_version" = "1.4." ]; then | ||
cat <<EOF >> /mnt/etc/containerd/config.toml | ||
[plugins.cri.registry.configs."$IMAGE_REGISTRY"] | ||
endpoint = ["http://$IMAGE_REGISTRY"] | ||
EOF | ||
else | ||
# Correct config for containerd 1.5 and above | ||
cat <<EOF >> /mnt/etc/containerd/config.toml | ||
[plugins."io.containerd.grpc.v1.cri".registry.mirrors."$IMAGE_REGISTRY"] | ||
endpoint = ["http://$IMAGE_REGISTRY"] | ||
EOF | ||
fi | ||
nsenter --target 1 --mount bash -c "systemctl is-active --quiet containerd && echo 'Restarting containerd' && systemctl restart containerd" | ||
# Wait for containerd to be ready so that skaffold doesn't fail. | ||
nsenter --target 1 --mount bash -c "while ! ctr -n k8s.io containers ls; do sleep 1; done" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. probably can be removed |
||
fi |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,41 @@ | ||
apiVersion: apps/v1 | ||
kind: DaemonSet | ||
metadata: | ||
name: configure-cri | ||
labels: | ||
app: configure-cri | ||
spec: | ||
selector: | ||
matchLabels: | ||
app: configure-cri | ||
template: | ||
metadata: | ||
labels: | ||
app: configure-cri | ||
spec: | ||
hostPID: true | ||
initContainers: | ||
- name: configure-cri | ||
image: ubuntu:22.04 | ||
command: ["/scripts/configure-cri.sh"] | ||
volumeMounts: | ||
- name: etc | ||
mountPath: "/mnt/etc" | ||
- mountPath: /scripts | ||
name: scripts | ||
securityContext: | ||
privileged: true | ||
volumes: | ||
- name: etc | ||
hostPath: | ||
path: /etc | ||
- name: scripts | ||
configMap: | ||
name: configure-cri | ||
defaultMode: 0744 | ||
containers: | ||
- name: pause | ||
image: gcr.io/google_containers/pause | ||
tolerations: | ||
- effect: NoSchedule | ||
operator: Exists |
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I think this should be included by default in all installation not just kind?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
I have done this in my PR/branch for now as well: 01d2ec1
Approach I took is add namespace.yaml under config/manager. I thought it wasn't worth creating a separate directory.