diff --git a/solutions/join-through-proxy/README.md b/solutions/join-through-proxy/README.md new file mode 100644 index 000000000..6cf2a9134 --- /dev/null +++ b/solutions/join-through-proxy/README.md @@ -0,0 +1,63 @@ +# Join hub through a forward proxy server + +This doc shows how a cluster joins an Open Cluster Management (OCM) hub cluster through a forward proxy server. + +## Prerequisite + +- [kind](https://kind.sigs.k8s.io) must be installed on your local machine. The Kubernetes version must be >= 1.19, see [kind user guide](https://kind.sigs.k8s.io/docs/user/quick-start/#creating-a-cluster) for more details. + +- Download and install [clusteradm](https://github.com/open-cluster-management-io/clusteradm/releases). For Linux OS, run the following commands: + + ``` + wget -qO- https://github.com/open-cluster-management-io/clusteradm/releases/latest/download/clusteradm_linux_amd64.tar.gz | sudo tar -xvz -C /usr/local/bin/ + + sudo chmod +x /usr/local/bin/clusteradm + ``` + +## Setup the proxy server + +Run `./setup-proxy.sh`. It creates a Kind cluster and runs an open source proxy server [squid](http://www.squid-cache.org/) on it. HTTP GET method is enabled on this proxy server and it is serving on ports 31280 and 31290 for HTTP and HTTPS service respectively. +```bash +kubectl -n squid get svc -o wide +NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE SELECTOR +squid NodePort 10.96.223.82 3128:31280/TCP,3129:31290/TCP 3h49m app=squid +``` + +## Setup OCM hub cluster and managed clusters + +Run `./setup-ocm.sh` to create three Kind clusters: +- `hub`, the cluster on which the OCM hub is running; +- `cluster1`, it joins the hub cluster through the HTTP proxy; +- `cluster2`, it joins the hub cluster through the HTTPS proxy; + +![join through proxy](./join-through-proxy.png) + +## Troubleshooting + +1. Check if the `bootstrap-hub-kubeconfig` and `hub-kubeconfig-secret` contains the proxy settings + +During bootstrapping, the Klusterlet agent uses the `bootstrap-hub-kubeconfig` to connect to the hub cluster. The proxy settings should be included in this kubeconfig. +```bash +kubectl -n open-cluster-management-agent get secret bootstrap-hub-kubeconfig -o jsonpath='{.data.kubeconfig}' | base64 -d | grep proxy-url + proxy-url: http://proxy-control-plane:31280 +``` +Once the cluster registered to the hub cluster, the generated `hub-kubeconfig-secret` should also include the same proxy settings. +```bash +kubectl -n open-cluster-management-agent get secret hub-kubeconfig-secret -o jsonpath='{.data.kubeconfig}' | base64 -d | grep proxy-url + proxy-url: http://proxy-control-plane:31280 +``` +If you enabled any add-on, the generated hub kubeconfig for add-on will have this proxy settings as well. + +2. Check if the proxy CA certificate is included in the `bootstrap-hub-kubeconfig` and `hub-kubeconfig-secret`. + +If HTTPS proxy is used, the CA certificate of the proxy server should be appended to the `certificate-authority-data` of the `bootstrap-hub-kubeconfig` and `hub-kubeconfig-secret` secret. + +```bash +kubectl -n open-cluster-management-agent get secret bootstrap-hub-kubeconfig -o jsonpath='{.data.kubeconfig}' | base64 -d | grep certificate-authority-data | awk '{print $2}' | base64 -d +-----BEGIN CERTIFICATE----- +MIIC/jCCA...O62WrVM= +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIIDDTCCA...INDFwtk= +-----END CERTIFICATE----- +``` diff --git a/solutions/join-through-proxy/config/server-cert.conf b/solutions/join-through-proxy/config/server-cert.conf new file mode 100644 index 000000000..d24a49626 --- /dev/null +++ b/solutions/join-through-proxy/config/server-cert.conf @@ -0,0 +1,12 @@ +[ req_ext ] +subjectAltName = @alt_names + +[ alt_names ] +DNS.1 = + +[ v3_ext ] +authorityKeyIdentifier=keyid,issuer:always +basicConstraints=CA:FALSE +keyUsage=keyEncipherment,dataEncipherment +extendedKeyUsage=serverAuth,clientAuth +subjectAltName=@alt_names \ No newline at end of file diff --git a/solutions/join-through-proxy/config/squid.conf b/solutions/join-through-proxy/config/squid.conf new file mode 100644 index 000000000..ee1ba3b31 --- /dev/null +++ b/solutions/join-through-proxy/config/squid.conf @@ -0,0 +1,40 @@ +acl localnet src 0.0.0.1-0.255.255.255 # RFC 1122 "this" network (LAN) +acl localnet src 10.0.0.0/8 # RFC 1918 local private network (LAN) +acl localnet src 100.64.0.0/10 # RFC 6598 shared address space (CGN) +acl localnet src 169.254.0.0/16 # RFC 3927 link-local (directly plugged) machines +acl localnet src 172.16.0.0/12 # RFC 1918 local private network (LAN) +acl localnet src 192.168.0.0/16 # RFC 1918 local private network (LAN) +acl localnet src fc00::/7 # RFC 4193 local private network range +acl localnet src fe80::/10 # RFC 4291 link-local (directly plugged) machines +acl SSL_ports port 443 +acl SSL_ports port 6443 +acl Safe_ports port 80 # http +acl Safe_ports port 21 # ftp +acl Safe_ports port 443 # https +acl Safe_ports port 70 # gopher +acl Safe_ports port 210 # wais +acl Safe_ports port 1025-65535 # unregistered ports +acl Safe_ports port 280 # http-mgmt +acl Safe_ports port 488 # gss-http +acl Safe_ports port 591 # filemaker +acl Safe_ports port 777 # multiling http +acl CONNECT method CONNECT +http_access deny !Safe_ports +http_access deny CONNECT !SSL_ports +http_access allow localhost manager +http_access deny manager +http_access allow localhost +http_access allow localnet +http_access deny all +http_port 3128 +https_port 3129 tls-cert=/etc/squid/squid-cert-key.pem +coredump_dir /var/spool/squid +refresh_pattern ^ftp: 1440 20% 10080 +refresh_pattern ^gopher: 1440 0% 1440 +refresh_pattern -i (/cgi-bin/|\?) 0 0% 0 +refresh_pattern \/(Packages|Sources)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern \/Release(|\.gpg)$ 0 0% 0 refresh-ims +refresh_pattern \/InRelease$ 0 0% 0 refresh-ims +refresh_pattern \/(Translation-.*)(|\.bz2|\.gz|\.xz)$ 0 0% 0 refresh-ims +refresh_pattern . 0 20% 4320 +logfile_rotate 0 diff --git a/solutions/join-through-proxy/join-through-proxy.png b/solutions/join-through-proxy/join-through-proxy.png new file mode 100644 index 000000000..b894a11b6 Binary files /dev/null and b/solutions/join-through-proxy/join-through-proxy.png differ diff --git a/solutions/join-through-proxy/manifests/squid-deploy.yaml b/solutions/join-through-proxy/manifests/squid-deploy.yaml new file mode 100644 index 000000000..fafbb97dd --- /dev/null +++ b/solutions/join-through-proxy/manifests/squid-deploy.yaml @@ -0,0 +1,50 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: squid + namespace: squid +spec: + replicas: 1 + selector: + matchLabels: + app: squid + template: + metadata: + labels: + app: squid + spec: + containers: + - name: squid + image: ubuntu/squid:5.2-22.04_beta + ports: + - containerPort: 3128 + name: http + protocol: TCP + - containerPort: 3129 + name: https + protocol: TCP + volumeMounts: + - name: squid-config-volume + mountPath: /etc/squid/squid.conf + subPath: squid.conf + - name: squid-tls-volume + mountPath: /etc/squid/squid-cert-key.pem + subPath: squid-cert-key.pem + - name: squid-data-volume + mountPath: /var/spool/squid + volumes: + - name: squid-config-volume + configMap: + name: squid-config + items: + - key: squid + path: squid.conf + - name: squid-tls-volume + secret: + secretName: squid-cert-key + items: + - key: squid-cert-key.pem + path: squid-cert-key.pem + - name: squid-data-volume + emptyDir: + medium: Memory diff --git a/solutions/join-through-proxy/manifests/squid-service.yaml b/solutions/join-through-proxy/manifests/squid-service.yaml new file mode 100644 index 000000000..01da46edf --- /dev/null +++ b/solutions/join-through-proxy/manifests/squid-service.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: Service +metadata: + name: squid + namespace: squid + labels: + app: squid +spec: + type: NodePort + ports: + - port: 3128 + nodePort: 31280 + name: http + - port: 3129 + nodePort: 31290 + name: https + selector: + app: squid diff --git a/solutions/join-through-proxy/setup-ocm.sh b/solutions/join-through-proxy/setup-ocm.sh new file mode 100755 index 000000000..fd8f3f879 --- /dev/null +++ b/solutions/join-through-proxy/setup-ocm.sh @@ -0,0 +1,42 @@ +#!/bin/bash +set -e + +proxy=${HUB_NAME:-proxy} +hub=${HUB_NAME:-hub} +cluster1=${CLUSTER_NAME:-cluster1} +cluster2=${CLUSTER_NAME:-cluster2} + +hubctx="kind-${hub}" +cluster1ctx="kind-${cluster1}" +cluster2ctx="kind-${cluster2}" + +# setup hub +kind create cluster --name "${hub}" +kubectl config use ${hubctx} +echo "Initialize the ocm hub cluster" +joincmd=$(clusteradm init --use-bootstrap-token | grep clusteradm) +kubectl wait --for=condition=HubRegistrationDegraded=false clustermanager cluster-manager --timeout=60s + +# register cluster1 +kind create cluster --name "${cluster1}" +kubectl config use ${cluster1ctx} +echo "Join ${cluster1} to ${hub} through HTTP proxy http://${proxy}-control-plane:31280" +$(echo ${joincmd} --singleton --force-internal-endpoint-lookup --proxy-url=http://${proxy}-control-plane:31280 --wait | sed "s//$cluster1/g") + +kubectl config use ${hubctx} +echo "Accept join of ${cluster1} on ${hub}" +clusteradm accept --clusters ${cluster1} --wait + +kubectl get managedclusters + +# register cluster2 +kind create cluster --name "${cluster2}" +kubectl config use ${cluster2ctx} +echo "Join ${cluster2} to ${hub} through HTTP proxy https://${proxy}-control-plane:31290" +$(echo ${joincmd} --singleton --force-internal-endpoint-lookup --proxy-url=https://${proxy}-control-plane:31290 --proxy-ca-file=tls/root-ca.crt --wait | sed "s//$cluster2/g") + +kubectl config use ${hubctx} +echo "Accept join of ${cluster2} on ${hub}" +clusteradm accept --clusters ${cluster2} --wait + +kubectl get managedclusters \ No newline at end of file diff --git a/solutions/join-through-proxy/setup-proxy.sh b/solutions/join-through-proxy/setup-proxy.sh new file mode 100755 index 000000000..df9d04cc8 --- /dev/null +++ b/solutions/join-through-proxy/setup-proxy.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -e + +proxy=${HUB_NAME:-proxy} +proxyctx="kind-${proxy}" + +# create server certificate for proxy server +rm -rf tls +mkdir tls +openssl genrsa -out tls/root-ca.key 2048 +openssl req -key tls/root-ca.key -subj "/CN=example.com" -new -x509 -days 3650 -out tls/root-ca.crt +openssl genrsa -out tls/squid.key 2048 +openssl req -new -key tls/squid.key -subj "/CN=${proxy}-control-plane" -out tls/squid.csr +cat server-cert.conf | sed "s//$proxy-control-plane/g" > tls/server-cert.conf +openssl x509 -req -in tls/squid.csr -CA tls/root-ca.crt -CAkey tls/root-ca.key -CAcreateserial -out tls/squid.crt -days 3600 -extensions v3_ext -extfile tls/server-cert.conf +cat tls/squid.crt tls/squid.key >> tls/squid-cert-key.pem + +kind create cluster --name "${proxy}" + +kubectl config use ${proxyctx} +echo "setup proxy server - squid" +kubectl create ns squid +kubectl -n squid create secret generic squid-cert-key --from-file=squid-cert-key.pem=tls/squid-cert-key.pem +kubectl -n squid create configmap squid-config --from-file=squid=squid.conf +kubectl apply -f manifests/squid-deploy.yaml +kubectl apply -f manifests/squid-service.yaml + +kubectl -n squid get pods