diff --git a/Deployment/dp-1/README.md b/Deployment/dp-1/README.md index 3a11050..033a695 100644 --- a/Deployment/dp-1/README.md +++ b/Deployment/dp-1/README.md @@ -3,7 +3,7 @@ This is a simple stateless deployment scenario. It can be used to both local and real cloud, such as AWS. ## Prerequisities -- Bootstrap +- Bootstrap AWS: ```console bin/bootstrap-aws.sh @@ -107,4 +107,4 @@ bin/cluster-run status ## Delete Cluster ### ```console ./stop -``` \ No newline at end of file +``` diff --git a/Deployment/dp-1/admin.yaml b/Deployment/dp-1/admin.yaml index 4100977..de3c3f9 100644 --- a/Deployment/dp-1/admin.yaml +++ b/Deployment/dp-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/Deployment/dp-1/esp-e1.yaml b/Deployment/dp-1/esp-e1.yaml index 5001eb1..52d9587 100644 --- a/Deployment/dp-1/esp-e1.yaml +++ b/Deployment/dp-1/esp-e1.yaml @@ -34,11 +34,11 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https containerPort: 18010 - name: wsecl diff --git a/Deployment/ebs/ebs-1/admin.yaml b/Deployment/ebs/ebs-1/admin.yaml index 4100977..de3c3f9 100644 --- a/Deployment/ebs/ebs-1/admin.yaml +++ b/Deployment/ebs/ebs-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/Deployment/ebs/ebs-1/esp-e1.yaml b/Deployment/ebs/ebs-1/esp-e1.yaml index 5001eb1..52d9587 100644 --- a/Deployment/ebs/ebs-1/esp-e1.yaml +++ b/Deployment/ebs/ebs-1/esp-e1.yaml @@ -34,11 +34,11 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https containerPort: 18010 - name: wsecl diff --git a/Deployment/ebs/ebs-1/start b/Deployment/ebs/ebs-1/start old mode 100644 new mode 100755 diff --git a/Deployment/ebs/ebs-1/stop b/Deployment/ebs/ebs-1/stop old mode 100644 new mode 100755 diff --git a/Deployment/efs/efs-1/README.md b/Deployment/efs/efs-1/README.md index 919ed79..fa0ac22 100644 --- a/Deployment/efs/efs-1/README.md +++ b/Deployment/efs/efs-1/README.md @@ -7,7 +7,7 @@ Current deployment has Sasha/DropZone in support Pod. Even EFS performance may not be good as EBS but EFS it is very convenient such as: - Don't need worry about cross AZz - Easy to share and re-use data -- Don't need to worry to delete volume after deleting Pod. +- Don't need to worry to delete volume after deleting Pod. EFS is little expensive than EBS. @@ -170,5 +170,3 @@ bin/cluster-run status ./stop ``` This does not delete volumes. Either use AWS Client or go to EC2 console to delete them. - - diff --git a/Deployment/efs/efs-1/admin.yaml b/Deployment/efs/efs-1/admin.yaml index 0b3f880..adc0999 100644 --- a/Deployment/efs/efs-1/admin.yaml +++ b/Deployment/efs/efs-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/Deployment/efs/efs-1/dali.yaml b/Deployment/efs/efs-1/dali.yaml index dd89624..42288d0 100644 --- a/Deployment/efs/efs-1/dali.yaml +++ b/Deployment/efs/efs-1/dali.yaml @@ -14,4 +14,3 @@ spec: # specification of the pod’s contents - name: dali-data persistentVolumeClaim: claimName: efs - diff --git a/Deployment/efs/efs-1/esp-e1.yaml b/Deployment/efs/efs-1/esp-e1.yaml index b2a33a1..25e0b07 100644 --- a/Deployment/efs/efs-1/esp-e1.yaml +++ b/Deployment/efs/efs-1/esp-e1.yaml @@ -34,20 +34,20 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https - containerPort: 18010 + containerPort: 18010 - name: wsecl - containerPort: 8010 + containerPort: 8010 - name: wsecl-https - containerPort: 18010 + containerPort: 18010 - name: wssql - containerPort: 8510 + containerPort: 8510 - name: wsssql-https - containerPort: 18510 + containerPort: 18510 - name: confmgr - containerPort: 8015 + containerPort: 8015 diff --git a/Deployment/efs/efs-1/roxie-r1.yaml b/Deployment/efs/efs-1/roxie-r1.yaml index 0aabeac..01d2023 100644 --- a/Deployment/efs/efs-1/roxie-r1.yaml +++ b/Deployment/efs/efs-1/roxie-r1.yaml @@ -21,10 +21,9 @@ spec: # specification of the pod’s contents - name: roxie-secure containerPort: 19876 volumeMounts: - - name: roxie1-efs-pvc + - name: roxie1-efs-pvc mountPath: "/var/lib/HPCCSystems/hpcc-data" volumes: - name: roxie1-efs-pvc persistentVolumeClaim: claimName: efs - diff --git a/Deployment/efs/efs-1/roxie-r2.yaml b/Deployment/efs/efs-1/roxie-r2.yaml index 6da3e00..7171b75 100644 --- a/Deployment/efs/efs-1/roxie-r2.yaml +++ b/Deployment/efs/efs-1/roxie-r2.yaml @@ -3,7 +3,7 @@ kind: Deployment metadata: name: roxie-roxie2 spec: # specification of the pod’s contents - replicas: 1 + replicas: 1 selector: matchLabels: app: roxie-roxie2 @@ -21,10 +21,9 @@ spec: # specification of the pod’s contents - name: roxie-secure containerPort: 19876 volumeMounts: - - name: roxie2-efs-pvc + - name: roxie2-efs-pvc mountPath: "/var/lib/HPCCSystems/hpcc-data" volumes: - name: roxie2-efs-pvc persistentVolumeClaim: claimName: efs - diff --git a/Pod/README.md b/Pod/README.md index b9f5d02..375ff95 100644 --- a/Pod/README.md +++ b/Pod/README.md @@ -13,16 +13,16 @@ kubectl create -f hpcc.yaml For single node deployment HPCC is not started. you can start it as: ```sh kubectl exec hpcc -- /etc/init.d/hpcc-init start -Starting mydafilesrv ... [ OK ] -Starting mydali ... [ OK ] -Starting mydfuserver ... [ OK ] -Starting myeclagent ... [ OK ] -Starting myeclccserver ... [ OK ] -Starting myeclscheduler ... [ OK ] -Starting myesp ... [ OK ] -Starting myroxie ... [ OK ] -Starting mysasha ... [ OK ] -Starting mythor ... [ OK ] +Starting mydafilesrv ... [ OK ] +Starting mydali ... [ OK ] +Starting mydfuserver ... [ OK ] +Starting myeclagent ... [ OK ] +Starting myeclccserver ... [ OK ] +Starting myeclscheduler ... [ OK ] +Starting myesp ... [ OK ] +Starting myroxie ... [ OK ] +Starting mysasha ... [ OK ] +Starting mythor ... [ OK ] ``` You also can access the contain to run commands: ```sh diff --git a/Pod/hpcc.yaml b/Pod/hpcc.yaml index ad5f9d9..2c78262 100644 --- a/Pod/hpcc.yaml +++ b/Pod/hpcc.yaml @@ -10,8 +10,8 @@ spec: # specification of the pod’s contents ports: - name: eclwatch containerPort: 8010 - - name: eclwatch-https - containerPort: 18010 + - name: eclwatch-https + containerPort: 18010 - name: wsecl containerPort: 8002 - name: wsecl-https diff --git a/README.md b/README.md index a09c5ed..7e0119c 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ We currently tested two cloud environments: [AWS/EKS](./aws/EKS/README.md) and [ ## Bootstrap Bootstrap will grant access permission for Kubernetes APIs as well create configmap for environmet.xml configuration Depands on the Kubernetes environment configmap files may be different. Currently there is aws/configmap/hpcc for AWS environment and the other one is local/configmap/hpcc for local deployment. -In bin directory +In bin directory ```sh # AWS bin/bootstrap-aws.sh @@ -23,22 +23,22 @@ bin/bootstrap-local.sh or on Windows bootstrap.bat ``` -User can modify security/cluster_role.yaml and files under configmap/hpcc +User can modify security/cluster_role.yaml and files under configmap/hpcc ## Pod Deploy HPCC Systems Platform on single node Reference [README.md](Pod/README.md) ## Deployment -Deploy HPCC Systems cluster with Deployment Pod definition. +Deploy HPCC Systems cluster with Deployment Pod definition. Reference [README.md](Deployment/dp-1/README.md) ## RelicationController -Deploy HPCC Systems cluster with RelicationController Pod definition. +Deploy HPCC Systems cluster with RelicationController Pod definition. It is recommended to use "Deployment" instead ## [StatefulSet](StatefulSet/README.md) -Deploy HPCC Systems cluster with StatefulSet Pod definition. +Deploy HPCC Systems cluster with StatefulSet Pod definition. It includs ebs and nfs examples Reference . [EBS README.md](StatefulSet/ebs/ebs-1/README.md) @@ -64,9 +64,8 @@ RBAC settings for Kubernetes environment AWS related settings ## elastic -Filebeat, Metricbeat, etc example on local Kubernetes environment. +Filebeat, Metricbeat, etc example on local Kubernetes environment. Still in progress ... ## performance To do -. diff --git a/ReplicationController/rc-1/README.md b/ReplicationController/rc-1/README.md index 0d7c03a..e0a9918 100644 --- a/ReplicationController/rc-1/README.md +++ b/ReplicationController/rc-1/README.md @@ -1,12 +1,12 @@ -# Deploy Cluster with ReplicationController -"Deployment" is prefered - -## Start a Cluster -```console -./start -``` - -## Stop a Cluster -```console -./stop -``` \ No newline at end of file +# Deploy Cluster with ReplicationController +"Deployment" is prefered + +## Start a Cluster +```console +./start +``` + +## Stop a Cluster +```console +./stop +``` diff --git a/ReplicationController/rc-1/admin.yaml b/ReplicationController/rc-1/admin.yaml index 4100977..de3c3f9 100644 --- a/ReplicationController/rc-1/admin.yaml +++ b/ReplicationController/rc-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/ReplicationController/rc-1/esp-e1.yaml b/ReplicationController/rc-1/esp-e1.yaml index 2af6189..604d1ed 100644 --- a/ReplicationController/rc-1/esp-e1.yaml +++ b/ReplicationController/rc-1/esp-e1.yaml @@ -31,11 +31,11 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https containerPort: 18010 - name: wsecl diff --git a/StatefulSet/README.md b/StatefulSet/README.md index 0c38726..f6213fc 100644 --- a/StatefulSet/README.md +++ b/StatefulSet/README.md @@ -1,8 +1,8 @@ -# Deploy HPCC Systems cluster with StatefulSet - -## Deployment Types - - - [EBS](ebs/ebs-1/README.md) - - [EFS](efs/efs-1/README.md) - -## Performance Comparison \ No newline at end of file +# Deploy HPCC Systems cluster with StatefulSet + +## Deployment Types + + - [EBS](ebs/ebs-1/README.md) + - [EFS](efs/efs-1/README.md) + +## Performance Comparison diff --git a/StatefulSet/ebs/ebs-1/README.md b/StatefulSet/ebs/ebs-1/README.md index 86f842c..0347c38 100644 --- a/StatefulSet/ebs/ebs-1/README.md +++ b/StatefulSet/ebs/ebs-1/README.md @@ -145,5 +145,3 @@ bin/cluster-run status ./stop ``` This does not delete volumes. Either use AWS Client or go to EC2 console to delete them. - - diff --git a/StatefulSet/ebs/ebs-1/admin.yaml b/StatefulSet/ebs/ebs-1/admin.yaml index 0b3f880..adc0999 100644 --- a/StatefulSet/ebs/ebs-1/admin.yaml +++ b/StatefulSet/ebs/ebs-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/StatefulSet/ebs/ebs-1/esp-e1.yaml b/StatefulSet/ebs/ebs-1/esp-e1.yaml index a0efd47..14502f6 100644 --- a/StatefulSet/ebs/ebs-1/esp-e1.yaml +++ b/StatefulSet/ebs/ebs-1/esp-e1.yaml @@ -34,18 +34,18 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https - containerPort: 18010 + containerPort: 18010 - name: wsecl - containerPort: 8002 + containerPort: 8002 - name: wsecl-https - containerPort: 18002 + containerPort: 18002 - name: wssql - containerPort: 8510 + containerPort: 8510 - name: wssql-https - containerPort: 18510 + containerPort: 18510 diff --git a/StatefulSet/ebs/ebs-1/roxie-r1.yaml b/StatefulSet/ebs/ebs-1/roxie-r1.yaml index 5db64e0..4a84205 100644 --- a/StatefulSet/ebs/ebs-1/roxie-r1.yaml +++ b/StatefulSet/ebs/ebs-1/roxie-r1.yaml @@ -4,7 +4,7 @@ metadata: name: svc-roxie-roxie1 # namespace: default labels: - app: roxie-roxie1 + app: roxie-roxie1 spec: selector: app: svc-roxie-roxie1 diff --git a/StatefulSet/ebs/ebs-1/roxie-r2.yaml b/StatefulSet/ebs/ebs-1/roxie-r2.yaml index 0033487..52f32e8 100644 --- a/StatefulSet/ebs/ebs-1/roxie-r2.yaml +++ b/StatefulSet/ebs/ebs-1/roxie-r2.yaml @@ -30,7 +30,7 @@ spec: # specification of the pod’s contents image: "hpccsystems/platform" volumeMounts: - mountPath: "/var/lib/HPCCSystems" - name: ebs + name: ebs volumeClaimTemplates: - metadata: name: ebs diff --git a/StatefulSet/efs/efs-1/.support.yaml.swp b/StatefulSet/efs/efs-1/.support.yaml.swp deleted file mode 100644 index 315a670..0000000 Binary files a/StatefulSet/efs/efs-1/.support.yaml.swp and /dev/null differ diff --git a/StatefulSet/efs/efs-1/README.md b/StatefulSet/efs/efs-1/README.md index ae3139f..8ab4976 100644 --- a/StatefulSet/efs/efs-1/README.md +++ b/StatefulSet/efs/efs-1/README.md @@ -1,6 +1,6 @@ # Deploy Dali/Sasha/DropZone/Roxie/Thor Pods as StatefulSet/EFS -Current deployment has Sasha/DropZone in support Pod. +Current deployment has Sasha/DropZone in support Pod. Attach ReadWriteMany EFS to Pods doesn't need StatefulSet. See Deployment/efs/efs-1/README.md. But for ReadWriteOnce EFS StatefulSet is required which is this setup about. @@ -69,7 +69,7 @@ To check clustr status: ```console bin/cluster_run.sh status ``` -## Access EclWatch +## Access EclWatch Get esp public ip: ```console kubectl get services diff --git a/StatefulSet/efs/efs-1/admin.yaml b/StatefulSet/efs/efs-1/admin.yaml index 0b3f880..adc0999 100644 --- a/StatefulSet/efs/efs-1/admin.yaml +++ b/StatefulSet/efs/efs-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/StatefulSet/efs/efs-1/dali.yaml b/StatefulSet/efs/efs-1/dali.yaml index dd89624..42288d0 100644 --- a/StatefulSet/efs/efs-1/dali.yaml +++ b/StatefulSet/efs/efs-1/dali.yaml @@ -14,4 +14,3 @@ spec: # specification of the pod’s contents - name: dali-data persistentVolumeClaim: claimName: efs - diff --git a/StatefulSet/efs/efs-1/esp-e1.yaml b/StatefulSet/efs/efs-1/esp-e1.yaml index b2a33a1..25e0b07 100644 --- a/StatefulSet/efs/efs-1/esp-e1.yaml +++ b/StatefulSet/efs/efs-1/esp-e1.yaml @@ -34,20 +34,20 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https - containerPort: 18010 + containerPort: 18010 - name: wsecl - containerPort: 8010 + containerPort: 8010 - name: wsecl-https - containerPort: 18010 + containerPort: 18010 - name: wssql - containerPort: 8510 + containerPort: 8510 - name: wsssql-https - containerPort: 18510 + containerPort: 18510 - name: confmgr - containerPort: 8015 + containerPort: 8015 diff --git a/StatefulSet/efs/efs-1/roxie-r1.yaml b/StatefulSet/efs/efs-1/roxie-r1.yaml index 5ce6f01..9a567cd 100644 --- a/StatefulSet/efs/efs-1/roxie-r1.yaml +++ b/StatefulSet/efs/efs-1/roxie-r1.yaml @@ -4,7 +4,7 @@ metadata: name: svc-roxie-roxie1 # namespace: default labels: - app: roxie-roxie1 + app: roxie-roxie1 spec: selector: app: svc-roxie-roxie1 @@ -34,7 +34,7 @@ spec: # specification of the pod’s contents - name: roxie-secure containerPort: 19876 volumeMounts: - - name: efs + - name: efs mountPath: "/var/lib/HPCCSystems/hpcc-data" volumeClaimTemplates: - metadata: diff --git a/StatefulSet/efs/efs-1/roxie-r2.yaml b/StatefulSet/efs/efs-1/roxie-r2.yaml index d99f39d..8af5e8a 100644 --- a/StatefulSet/efs/efs-1/roxie-r2.yaml +++ b/StatefulSet/efs/efs-1/roxie-r2.yaml @@ -4,7 +4,7 @@ metadata: name: svc-roxie-roxie2 # namespace: default labels: - app: roxie-roxie2 + app: roxie-roxie2 spec: selector: app: svc-roxie-roxie2 @@ -34,7 +34,7 @@ spec: # specification of the pod’s contents - name: roxie-secure containerPort: 19876 volumeMounts: - - name: efs + - name: efs mountPath: "/var/lib/HPCCSystems/hpcc-data" volumeClaimTemplates: - metadata: diff --git a/StatefulSet/efs/efs-1/simple/apply.sh b/StatefulSet/efs/efs-1/simple/apply.sh deleted file mode 100644 index 5200626..0000000 --- a/StatefulSet/efs/efs-1/simple/apply.sh +++ /dev/null @@ -1,3 +0,0 @@ - -kubectl apply -f rbac.yaml -kubectl apply -f efs-provisioner.yaml diff --git a/StatefulSet/efs/efs-1/simple/delete.sh b/StatefulSet/efs/efs-1/simple/delete.sh deleted file mode 100644 index 3c0aaaf..0000000 --- a/StatefulSet/efs/efs-1/simple/delete.sh +++ /dev/null @@ -1,3 +0,0 @@ - -kubectl delete -f rbac.yaml -kubectl delete -f efs-provisioner.yaml diff --git a/StatefulSet/efs/efs-1/simple/efs.log b/StatefulSet/efs/efs-1/simple/efs.log deleted file mode 100644 index 0184f03..0000000 --- a/StatefulSet/efs/efs-1/simple/efs.log +++ /dev/null @@ -1,54 +0,0 @@ -Name: efs-provisioner-64d55b8f76-hhknq -Namespace: default -Priority: 0 -Node: ip-172-31-21-220.ec2.internal/172.31.21.220 -Start Time: Sat, 14 Sep 2019 17:22:48 -0400 -Labels: app=efs-provisioner - pod-template-hash=64d55b8f76 -Annotations: kubernetes.io/psp: eks.privileged -Status: Pending -IP: -Controlled By: ReplicaSet/efs-provisioner-64d55b8f76 -Containers: - efs-provisioner: - Container ID: - Image: quay.io/external_storage/efs-provisioner:v0.1.0 - Image ID: - Port: - Host Port: - State: Waiting - Reason: ContainerCreating - Ready: False - Restart Count: 0 - Environment: - FILE_SYSTEM_ID: fs-9ba53979 - AWS_REGION: us-east-1 - PROVISIONER_NAME: eks-course/aws-efs - Mounts: - /persistentvolumes from pv-volume (rw) - /var/run/secrets/kubernetes.io/serviceaccount from default-token-bbjgp (ro) -Conditions: - Type Status - Initialized True - Ready False - ContainersReady False - PodScheduled True -Volumes: - pv-volume: - Type: NFS (an NFS mount that lasts the lifetime of a pod) - Server: fs-9ba53979.efs.us-east-1.amazonaws.com - Path: / - ReadOnly: false - default-token-bbjgp: - Type: Secret (a volume populated by a Secret) - SecretName: default-token-bbjgp - Optional: false -QoS Class: BestEffort -Node-Selectors: -Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s - node.kubernetes.io/unreachable:NoExecute for 300s -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Scheduled 3m20s default-scheduler Successfully assigned default/efs-provisioner-64d55b8f76-hhknq to ip-172-31-21-220.ec2.internal - Warning FailedMount 77s kubelet, ip-172-31-21-220.ec2.internal Unable to mount volumes for pod "efs-provisioner-64d55b8f76-hhknq_default(cd50b06c-d735-11e9-9e51-0a9464b954ac)": timeout expired waiting for volumes to attach or mount for pod "default"/"efs-provisioner-64d55b8f76-hhknq". list of unmounted volumes=[pv-volume]. list of unattached volumes=[pv-volume default-token-bbjgp] diff --git a/aws/EKS/README.md b/aws/EKS/README.md index f6aabb5..657ede8 100644 --- a/aws/EKS/README.md +++ b/aws/EKS/README.md @@ -20,12 +20,12 @@ https://docs.aws.amazon.com/eks/latest/userguide/getting-started-eksctl.html ## Run EKS Cluster ### Create EKS cluster ### EKS cluster requires at two Available Zones (AZs). -Two samples are provided: -. eks_1.sh +Two samples are provided: +. eks_1.sh . eks_vpc.sh: this try to use existing VPC/Subnets but doesn't work. We are working with AWS support on it now ### Query EKS ### -Query EKS cluster +Query EKS cluster ```sh eksctl get cluster ``` @@ -40,5 +40,3 @@ eksctl utils write-kubeconfig --name ./stop ``` The default is "hpcc1" which is defined in start script. - - diff --git a/aws/EKS/eks_1.sh b/aws/EKS/eks_1.sh index 7d45c2a..c355e3c 100644 --- a/aws/EKS/eks_1.sh +++ b/aws/EKS/eks_1.sh @@ -18,4 +18,3 @@ eksctl create cluster \ --tags "owner_email=xiaoming.wang@lexisnexis.com" \ --tags "support_email=xiaoming.wang@lexisnexis.com" \ --tags "product=hpccsystems,project=hpcc builds,service=ecs" - diff --git a/aws/EKS/eks_vpc.sh b/aws/EKS/eks_vpc.sh index 5e09d31..7cfb8bf 100644 --- a/aws/EKS/eks_vpc.sh +++ b/aws/EKS/eks_vpc.sh @@ -20,6 +20,5 @@ eksctl create cluster \ --tags "owner_email=xiaoming.wang@lexisnexis.com" \ --tags "support_email=xiaoming.wang@lexisnexis.com" \ --tags "product=hpccsystems,project=hpcc builds,service=ecs" - - # --region us-east-1 \ +# --region us-east-1 \ # vpc setting can co-existi with zones settings diff --git a/aws/configmap/README.md b/aws/configmap/README.md index cf542bc..ab35ff7 100644 --- a/aws/configmap/README.md +++ b/aws/configmap/README.md @@ -6,7 +6,7 @@ ConfigMap is created with bin/bootstrap-xxx.sh or bin/bootstrap.bat The files are available on hpcc-admin container under /etc/hpcc-config-map/ Following configuration are availble now -Under hpcc directory: +Under hpcc directory: ### topology.properties This file defines how topology in environment.xml can be constructed @@ -15,7 +15,7 @@ This file defines how topology in environment.xml can be constructed Add/modify Software/Category directories as "-o" option in envgen/envgen2 ### override.envgen2 -Override all component properties with the give xpath for the given buildset with the value provided. This is "-override" option in envgen/envgen2 +Override all component properties with the give xpath for the given buildset with the value provided. This is "-override" option in envgen/envgen2 ### xpath_attributes.envgen2 @@ -28,9 +28,3 @@ Add/modify component settings with envgen2 syntax ### contents/*.xml These xml files will be inserted to environment.xml to xpath provided in in the file These files will be sorted first so users can use file names to control the execution order. - - - - - - diff --git a/aws/configmap/hpcc/override.envgen2 b/aws/configmap/hpcc/override.envgen2 index 193ab66..bca1acb 100644 --- a/aws/configmap/hpcc/override.envgen2 +++ b/aws/configmap/hpcc/override.envgen2 @@ -7,4 +7,3 @@ roxie,@copyResources,true roxie,@roxieMulticastEnabled,false thor,@replicateOutputs,true esp,@method,htpasswd - diff --git a/aws/configmap/hpcc/xpath_attributes.envgen2 b/aws/configmap/hpcc/xpath_attributes.envgen2 index 53c594c..5bbbf6f 100644 --- a/aws/configmap/hpcc/xpath_attributes.envgen2 +++ b/aws/configmap/hpcc/xpath_attributes.envgen2 @@ -3,5 +3,3 @@ #Software/Topology/Cluster[@name=\\\"thor\\\"]/ThorCluster,@process,thor123 Software/RoxieCluster/",roxieMulticastEnabled,false - - diff --git a/aws/nfs/README.md b/aws/nfs/README.md index 89c6a64..a4a40b5 100644 --- a/aws/nfs/README.md +++ b/aws/nfs/README.md @@ -54,7 +54,7 @@ spec: volumeID: vol-4aa30797 fsType: ext4 ``` -Deploy nfs server: +Deploy nfs server: ```sh kubectl create -f nfs-server-rc.yaml ``` @@ -179,8 +179,3 @@ To access to check the volumes: kubectl exec -i -t -- bash -il dk -h ``` - - - - - diff --git a/aws/nfs/create-ebs.sh b/aws/nfs/create-ebs.sh index 511bd3c..de604b1 100755 --- a/aws/nfs/create-ebs.sh +++ b/aws/nfs/create-ebs.sh @@ -34,13 +34,13 @@ done shift $((OPTIND-1)) if [ -z "$availability_zone" ]; then - echo "Missing availability zone" + echo "Missing availability zone" usage exit 1 fi if [ -z "$volume_size" ]; then - echo "Missing volume size" + echo "Missing volume size" usage exit 1 fi @@ -48,5 +48,5 @@ fi aws ec2 create-volume --availability-zone $availability_zone \ --size $volume_size --volume-type $volume_type | grep "VolumeId" | \ cut -d':' -f2 | sed 's/[[:space:]][[:space:]]*\"\(.*\)\",/\1/' - + # "VolumeId": "vol-233c99fe", diff --git a/aws/nfs/roxie-controller.yaml b/aws/nfs/roxie-controller.yaml index 28c4afd..dd6f09b 100644 --- a/aws/nfs/roxie-controller.yaml +++ b/aws/nfs/roxie-controller.yaml @@ -20,10 +20,10 @@ spec: - mountPath: "/disk1" name: disk1 volumes: - - name: nfs + - name: nfs persistentVolumeClaim: claimName: nfs - - name: disk1 + - name: disk1 awsElasticBlockStore: volumeID: vol-233c99fe fsType: ext4 diff --git a/aws/volumes/gp2.yaml b/aws/volumes/gp2.yaml index d25aab5..637616d 100644 --- a/aws/volumes/gp2.yaml +++ b/aws/volumes/gp2.yaml @@ -4,8 +4,8 @@ kind: StorageClass metadata: name: gp2 provisioner: kubernetes.io/aws-ebs - annotation: - storageclass.kubernetes.io/is-default-class: true + annotation: + storageclass.kubernetes.io/is-default-class: true parameters: type: gp2 reclaimPolicy: Retain diff --git a/bin/bootstrap-aws.sh b/bin/bootstrap-aws.sh index f5ec07f..c8979e5 100755 --- a/bin/bootstrap-aws.sh +++ b/bin/bootstrap-aws.sh @@ -3,4 +3,4 @@ WORK_DIR=$(dirname $0) . ${WORK_DIR}/common ${KUBECTL} create configmap hpcc-config --from-file=${WORK_DIR}/../aws/configmap/hpcc/ -${KUBECTL} apply -f ${WORK_DIR}/../security/cluster_role.yaml +${KUBECTL} apply -f ${WORK_DIR}/../security/cluster_role.yaml diff --git a/bin/bootstrap-local.sh b/bin/bootstrap-local.sh index a16bdf9..d712626 100755 --- a/bin/bootstrap-local.sh +++ b/bin/bootstrap-local.sh @@ -3,4 +3,4 @@ WORK_DIR=$(dirname $0) . ${WORK_DIR}/common ${KUBECTL} create configmap hpcc-config --from-file=${WORK_DIR}/../local/configmap/hpcc/ --from-file=${WORK_DIR}/../local/configmap/hpcc/contents/ -${KUBECTL} apply -f ${WORK_DIR}/../security/cluster_role.yaml +${KUBECTL} apply -f ${WORK_DIR}/../security/cluster_role.yaml diff --git a/bin/cluster_config.sh b/bin/cluster_config.sh index f0df8fb..8225d3a 100755 --- a/bin/cluster_config.sh +++ b/bin/cluster_config.sh @@ -68,4 +68,3 @@ else fi echo "$cmd" eval $cmd - diff --git a/bin/cluster_query.ps1 b/bin/cluster_query.ps1 index f28174a..9c15d1f 100644 --- a/bin/cluster_query.ps1 +++ b/bin/cluster_query.ps1 @@ -3,10 +3,10 @@ Query HPCC Systems Cluster Pods .DESCRIPTION -This is top Poswershell script to query HPCC Systems Cluster Pods +This is top Poswershell script to query HPCC Systems Cluster Pods + + Usage: cluster_query.ps1 -namespace -pattern - Usage: cluster_query.ps1 -namespace -pattern - .Example ./cluster_query.ps1 @@ -32,7 +32,7 @@ if ( $pod ) Function get_pod_names { - + #(kubectl.exe get pods -n ${namespace} | select-string -pattern "${pod_pattern}" | out-string).trim().split(' ')[0] foreach ( $line in (kubectl.exe get pods -n ${namespace} | select-string -pattern "${pod_pattern}" )) { @@ -42,4 +42,4 @@ Function get_pod_names } -get_pod_names \ No newline at end of file +get_pod_names diff --git a/bin/cluster_query.sh b/bin/cluster_query.sh index 7700021..d9d2500 100755 --- a/bin/cluster_query.sh +++ b/bin/cluster_query.sh @@ -78,4 +78,3 @@ then fi exit - diff --git a/bin/cluster_run.ps1 b/bin/cluster_run.ps1 index a27495e..2fc4f01 100644 --- a/bin/cluster_run.ps1 +++ b/bin/cluster_run.ps1 @@ -6,12 +6,12 @@ HPCC Systems Cluster Operations Get status/start/stop HPCC Systems Cluster Usage: cluster_run.ps1 -action -namespace - -component -pod-name - + -component -pod-name + .Example ./cluster_run.ps1 -action status -. +. .LINK https://github.com/xwang2713/HPCC-Kubernetes @@ -35,12 +35,12 @@ function get_cluster_status() { kubectl.exe exec $pod /etc/init.d/hpcc-init status "" - } + } } function runHPCC ($a) { - if ( "$pod_name" -eq "" ) + if ( "$pod_name" -eq "" ) { return 1 } @@ -50,7 +50,7 @@ function runHPCC ($a) $cmd = "$cmd -c $comp_name" } $cmd = "$cmd $a" - "$cmd" + "$cmd" iex "$cmd" } @@ -86,29 +86,29 @@ switch ( $action ) { get_cluster_status } - else + else { runHPCC $action } - + } - "start" - { + "start" + { if ( $component -ieq "configmgr") { kubectl.exe exec -it $admin_pod /opt/HPCCSystems/sbin/configmgr } - else + else { stxxHPCC "start" } } - "stop" + "stop" { stxxHPCC "stop" } "restart" - { + { stxxHPCC "stop" stxxHPCC "start" } diff --git a/bin/common b/bin/common index e63b4a0..b8b930d 100644 --- a/bin/common +++ b/bin/common @@ -9,5 +9,3 @@ then KUBECTL=kubectl fi fi - - diff --git a/charts/hpcc-dp/Chart.yaml b/charts/hpcc-dp/Chart.yaml index fb679af..f59bd01 100644 --- a/charts/hpcc-dp/Chart.yaml +++ b/charts/hpcc-dp/Chart.yaml @@ -1,12 +1,12 @@ -apiVersion: v1 -name: hpccsystems -version: 1.0.0 -appVersion: 7.4.14 -description: End-to-end big data in a massively scalable supercomputing platform. -home: http://hpccsystems.com -icon: https://hpccsystems.com/sites/default/files/header-logo_0.png -sources: - - https://github.com/hpcc-systems/HPCC-Platform -maintainers: - - name: Wang,Xiaoming - email: xiaoming.wang@lexisnexis.com \ No newline at end of file +apiVersion: v1 +name: hpccsystems +version: 1.0.0 +appVersion: 7.4.14 +description: End-to-end big data in a massively scalable supercomputing platform. +home: http://hpccsystems.com +icon: https://hpccsystems.com/sites/default/files/header-logo_0.png +sources: + - https://github.com/hpcc-systems/HPCC-Platform +maintainers: + - name: Wang,Xiaoming + email: xiaoming.wang@lexisnexis.com diff --git a/charts/hpcc-dp/templates/admin.yaml b/charts/hpcc-dp/templates/admin.yaml index 4100977..de3c3f9 100644 --- a/charts/hpcc-dp/templates/admin.yaml +++ b/charts/hpcc-dp/templates/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/charts/hpcc-dp/templates/configmap.yaml b/charts/hpcc-dp/templates/configmap.yaml index 1c5b78c..91105d4 100644 --- a/charts/hpcc-dp/templates/configmap.yaml +++ b/charts/hpcc-dp/templates/configmap.yaml @@ -22,4 +22,3 @@ data: roxie.properties: | default.roxieMulticastEnabled=false {{- end }} - diff --git a/charts/hpcc-dp/templates/esp-e1.yaml b/charts/hpcc-dp/templates/esp-e1.yaml index 5001eb1..52d9587 100644 --- a/charts/hpcc-dp/templates/esp-e1.yaml +++ b/charts/hpcc-dp/templates/esp-e1.yaml @@ -34,11 +34,11 @@ spec: # specification of the pod’s contents app: esp-esp1 spec: containers: - - name: esp-esp1 + - name: esp-esp1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: eclwatch-https containerPort: 18010 - name: wsecl diff --git a/efs/apply.sh b/efs/apply.sh index 1ea3281..9603c7e 100644 --- a/efs/apply.sh +++ b/efs/apply.sh @@ -1,3 +1,2 @@ - kubectl apply -f rbac.yaml kubectl apply -f manifest.yaml diff --git a/efs/delete.sh b/efs/delete.sh index e416502..5ff3fbb 100644 --- a/efs/delete.sh +++ b/efs/delete.sh @@ -1,3 +1,2 @@ - kubectl delete -f rbac.yaml kubectl delete -f maniftest.yaml diff --git a/efs/simple/describe-pod-efs b/efs/simple/describe-pod-efs deleted file mode 100644 index b97483c..0000000 --- a/efs/simple/describe-pod-efs +++ /dev/null @@ -1,101 +0,0 @@ -Name: efs-provisioner-64d55b8f76-scjw6 -Namespace: default -Priority: 0 -Node: ip-192-168-28-184.ec2.internal/192.168.28.184 -Start Time: Thu, 05 Sep 2019 10:38:59 -0400 -Labels: app=efs-provisioner - pod-template-hash=64d55b8f76 -Annotations: kubernetes.io/psp: eks.privileged -Status: Pending -IP: -Controlled By: ReplicaSet/efs-provisioner-64d55b8f76 -Containers: - efs-provisioner: - Container ID: - Image: quay.io/external_storage/efs-provisioner:v0.1.0 - Image ID: - Port: - Host Port: - State: Waiting - Reason: ContainerCreating - Ready: False - Restart Count: 0 - Environment: - FILE_SYSTEM_ID: fs-9ba53979 - AWS_REGION: us-east-1 - PROVISIONER_NAME: eks-course/aws-efs - Mounts: - /persistentvolumes from pv-volume (rw) - /var/run/secrets/kubernetes.io/serviceaccount from default-token-skj2v (ro) -Conditions: - Type Status - Initialized True - Ready False - ContainersReady False - PodScheduled True -Volumes: - pv-volume: - Type: NFS (an NFS mount that lasts the lifetime of a pod) - Server: fs-9ba53979.efs.us-east-1.amazonaws.com - Path: / - ReadOnly: false - default-token-skj2v: - Type: Secret (a volume populated by a Secret) - SecretName: default-token-skj2v - Optional: false -QoS Class: BestEffort -Node-Selectors: -Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s - node.kubernetes.io/unreachable:NoExecute for 300s -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Scheduled 67s default-scheduler Successfully assigned default/efs-provisioner-64d55b8f76-scjw6 to ip-192-168-28-184.ec2.internal - Warning FailedMount 67s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23114.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 66s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23166.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 65s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23198.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 63s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23229.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 59s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23265.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 51s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23297.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 35s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23349.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress - Warning FailedMount 3s kubelet, ip-192-168-28-184.ec2.internal MountVolume.SetUp failed for volume "pv-volume" : mount failed: exit status 32 -Mounting command: systemd-run -Mounting arguments: --description=Kubernetes transient mount for /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume --scope -- mount -t nfs fs-9ba53979.efs.us-east-1.amazonaws.com:/ /var/lib/kubelet/pods/e5978615-cfea-11e9-9ea1-0ea6012a2afe/volumes/kubernetes.io~nfs/pv-volume -Output: Running scope as unit run-23424.scope. -mount.nfs: Failed to resolve server fs-9ba53979.efs.us-east-1.amazonaws.com: Name or service not known -mount.nfs: Operation already in progress diff --git a/efs/simple/efs.log b/efs/simple/efs.log deleted file mode 100644 index 0184f03..0000000 --- a/efs/simple/efs.log +++ /dev/null @@ -1,54 +0,0 @@ -Name: efs-provisioner-64d55b8f76-hhknq -Namespace: default -Priority: 0 -Node: ip-172-31-21-220.ec2.internal/172.31.21.220 -Start Time: Sat, 14 Sep 2019 17:22:48 -0400 -Labels: app=efs-provisioner - pod-template-hash=64d55b8f76 -Annotations: kubernetes.io/psp: eks.privileged -Status: Pending -IP: -Controlled By: ReplicaSet/efs-provisioner-64d55b8f76 -Containers: - efs-provisioner: - Container ID: - Image: quay.io/external_storage/efs-provisioner:v0.1.0 - Image ID: - Port: - Host Port: - State: Waiting - Reason: ContainerCreating - Ready: False - Restart Count: 0 - Environment: - FILE_SYSTEM_ID: fs-9ba53979 - AWS_REGION: us-east-1 - PROVISIONER_NAME: eks-course/aws-efs - Mounts: - /persistentvolumes from pv-volume (rw) - /var/run/secrets/kubernetes.io/serviceaccount from default-token-bbjgp (ro) -Conditions: - Type Status - Initialized True - Ready False - ContainersReady False - PodScheduled True -Volumes: - pv-volume: - Type: NFS (an NFS mount that lasts the lifetime of a pod) - Server: fs-9ba53979.efs.us-east-1.amazonaws.com - Path: / - ReadOnly: false - default-token-bbjgp: - Type: Secret (a volume populated by a Secret) - SecretName: default-token-bbjgp - Optional: false -QoS Class: BestEffort -Node-Selectors: -Tolerations: node.kubernetes.io/not-ready:NoExecute for 300s - node.kubernetes.io/unreachable:NoExecute for 300s -Events: - Type Reason Age From Message - ---- ------ ---- ---- ------- - Normal Scheduled 3m20s default-scheduler Successfully assigned default/efs-provisioner-64d55b8f76-hhknq to ip-172-31-21-220.ec2.internal - Warning FailedMount 77s kubelet, ip-172-31-21-220.ec2.internal Unable to mount volumes for pod "efs-provisioner-64d55b8f76-hhknq_default(cd50b06c-d735-11e9-9e51-0a9464b954ac)": timeout expired waiting for volumes to attach or mount for pod "default"/"efs-provisioner-64d55b8f76-hhknq". list of unmounted volumes=[pv-volume]. list of unattached volumes=[pv-volume default-token-bbjgp] diff --git a/elastic/elastic-1/admin.yaml b/elastic/elastic-1/admin.yaml index 4100977..de3c3f9 100644 --- a/elastic/elastic-1/admin.yaml +++ b/elastic/elastic-1/admin.yaml @@ -12,7 +12,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/elastic/elastic-1/esp-e1.yaml b/elastic/elastic-1/esp-e1.yaml index b6241b2..6352111 100644 --- a/elastic/elastic-1/esp-e1.yaml +++ b/elastic/elastic-1/esp-e1.yaml @@ -34,12 +34,12 @@ spec: # specification of the pod’s contents app: esp-e1 spec: containers: - - name: esp-e1 + - name: esp-e1 image: "hpccsystems/hpcc-elastic" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: wsecl - containerPort: 8010 + containerPort: 8010 - name: roxie - containerPort: 9876 + containerPort: 9876 diff --git a/istio/demo/README.md b/istio/demo/README.md index 2605a91..df5a053 100644 --- a/istio/demo/README.md +++ b/istio/demo/README.md @@ -3,8 +3,8 @@ ### Prepqre ISTIO ### Follow https://istio.io/docs/setup/kubernetes/install/kubernetes/ to install Download ISTIO Unpack downloaded file -export ISTIO_HOME= -Add ${ISTIO_HOME}/bin to PATH +export ISTIO_HOME= +Add ${ISTIO_HOME}/bin to PATH start kubernetes and run bin/bootstrap-local.sh @@ -44,12 +44,12 @@ kubectl.sh get service -n istio-system istio-ingressgateway ``` It should be 31381 if istio-ingressgateway not changed -Try http://:31387 +Try http://:31387 It should not work ### Allow ECLWatch access ### -In network/ +In network/ ```console kubectl.sh apply -f eclwatch-gateway.yaml ``` @@ -60,6 +60,3 @@ In secuity/rbac ./apply-eclwatch.sh ``` This will allow accessing eclwath. Run playground sample to verify it works. - - - diff --git a/istio/demo/admin.yaml b/istio/demo/admin.yaml index 5dc7996..ec9b325 100644 --- a/istio/demo/admin.yaml +++ b/istio/demo/admin.yaml @@ -24,7 +24,7 @@ spec: # specification of the pod’s contents volumeMounts: - mountPath: /etc/hpcc_config_map name: env-config - command: ["/opt/hpcc-tools/run"] + command: ["/opt/hpcc-tools/run"] args: ["-e", "kube"] #args: ["-D"] volumes: diff --git a/istio/demo/esp-e1.yaml b/istio/demo/esp-e1.yaml index 2384969..e13db63 100644 --- a/istio/demo/esp-e1.yaml +++ b/istio/demo/esp-e1.yaml @@ -37,12 +37,12 @@ spec: # specification of the pod’s contents app: esp-e1 spec: containers: - - name: esp-e1 + - name: esp-e1 image: "hpccsystems/platform" ports: - name: eclwatch - containerPort: 8010 + containerPort: 8010 - name: wsecl - containerPort: 8010 + containerPort: 8010 - name: roxie - containerPort: 9876 + containerPort: 9876 diff --git a/istio/demo/install.sh b/istio/demo/install.sh index 5235cea..763f0c2 100755 --- a/istio/demo/install.sh +++ b/istio/demo/install.sh @@ -13,7 +13,7 @@ cd ${ISTIO_HOME} echo "Install all the Istio Custom Resource Definitions (CRDs)" for i in install/kubernetes/helm/istio-init/files/crd*yaml -do +do ${KUBECTL} apply -f $i done @@ -30,5 +30,3 @@ ${KUBECTL} get svc -n istio-system #echo "Label namespace as istio-injection-enabled" echo "${KUBECTL} label namespace ${namespace} istio-injection=enabled" #${KUBECTL} label namespace ${namespace} istio-injection=enabled - - diff --git a/istio/demo/security/rbac/internal-http-bind.yaml b/istio/demo/security/rbac/internal-http-bind.yaml index d16ddf1..b6174b0 100644 --- a/istio/demo/security/rbac/internal-http-bind.yaml +++ b/istio/demo/security/rbac/internal-http-bind.yaml @@ -6,11 +6,10 @@ metadata: spec: subjects: - properties: - source.namespace: "istio-system" + source.namespace: "istio-system" - properties: - source.namespace: "default" + source.namespace: "default" roleRef: kind: ServiceRole name: "internal-http" #mode: PERMISSIVE - diff --git a/istio/demo/security/rbac/internal-http-role.yaml b/istio/demo/security/rbac/internal-http-role.yaml index 6c5c5a0..730a29e 100644 --- a/istio/demo/security/rbac/internal-http-role.yaml +++ b/istio/demo/security/rbac/internal-http-role.yaml @@ -11,4 +11,3 @@ spec: - key: "destination.labels[app]" values: ["esp-e1"] # values: ["esp-e1", "roxie-r1", "support"] - diff --git a/istio/demo/security/rbac/tcp-access-bind.yaml b/istio/demo/security/rbac/tcp-access-bind.yaml index cf2ecf2..beb882a 100644 --- a/istio/demo/security/rbac/tcp-access-bind.yaml +++ b/istio/demo/security/rbac/tcp-access-bind.yaml @@ -6,7 +6,7 @@ metadata: spec: subjects: - properties: - source.namespace: "default" + source.namespace: "default" roleRef: kind: ServiceRole name: "tcp-access" diff --git a/istio/demo/security/rbac/tcp-access.yaml b/istio/demo/security/rbac/tcp-access.yaml index e6fca2a..364f32a 100644 --- a/istio/demo/security/rbac/tcp-access.yaml +++ b/istio/demo/security/rbac/tcp-access.yaml @@ -9,4 +9,3 @@ spec: constrains: - key: "destination.port" values: ["22"] - diff --git a/local/LINUX.md b/local/LINUX.md index 21169f5..35926eb 100644 --- a/local/LINUX.md +++ b/local/LINUX.md @@ -4,7 +4,7 @@ - Install Docker CE https://docs.docker.com/install/ - Install Go https://golang.org/doc/install - git clone --depth=1 https://github.com/kubernetes/kubernetes.git -- cd to Kubernetes and run as root: +- cd to Kubernetes and run as root: ```console ./hack/local-up-cluster.sh ``` diff --git a/local/MACOS.md b/local/MACOS.md index a916718..29f9368 100644 --- a/local/MACOS.md +++ b/local/MACOS.md @@ -1,10 +1,10 @@ -# Getting started locally on MacOS - -### Brief Instructions -- Install Docker CE https://docs.docker.com/install/ -- Install Go https://golang.org/doc/install -- git clone --depth=1 https://github.com/kubernetes/kubernetes.git -- cd to Kubernetes and run as root: ./hack/local-up-cluster.sh - -### Reference -https://github.com/kubernetes/community/blob/master/contributors/devel/running-locally.md +# Getting started locally on MacOS + +### Brief Instructions +- Install Docker CE https://docs.docker.com/install/ +- Install Go https://golang.org/doc/install +- git clone --depth=1 https://github.com/kubernetes/kubernetes.git +- cd to Kubernetes and run as root: ./hack/local-up-cluster.sh + +### Reference +https://github.com/kubernetes/community/blob/master/contributors/devel/running-locally.md diff --git a/local/README.md b/local/README.md index 9979e0b..6852f52 100644 --- a/local/README.md +++ b/local/README.md @@ -6,7 +6,6 @@ ## Tested deployments on local Kubernetes: - Pod - Deployment/dp-1 -- local/hpcc_dns +- local/hpcc_dns - istio - elastic - diff --git a/local/WINDOWS.md b/local/WINDOWS.md index b5cfa82..fb49dbb 100644 --- a/local/WINDOWS.md +++ b/local/WINDOWS.md @@ -1 +1 @@ -Wait for WSL2 release \ No newline at end of file +Wait for WSL2 release diff --git a/local/configmap/README.md b/local/configmap/README.md index cf542bc..ab35ff7 100644 --- a/local/configmap/README.md +++ b/local/configmap/README.md @@ -6,7 +6,7 @@ ConfigMap is created with bin/bootstrap-xxx.sh or bin/bootstrap.bat The files are available on hpcc-admin container under /etc/hpcc-config-map/ Following configuration are availble now -Under hpcc directory: +Under hpcc directory: ### topology.properties This file defines how topology in environment.xml can be constructed @@ -15,7 +15,7 @@ This file defines how topology in environment.xml can be constructed Add/modify Software/Category directories as "-o" option in envgen/envgen2 ### override.envgen2 -Override all component properties with the give xpath for the given buildset with the value provided. This is "-override" option in envgen/envgen2 +Override all component properties with the give xpath for the given buildset with the value provided. This is "-override" option in envgen/envgen2 ### xpath_attributes.envgen2 @@ -28,9 +28,3 @@ Add/modify component settings with envgen2 syntax ### contents/*.xml These xml files will be inserted to environment.xml to xpath provided in in the file These files will be sorted first so users can use file names to control the execution order. - - - - - - diff --git a/local/configmap/hpcc/contents/machine.xml b/local/configmap/hpcc/contents/machine.xml index 769a0c8..0f3c184 100644 --- a/local/configmap/hpcc/contents/machine.xml +++ b/local/configmap/hpcc/contents/machine.xml @@ -5,4 +5,3 @@ name="dell-server-1" nicSpeed="1000" opSys="linux"/> - diff --git a/local/configmap/hpcc/override.envgen2 b/local/configmap/hpcc/override.envgen2 index d61f3d4..6e228b8 100644 --- a/local/configmap/hpcc/override.envgen2 +++ b/local/configmap/hpcc/override.envgen2 @@ -8,4 +8,3 @@ roxie,@copyResources,true thor,@replicateOutputs,true - diff --git a/local/configmap/hpcc/xpath_attributes.envgen2 b/local/configmap/hpcc/xpath_attributes.envgen2 index b166594..a1beb68 100644 --- a/local/configmap/hpcc/xpath_attributes.envgen2 +++ b/local/configmap/hpcc/xpath_attributes.envgen2 @@ -3,5 +3,3 @@ #Software/Topology/Cluster[@name=\\\"thor\\\"]/ThorCluster,@process,thor123 #Software/RoxieCluster/",roxieMulticastEnabled,false - - diff --git a/volumes/support-pv.yaml b/volumes/support-pv.yaml index 5aea306..de34d27 100644 --- a/volumes/support-pv.yaml +++ b/volumes/support-pv.yaml @@ -7,6 +7,6 @@ spec: storage: 160Gi accessModes: - ReadWriteMany - awsElasticBlockStore: - fsType: "ext4" + awsElasticBlockStore: + fsType: "ext4" volumeID: ""