From 1fd346d3ceb0dec2670142f5704979e7aec580fe Mon Sep 17 00:00:00 2001 From: Mike D'Amato <7297973+mddamato@users.noreply.github.com> Date: Mon, 2 Aug 2021 12:30:00 -0400 Subject: [PATCH] Refactor 2 (#54) * Refactor 2 * add assertion for ansible version Added an assertion to check for Ansible version prior to running other tasks. This is due to the Ubuntu AMI failing with the standard Ansible install (apt install ansible) as the version in apt is 2.9.6 - which fails due to a missing critical update packaged into 2.9.8. If the user runs the script with anything earlier than 2.9.8, it will fail and request that they upgrade Ansible. * Revert "add assertion for ansible version" This reverts commit 486df1e76a57cedaa5049c9a78082cea0b7aa87b. * Add changes from review Co-authored-by: Michael DAmato Co-authored-by: Tucker Blue --- .gitignore | 6 +- .yamllint | 2 + README.md | 2 + ansible.cfg | 2 +- inventory/sample/group_vars/rke2_agents.yml | 18 +-- inventory/sample/group_vars/rke2_servers.yml | 37 ++---- inventory/sample/hosts.ini | 4 +- roles/rke2_agent/tasks/main.yml | 25 +--- roles/rke2_common/defaults/main.yml | 2 + .../rke2_common/tasks/add-manifest-addons.yml | 9 ++ roles/rke2_common/tasks/airgap.yml | 57 -------- roles/rke2_common/tasks/cis-hardening.yml | 6 +- roles/rke2_common/tasks/config.yml | 48 ++----- .../tasks/images_tarball_install.yml | 28 ++++ roles/rke2_common/tasks/iptables_rules.yml | 123 ++++++++++++++++++ roles/rke2_common/tasks/main.yml | 51 ++++---- .../tasks/network_manager_fix.yaml | 41 ++++++ .../tasks/{yum.yml => rpm_install.yml} | 37 +++++- roles/rke2_common/tasks/tarball_install.yml | 60 +++++---- roles/rke2_server/tasks/first_server.yml | 40 +++++- roles/rke2_server/tasks/install.yml | 25 ---- roles/rke2_server/tasks/main.yml | 6 + roles/rke2_server/tasks/other_servers.yml | 49 +++++-- sample_files/manifest/manifest-example.yaml | 28 ++++ site.yml | 8 +- 25 files changed, 454 insertions(+), 260 deletions(-) create mode 100644 roles/rke2_common/tasks/add-manifest-addons.yml delete mode 100644 roles/rke2_common/tasks/airgap.yml create mode 100644 roles/rke2_common/tasks/images_tarball_install.yml create mode 100644 roles/rke2_common/tasks/iptables_rules.yml create mode 100644 roles/rke2_common/tasks/network_manager_fix.yaml rename roles/rke2_common/tasks/{yum.yml => rpm_install.yml} (75%) delete mode 100644 roles/rke2_server/tasks/install.yml create mode 100644 sample_files/manifest/manifest-example.yaml diff --git a/.gitignore b/.gitignore index 2e8ae682..782a0c73 100644 --- a/.gitignore +++ b/.gitignore @@ -7,5 +7,7 @@ test_inventory* rke2-images.linux-amd64.tar.gz rke2.linux-amd64.tar.gz -tarball_install/rke2-images.linux-amd64.tar.gz -tarball_install/rke2.linux-amd64.tar.gz \ No newline at end of file + + +tarball_install/* +!tarball_install/README.md \ No newline at end of file diff --git a/.yamllint b/.yamllint index 02ca2a27..b81f5ac1 100644 --- a/.yamllint +++ b/.yamllint @@ -10,3 +10,5 @@ rules: ignore: | .github/ + inventory/sample/group_vars/rke2_servers.yml + inventory/sample/group_vars/rke2_agents.yml diff --git a/README.md b/README.md index e31efa14..ef888fa7 100644 --- a/README.md +++ b/README.md @@ -132,3 +132,5 @@ Author Information [Brandon Gulla](https://github.com/bgulla) [Rancher Federal](https://rancherfederal.com/) + +[Mike D'Amato](https://github.com/mdamato) diff --git a/ansible.cfg b/ansible.cfg index 504fb982..cea44d1e 100644 --- a/ansible.cfg +++ b/ansible.cfg @@ -6,7 +6,7 @@ inventory = ./inventory/my-cluster/hosts.ini remote_tmp = $HOME/.ansible/tmp local_tmp = $HOME/.ansible/tmp pipelining = True -become = True +#become = True host_key_checking = False deprecation_warnings = False callback_whitelist = profile_roles, timer diff --git a/inventory/sample/group_vars/rke2_agents.yml b/inventory/sample/group_vars/rke2_agents.yml index 0c3648fe..e9d13353 100644 --- a/inventory/sample/group_vars/rke2_agents.yml +++ b/inventory/sample/group_vars/rke2_agents.yml @@ -1,18 +1,10 @@ --- -# Primary RKE2 agent configuration parameters. -# See https://docs.rke2.io/install/install_options/agent_config/ for more information. -rke2_config: - selinux: true - profile: cis-1.5 - -# node-label -rke2_node_labels: - - "ansible-provisioned-agent=true" - -# kubelet-arg -rke2_kubelet_args: - - "feature-gates=DynamicKubeletConfig=false" +# Primary RKE2 agent configuration parameters. Remove the curly braces ( {} ) and add your configuration. +# See https://docs.rke2.io/install/install_options/linux_agent_config/ for all configuration options. +rke2_config: {} + # debug: false # See https://docs.rke2.io/install/containerd_registry_configuration/ +# Add a registry configuration file by specifying the file path on the control host # registry_config_file_path: "{{ playbook_dir }}/sample_files/registries.yaml" diff --git a/inventory/sample/group_vars/rke2_servers.yml b/inventory/sample/group_vars/rke2_servers.yml index c2cdea9b..1b10d1e5 100644 --- a/inventory/sample/group_vars/rke2_servers.yml +++ b/inventory/sample/group_vars/rke2_servers.yml @@ -1,34 +1,17 @@ --- -# Primary RKE2 server configuration parameters. -# See https://docs.rke2.io/install/install_options/server_config/ for more information. - -rke2_config: - selinux: true - write-kubeconfig-mode: "0640" - profile: cis-1.5 - -# node-label -rke2_node_labels: - - "ansible-provisioned-server=true" - -# kube-apiserver-arg -rke2_kube_api_server_args: - - "tls-min-version=VersionTLS12" - -# kube-scheduler-arg -rke2_kube_scheduler_args: - - "tls-min-version=VersionTLS12" - -# kube-controller-manager-arg -rke2_kube_controller_manager_args: - - "tls-min-version=VersionTLS12" - -# kubelet-arg -rke2_kubelet_args: - - "feature-gates=DynamicKubeletConfig=false" +# Primary RKE2 server configuration parameters. Remove the curly braces ( {} ) and add your configuration. +# See https://docs.rke2.io/install/install_options/server_config/ for all configuration options. +rke2_config: {} + # debug: false # See https://kubernetes.io/docs/tasks/debug-application-cluster/audit/ +# Add a policy configuration file by specifying the file path on the control host # audit_policy_config_file_path: "{{ playbook_dir }}/sample_files/audit-policy.yaml" # See https://docs.rke2.io/install/containerd_registry_configuration/ +# Add a registry configuration file by specifying the file path on the control host # registry_config_file_path: "{{ playbook_dir }}/sample_files/registries.yaml" + +# See https://docs.rke2.io/helm/#automatically-deploying-manifests-and-helm-charts +# Add manifest files by specifying the directory path on the control host +# manifest_config_file_path: "{{ playbook_dir }}/sample_files/manifest/" \ No newline at end of file diff --git a/inventory/sample/hosts.ini b/inventory/sample/hosts.ini index 9fb8c7c7..7c5f7126 100644 --- a/inventory/sample/hosts.ini +++ b/inventory/sample/hosts.ini @@ -1,6 +1,6 @@ [rke2_servers] -; host0 -; host1 +; host0 extra_node_labels='["extraLabel0=true", "extraLabel1=true"]' +; host1 extra_node_labels='["extraLabel2=true", "extraLabel3=true"]' ; host2 [rke2_agents] diff --git a/roles/rke2_agent/tasks/main.yml b/roles/rke2_agent/tasks/main.yml index 9301a78a..46f532d3 100644 --- a/roles/rke2_agent/tasks/main.yml +++ b/roles/rke2_agent/tasks/main.yml @@ -1,21 +1,9 @@ --- -- name: YUM-Based | Install rke2-agent - yum: - name: rke2-agent - state: latest # noqa package-latest - when: - - ansible_facts['os_family'] == 'RedHat' - - not got_rke2_install_ball.stat.exists -- name: TARBALL | Moving Systemd units to /etc/systemd/system - ansible.builtin.copy: - src: /usr/local/lib/systemd/system/rke2-agent.service - dest: /etc/systemd/system/rke2-agent.service - mode: '0644' - owner: root - group: root - remote_src: yes - when: ansible_facts['os_family'] != 'RedHat' or got_rke2_install_ball.stat.exists +- name: RKE2 agent and server tasks + include_role: + name: rke2_common + tasks_from: main - name: Add server url to config file lineinfile: @@ -27,11 +15,6 @@ dest: /etc/rancher/rke2/config.yaml line: "token: {{ hostvars[groups['rke2_servers'][0]].rke2_config_token }}" -- name: Run CIS-Hardening Tasks - include_role: - name: rke2_common - tasks_from: cis-hardening - - name: Start rke2-agent ansible.builtin.systemd: name: rke2-agent diff --git a/roles/rke2_common/defaults/main.yml b/roles/rke2_common/defaults/main.yml index 8a6a6879..ba25faab 100644 --- a/roles/rke2_common/defaults/main.yml +++ b/roles/rke2_common/defaults/main.yml @@ -3,3 +3,5 @@ tarball_dir: "/usr/local" rke2_channel: stable audit_policy_config_file_path: "" registry_config_file_path: "" +add_iptables_rules: false +rke2_config: {} diff --git a/roles/rke2_common/tasks/add-manifest-addons.yml b/roles/rke2_common/tasks/add-manifest-addons.yml new file mode 100644 index 00000000..fc43461e --- /dev/null +++ b/roles/rke2_common/tasks/add-manifest-addons.yml @@ -0,0 +1,9 @@ +--- + +- name: Add manifest addons files + copy: + src: "{{ manifest_config_file_path }}" + dest: "/var/lib/rancher/rke2/server/manifests/" + mode: '0640' + owner: root + group: root diff --git a/roles/rke2_common/tasks/airgap.yml b/roles/rke2_common/tasks/airgap.yml deleted file mode 100644 index 08eef74c..00000000 --- a/roles/rke2_common/tasks/airgap.yml +++ /dev/null @@ -1,57 +0,0 @@ ---- -- name: Air-Gap Images - become: yes - when: got_images_gz.stat.exists or got_images_zst.stat.exists - block: - - name: Does the airgap images folder exist? - stat: - path: /var/lib/rancher/rke2/agent/images - register: rke2_agent_images - - - name: create the airgap images folder - file: - path: /var/lib/rancher/rke2/agent/images - state: directory - recurse: yes - when: not rke2_agent_images.stat.exists - - - name: Add images tar.gz to needed directory if provided - copy: - src: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.gz" - dest: /var/lib/rancher/rke2/agent/images/ - mode: '0644' - when: got_images_gz.stat.exists - - - name: Add images tar.zst to needed directory if provided - copy: - src: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.zst" - dest: /var/lib/rancher/rke2/agent/images/ - mode: '0644' - when: got_images_zst.stat.exists - -- name: Air-Gap Tarball Install - become: yes - when: got_rke2_install_ball.stat.exists == true - block: - - name: TARBALL | Make temp dir - ansible.builtin.tempfile: - state: directory - suffix: rke2-install.XXXXXXXXXX - register: temp_dir - - - name: Add tarball to needed directory if provided - copy: - src: "{{ playbook_dir }}/tarball_install/rke2.linux-amd64.tar.gz" - dest: "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" - mode: '0644' - - - name: TARBALL | Extract the tarball # noqa command-instead-of-module - command: - cmd: tar -xf "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" -C "{{ tarball_dir }}" - changed_when: false - - - name: TARBALL | Remove the temp_dir - ansible.builtin.file: - path: "{{ temp_dir.path }}" - state: absent - when: temp_dir.path is defined diff --git a/roles/rke2_common/tasks/cis-hardening.yml b/roles/rke2_common/tasks/cis-hardening.yml index 409a7619..49d369f5 100644 --- a/roles/rke2_common/tasks/cis-hardening.yml +++ b/roles/rke2_common/tasks/cis-hardening.yml @@ -27,18 +27,20 @@ src: /usr/share/rke2/rke2-cis-sysctl.conf dest: /etc/sysctl.d/60-rke2-cis.conf remote_src: yes + mode: 0600 register: sysctl_operation_yum when: - ansible_os_family == 'RedHat' - - not got_rke2_install_ball.stat.exists + - not rke2_binary_tarball_check.stat.exists - name: Copy systemctl file for kernel hardening for non-yum installs ansible.builtin.copy: src: /usr/local/share/rke2/rke2-cis-sysctl.conf dest: /etc/sysctl.d/60-rke2-cis.conf remote_src: yes + mode: 0600 register: sysctl_operation_tarball - when: ansible_facts['os_family'] != 'RedHat' or got_rke2_install_ball.stat.exists + when: ansible_facts['os_family'] != 'RedHat' or rke2_binary_tarball_check.stat.exists - name: restart systemd-sysctl service: diff --git a/roles/rke2_common/tasks/config.yml b/roles/rke2_common/tasks/config.yml index 611d65bb..41843b5b 100644 --- a/roles/rke2_common/tasks/config.yml +++ b/roles/rke2_common/tasks/config.yml @@ -14,45 +14,21 @@ owner: root group: root +- name: Set a fact + ansible.builtin.set_fact: + all_node_labels: "{{ rke2_config['node-label'] | default([]) }} + {{ extra_node_labels | default([]) }}" + +- name: Add node labels to rke2_config + ansible.utils.update_fact: + updates: + - path: rke2_config["node-label"] + value: "{{ all_node_labels }}" + register: updated_rke2_config + - name: Add primary configuration items copy: - content: "{{ rke2_config | to_nice_yaml(indent=0) }}" + content: "{{ updated_rke2_config.rke2_config | to_nice_yaml(indent=0) }}" dest: /etc/rancher/rke2/config.yaml mode: "0640" owner: root group: root - -- name: Add rke2_kube_api_server_args - lineinfile: - path: /etc/rancher/rke2/config.yaml - line: "kube-apiserver-arg: {{ item }}" - with_items: - - "{{ rke2_kube_api_server_args | default([]) }}" - -- name: Add rke2_kube_scheduler_args - lineinfile: - path: /etc/rancher/rke2/config.yaml - line: "kube-scheduler-arg: {{ item }}" - with_items: - - "{{ rke2_kube_scheduler_args | default([]) }}" - -- name: Add rke2_kube_controller_manager_args - lineinfile: - path: /etc/rancher/rke2/config.yaml - line: "kube-controller-manager-arg: {{ item }}" - with_items: - - "{{ rke2_kube_controller_manager_args | default([]) }}" - -- name: Add rke2_kubelet_args - lineinfile: - path: /etc/rancher/rke2/config.yaml - line: "kubelet-arg: {{ item }}" - with_items: - - "{{ rke2_kubelet_args | default([]) }}" - -- name: Add node-labels - lineinfile: - path: /etc/rancher/rke2/config.yaml - line: "node-label: {{ item }}" - with_items: - - "{{ rke2_node_labels | default([]) }}" diff --git a/roles/rke2_common/tasks/images_tarball_install.yml b/roles/rke2_common/tasks/images_tarball_install.yml new file mode 100644 index 00000000..7233da6e --- /dev/null +++ b/roles/rke2_common/tasks/images_tarball_install.yml @@ -0,0 +1,28 @@ +--- +- name: "Check for images tar.gz in {{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.gz" + stat: + path: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.gz" + register: got_images_gz + delegate_to: 127.0.0.1 + become: no + +- name: "Check for images tar.zst in {{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.zst" + stat: + path: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.zst" + register: got_images_zst + delegate_to: 127.0.0.1 + become: no + +- name: Add images tar.gz to needed directory if provided + copy: + src: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.gz" + dest: /var/lib/rancher/rke2/agent/images/ + mode: '0644' + when: got_images_gz.stat.exists + +- name: Add images tar.zst to needed directory if provided + copy: + src: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.zst" + dest: /var/lib/rancher/rke2/agent/images/ + mode: '0644' + when: got_images_zst.stat.exists diff --git a/roles/rke2_common/tasks/iptables_rules.yml b/roles/rke2_common/tasks/iptables_rules.yml new file mode 100644 index 00000000..82e09f6a --- /dev/null +++ b/roles/rke2_common/tasks/iptables_rules.yml @@ -0,0 +1,123 @@ +--- + +- name: Allow 9345 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "9345" + ctstate: NEW + jump: ACCEPT + when: inventory_hostname in groups['rke2_servers'] + +- name: Allow 6443 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "6443" + ctstate: NEW + jump: ACCEPT + when: inventory_hostname in groups['rke2_servers'] + +- name: Allow udp 8472 + iptables: + action: insert + chain: INPUT + protocol: udp + destination_port: "8472" + ctstate: NEW + jump: ACCEPT + +- name: Allow 10250 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "10250" + ctstate: NEW + jump: ACCEPT + +- name: Allow 2379 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "2379" + ctstate: NEW + jump: ACCEPT + when: inventory_hostname in groups['rke2_servers'] + +- name: Allow 2380 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "2380" + ctstate: NEW + jump: ACCEPT + when: inventory_hostname in groups['rke2_servers'] + +- name: Allow 30000:32767 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "30000:32767" + ctstate: NEW + jump: ACCEPT + +- name: Allow 443 + iptables: + action: insert + chain: INPUT + protocol: tcp + destination_port: "443" + ctstate: NEW + jump: ACCEPT + +- name: "Allow cluster-cidr forward" + ansible.builtin.iptables: + action: insert + chain: FORWARD + source: '{{ rke2_config["cluster-cidr"] | default("10.42.0.0/16") | string }}' + jump: ACCEPT + +- name: "Allow cluster-cidr forward" + ansible.builtin.iptables: + action: insert + chain: FORWARD + destination: '{{ rke2_config["cluster-cidr"] | default("10.42.0.0/16") | string }}' + jump: ACCEPT + +- name: "Allow cluster-cidr input" + ansible.builtin.iptables: + action: insert + chain: INPUT + source: '{{ rke2_config["cluster-cidr"] | default("10.42.0.0/16") | string }}' + jump: ACCEPT + +- name: "Allow cluster-cidr input" + ansible.builtin.iptables: + action: insert + chain: INPUT + destination: '{{ rke2_config["cluster-cidr"] | default("10.42.0.0/16") | string }}' + jump: ACCEPT + +- name: save iptables + shell: "iptables-save > /etc/sysconfig/iptables" + become: true + changed_when: false + when: ansible_facts['os_family'] == 'RedHat' + +- name: Install iptables-persistent + apt: + name: iptables-persistent + state: present + when: ansible_facts['os_family'] == 'Debian' + +- name: save iptables + shell: "iptables-save > /etc/iptables/rules.v4" + become: true + changed_when: false + when: ansible_facts['os_family'] == 'Debian' diff --git a/roles/rke2_common/tasks/main.yml b/roles/rke2_common/tasks/main.yml index 24561307..d0372923 100644 --- a/roles/rke2_common/tasks/main.yml +++ b/roles/rke2_common/tasks/main.yml @@ -7,48 +7,32 @@ ansible.builtin.package_facts: manager: auto -- name: Is the images tar.gz in the tarball_install/ diretory - stat: - path: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.gz" - register: got_images_gz - delegate_to: 127.0.0.1 - become: no +- include: images_tarball_install.yml -- name: Is the images tar.zst in the tarball_install/ diretory - stat: - path: "{{ playbook_dir }}/tarball_install/rke2-images.linux-amd64.tar.zst" - register: got_images_zst - delegate_to: 127.0.0.1 - become: no - -- name: Is the install tarball in the tarball_install/ diretory +- name: "Check for binary tarball in {{ playbook_dir }}/tarball_install/rke2.linux-amd64.tar.gz" stat: path: "{{ playbook_dir }}/tarball_install/rke2.linux-amd64.tar.gz" - register: got_rke2_install_ball + register: rke2_binary_tarball_check delegate_to: 127.0.0.1 become: no -- name: RHEL/CentOS requirements +- name: SLES/Ubuntu/Tarball Installation + include: tarball_install.yml + when: + - ansible_facts['os_family'] != 'RedHat' or rke2_binary_tarball_check.stat.exists == true + +- name: RHEL/CentOS Installation block: - # For use with ansible_lsb - name: Install redhat-lsb-core ansible.builtin.yum: name=redhat-lsb-core state=present when: "'redhat-lsb-core' not in ansible_facts.packages" - name: Reread ansible_lsb facts ansible.builtin.setup: filter=ansible_lsb* when: "'redhat-lsb-core' not in ansible_facts.packages" - - # Setup Yum repos - - include: yum.yml + - include: rpm_install.yml when: - ansible_os_family == 'RedHat' - - got_rke2_install_ball.stat.exists == false - -- name: Tarball install for SLES and Ubuntu - include: tarball_install.yml - when: - - ansible_facts['os_family'] != 'RedHat' - - got_rke2_install_ball.stat.exists == false + - rke2_binary_tarball_check.stat.exists == false # Disable Firewalld # We recommend disabling firewalld. For Kubernetes 1.19+, firewalld must be turned off. @@ -59,12 +43,23 @@ enabled: no when: ansible_facts.services["firewalld.service"] is defined +- include: network_manager_fix.yaml + - include: config.yml +- name: Add server iptables rules + include_tasks: iptables_rules.yml + when: + - ansible_facts.services["iptables.service"] is defined + - add_iptables_rules is true + - include: add-audit-policy-config.yml when: audit_policy_config_file_path | length > 0 - include: add-registry-config.yml when: registry_config_file_path | length > 0 -- include: airgap.yml +- name: Run CIS-Hardening Tasks + include_role: + name: rke2_common + tasks_from: cis-hardening diff --git a/roles/rke2_common/tasks/network_manager_fix.yaml b/roles/rke2_common/tasks/network_manager_fix.yaml new file mode 100644 index 00000000..34842b31 --- /dev/null +++ b/roles/rke2_common/tasks/network_manager_fix.yaml @@ -0,0 +1,41 @@ +--- + +# This fixes known issue with NetworkManager +# https://docs.rke2.io/known_issues/#networkmanager + +- name: Add NetworkManager fix to rke2-canal.conf + blockinfile: + path: /etc/NetworkManager/conf.d/rke2-canal.conf + block: | + [keyfile] + unmanaged-devices=interface-name:cali*;interface-name:flannel* + create: yes + mode: 0600 + when: ansible_facts.services["NetworkManager.service"] is defined + +- name: Does rke2-canal.conf exist + stat: + path: /etc/NetworkManager/conf.d/rke2-canal.conf + register: rke2_canal_file + +- name: Set rke2-canal.conf file permissions + ansible.builtin.file: + path: /etc/NetworkManager/conf.d/rke2-canal.conf + mode: '0600' + owner: root + group: root + when: rke2_canal_file.stat.exists + +- name: Disable service nm-cloud-setup + ansible.builtin.systemd: + name: nm-cloud-setup.service + enabled: no + state: stopped + when: ansible_facts.services["nm-cloud-setup.service"] is defined + +- name: Disable nm-cloud-setup.timer unit + ansible.builtin.systemd: + name: nm-cloud-setup.timer + state: stopped + enabled: no + when: ansible_facts.services["nm-cloud-setup.service"] is defined diff --git a/roles/rke2_common/tasks/yum.yml b/roles/rke2_common/tasks/rpm_install.yml similarity index 75% rename from roles/rke2_common/tasks/yum.yml rename to roles/rke2_common/tasks/rpm_install.yml index 82cef8ab..ce90a201 100644 --- a/roles/rke2_common/tasks/yum.yml +++ b/roles/rke2_common/tasks/rpm_install.yml @@ -1,5 +1,4 @@ --- - - name: Stop if the provided channel is not valid fail: msg: "Provided channel is not valid" @@ -34,10 +33,9 @@ - name: Describe versions debug: - msg: - - "Full version: {{ rke2_full_version.stdout }}" - - "dot version: {{ rke2_version_dot.stdout }}" - - "Maj.Min version: {{ rke2_version.stdout }}" + msg: > + Full version: {{ rke2_full_version.stdout }}, dot version: {{ rke2_version_dot.stdout }} + Maj.Min version: {{ rke2_version.stdout }}" # Does the Rancher RKE2 Common repo exist already - name: Check to see if rke2-common.repo exists @@ -92,3 +90,32 @@ gpgkey: "https://rpm.rancher.io/public.key" enabled: yes when: (not stat_rke2_versioned_repo.stat.exists) and (ansible_lsb.major_release == '8') + +- name: YUM-Based | Install rke2-server + yum: + name: rke2-server + state: latest # noqa package-latest + when: + - ansible_facts['os_family'] == 'RedHat' + - not rke2_binary_tarball_check.stat.exists + - inventory_hostname in groups['rke2_servers'] + +- name: YUM-Based | Install rke2-agent + yum: + name: rke2-agent + state: latest # noqa package-latest + when: + - ansible_facts['os_family'] == 'RedHat' + - not rke2_binary_tarball_check.stat.exists + - inventory_hostname in groups['rke2_agents'] + +- name: TARBALL | Moving Systemd units to /etc/systemd/system + ansible.builtin.copy: + src: /usr/local/lib/systemd/system/rke2-server.service + dest: /etc/systemd/system/rke2-server.service + mode: '0644' + owner: root + group: root + remote_src: yes + when: + - ansible_facts['os_family'] != 'RedHat' or rke2_binary_tarball_check.stat.exists diff --git a/roles/rke2_common/tasks/tarball_install.yml b/roles/rke2_common/tasks/tarball_install.yml index c21ca624..fe70fca0 100644 --- a/roles/rke2_common/tasks/tarball_install.yml +++ b/roles/rke2_common/tasks/tarball_install.yml @@ -11,8 +11,21 @@ # unpack_tarball # } +- name: TARBALL | Make temp dir + ansible.builtin.tempfile: + state: directory + suffix: rke2-install.XXXXXXXXXX + register: temp_dir -- name: TARBALL | Get Release Version +- name: Send provided tarball if available + copy: + src: "{{ playbook_dir }}/tarball_install/rke2.linux-amd64.tar.gz" + dest: "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" + mode: '0644' + when: rke2_binary_tarball_check.stat.exists + +- name: Download tarball + when: rke2_binary_tarball_check.stat.exists == false block: - name: Stop if the provided channel is not valid fail: @@ -43,7 +56,9 @@ delegate_to: 127.0.0.1 - name: Set Maj.Min version - shell: set -o pipefail && echo {{ rke2_full_version.stdout }} | awk -F'.' '{ print $1"."$2 }' | sed "s|^v||g" + shell: >- + set -o pipefail && echo {{ rke2_full_version.stdout }} + | awk -F'.' '{ print $1"."$2 }' | sed "s|^v||g" register: rke2_version changed_when: false args: @@ -57,6 +72,22 @@ - "dot version: {{ rke2_version_dot.stdout }}" - "Maj.Min version: {{ rke2_version.stdout }}" + - name: TARBALL | Download the tarball + get_url: + url: https://github.com/rancher/rke2/releases/download/{{ rke2_full_version.stdout }}/rke2.linux-amd64.tar.gz + dest: "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" + +- name: TARBALL | Extract the tarball # noqa command-instead-of-module + command: + cmd: tar -xf "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" -C "{{ tarball_dir }}" + changed_when: false + +- name: TARBALL | Remove the temp_dir + ansible.builtin.file: + path: "{{ temp_dir.path }}" + state: absent + when: temp_dir.path is defined + - name: TARBALL | Check Target Mountpoint command: mountpoint -q {{ tarball_dir }} register: tarball_dir_stat @@ -69,27 +100,6 @@ when: tarball_dir_stat.rc == 0 - name: TARBALL | Using /opt/rke2 - debug: msg="Using /opt/rke2 for install directory" + debug: + msg: "Using /opt/rke2 for install directory" when: tarball_dir_stat.rc == 0 - -- name: TARBALL | Make temp dir - ansible.builtin.tempfile: - state: directory - suffix: rke2-install.XXXXXXXXXX - register: temp_dir - -- name: TARBALL | Download the tarball - get_url: - url: https://github.com/rancher/rke2/releases/download/{{ rke2_full_version.stdout }}/rke2.linux-amd64.tar.gz - dest: "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" - -- name: TARBALL | Extract the tarball # noqa command-instead-of-module - command: - cmd: tar -xf "{{ temp_dir.path }}/rke2.linux-amd64.tar.gz" -C "{{ tarball_dir }}" - changed_when: false - -- name: TARBALL | Use the registered var and the file module to remove the temporary file - ansible.builtin.file: - path: "{{ temp_dir.path }}" - state: absent - when: temp_dir.path is defined diff --git a/roles/rke2_server/tasks/first_server.yml b/roles/rke2_server/tasks/first_server.yml index a299a8b9..b55fa26a 100644 --- a/roles/rke2_server/tasks/first_server.yml +++ b/roles/rke2_server/tasks/first_server.yml @@ -1,7 +1,10 @@ --- -- name: Install - include_tasks: install.yml +- name: Add manifest files + include_role: + name: rke2_common + tasks_from: add-manifest-addons.yml + when: manifest_config_file_path is defined - name: Start rke2-server ansible.builtin.systemd: @@ -9,6 +12,39 @@ state: started enabled: yes +- name: Wait for k8s apiserver + wait_for: + host: "{{ kubernetes_api_server_host }}" + port: "6443" + state: present + timeout: 300 + +- name: Wait for kubelet process to be present on host + ansible.builtin.command: >- + ps -C kubelet -F -ww --no-headers + register: kubelet_check + until: kubelet_check.rc == 0 + retries: 20 + delay: 10 + changed_when: false + +- name: Extract the hostname-override parameter from the kubelet process + set_fact: + kubelet_hostname_override_parameter: "{{ kubelet_check.stdout |\ + regex_search('\\s--hostname-override=((([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\\-]*[A-Za-z0-9]))\\s',\ + '\\1') }}" + +- name: Wait for node to show Ready status + ansible.builtin.command: >- + /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml + --server https://127.0.0.1:6443 get no {{ kubelet_hostname_override_parameter[0] }} + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + register: status_result + until: status_result.stdout.find("True") != -1 + retries: 20 + delay: 10 + changed_when: false + - name: Add generated Token if none provided block: - name: Wait for node-token diff --git a/roles/rke2_server/tasks/install.yml b/roles/rke2_server/tasks/install.yml deleted file mode 100644 index 86a1d7a9..00000000 --- a/roles/rke2_server/tasks/install.yml +++ /dev/null @@ -1,25 +0,0 @@ ---- - -- name: YUM-Based | Install rke2-server - yum: - name: rke2-server - state: latest # noqa package-latest - when: - - ansible_facts['os_family'] == 'RedHat' - - not got_rke2_install_ball.stat.exists - -- name: TARBALL | Moving Systemd units to /etc/systemd/system - ansible.builtin.copy: - src: /usr/local/lib/systemd/system/rke2-server.service - dest: /etc/systemd/system/rke2-server.service - mode: '0644' - owner: root - group: root - remote_src: yes - when: - - ansible_facts['os_family'] != 'RedHat' or got_rke2_install_ball.stat.exists - -- name: Run CIS-Hardening Tasks - include_role: - name: rke2_common - tasks_from: cis-hardening diff --git a/roles/rke2_server/tasks/main.yml b/roles/rke2_server/tasks/main.yml index 0310caa2..3ff3064b 100644 --- a/roles/rke2_server/tasks/main.yml +++ b/roles/rke2_server/tasks/main.yml @@ -1,4 +1,10 @@ --- + +- name: RKE2 agent and server tasks + include_role: + name: rke2_common + tasks_from: main + - name: Setup initial server include_tasks: first_server.yml when: inventory_hostname in groups['rke2_servers'][0] diff --git a/roles/rke2_server/tasks/other_servers.yml b/roles/rke2_server/tasks/other_servers.yml index eb75fbcd..9e871db9 100644 --- a/roles/rke2_server/tasks/other_servers.yml +++ b/roles/rke2_server/tasks/other_servers.yml @@ -1,8 +1,5 @@ --- -- name: Install - include_tasks: install.yml - - name: Add server url to config file lineinfile: dest: /etc/rancher/rke2/config.yaml @@ -13,8 +10,44 @@ dest: /etc/rancher/rke2/config.yaml line: "token: {{ hostvars[groups['rke2_servers'][0]].rke2_config_token }}" -- name: Start rke2-server - ansible.builtin.systemd: - name: rke2-server - state: started - enabled: yes +- name: Start and wait for healthy node + throttle: 1 + block: + - name: Start rke2-server + ansible.builtin.systemd: + name: rke2-server + state: started + enabled: yes + + - name: Wait for k8s apiserver reachability + wait_for: + host: "{{ kubernetes_api_server_host }}" + port: "6443" + state: present + timeout: 300 + + - name: Wait for kubelet process to be present on host + ansible.builtin.command: >- + ps -C kubelet -F -ww --no-headers + register: kubelet_check + until: kubelet_check.rc == 0 + retries: 20 + delay: 10 + changed_when: false + + - name: Extract the hostname-override parameter from the kubelet process + set_fact: + kubelet_hostname_override_parameter: "{{ kubelet_check.stdout |\ + regex_search('\\s--hostname-override=((([a-zA-Z]|[a-zA-Z][a-zA-Z0-9\\-]*[a-zA-Z0-9])\\.)*([A-Za-z]|[A-Za-z][A-Za-z0-9\\-]*[A-Za-z0-9]))\\s',\ + '\\1') }}" + + - name: Wait for node to show Ready status + ansible.builtin.command: >- + /var/lib/rancher/rke2/bin/kubectl --kubeconfig /etc/rancher/rke2/rke2.yaml + --server https://127.0.0.1:6443 get no {{ kubelet_hostname_override_parameter[0] }} + -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' + register: status_result + until: status_result.stdout.find("True") != -1 + retries: 20 + delay: 10 + changed_when: false diff --git a/sample_files/manifest/manifest-example.yaml b/sample_files/manifest/manifest-example.yaml new file mode 100644 index 00000000..64a1db69 --- /dev/null +++ b/sample_files/manifest/manifest-example.yaml @@ -0,0 +1,28 @@ +# See https://docs.rke2.io/helm/ +# In this directory "manifest" you can add every kubernetes .yaml file that you will add to your rke2 cluster. +# "kubectl apply" every file on this directory + +# See https://docs.rke2.io/install/network_options/#canal-options +# You can also override pre-install helm chart like canal, nginx-ingress, etc. + +# Example from https://docs.rke2.io/helm/#using-the-helm-crd + +# apiVersion: helm.cattle.io/v1 +# kind: HelmChart +# metadata: +# name: grafana +# namespace: kube-system +# spec: +# chart: stable/grafana +# targetNamespace: monitoring +# set: +# adminPassword: "NotVerySafePassword" +# valuesContent: |- +# image: +# tag: master +# env: +# GF_EXPLORE_ENABLED: true +# adminUser: admin +# sidecar: +# datasources: +# enabled: true diff --git a/site.yml b/site.yml index a98b9b8d..8d905075 100644 --- a/site.yml +++ b/site.yml @@ -1,17 +1,13 @@ --- -- hosts: rke2_cluster - gather_facts: yes - become: yes - roles: - - role: rke2_common - - hosts: rke2_servers become: yes roles: - role: rke2_server + serial: 3 - hosts: rke2_agents become: yes roles: - role: rke2_agent + serial: 3