diff --git a/roles/create_infrastructure/defaults/main.yml b/roles/create_infrastructure/defaults/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..2ec06f1d2225a28aaeb478d015b20002749c8f8e --- /dev/null +++ b/roles/create_infrastructure/defaults/main.yml @@ -0,0 +1,19 @@ +ssh_key_name: "rke2-{{ cluster_name }}-ssh-key" +ssh_key_file: "{{ ssh_key_name }}.pem" +network_name: "rke2-{{ cluster_name }}" +subnet_name: "rke2-{{ cluster_name }}" +cidr: 10.0.0.0/24 +router_name: "rke2-{{ cluster_name }}" + +server_volume_size: 50 +agent_volume_size: 100 +server_flavor: m1a.large +agent_flavor: m1a.xlarge +image: 1fe615f0-9dad-447d-bf54-9071defafb77 + +server_count: 3 +agent_count: 3 + +loadbalancer_name: "rke2_{{ cluster_name }}" +security_group: "rke2_{{ cluster_name }}" +state: present diff --git a/roles/create_infrastructure/tasks/inventory.yml b/roles/create_infrastructure/tasks/inventory.yml new file mode 100644 index 0000000000000000000000000000000000000000..c50e818118f0182ded2c4fb5af064dca1777cc39 --- /dev/null +++ b/roles/create_infrastructure/tasks/inventory.yml @@ -0,0 +1,60 @@ +- block: + # - pause: + # seconds: 5 + + - openstack.cloud.server_info: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + server: "rke2-{{ cluster_name }}-server-{{ item }}" + loop: "{{ range(1, agent_count, 1) | list }}" + register: servers_result + + - openstack.cloud.server_info: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + server: "rke2-{{ cluster_name }}-agent-{{ item }}" + loop: "{{ range(0, agent_count, 1) | list }}" + register: agents_result + + - set_fact: + agents: "{{ agents_result.results | community.general.json_query('[].openstack_servers[].{name: name, private_v4: private_v4}') }}" + servers: "{{ servers_result.results | community.general.json_query('[].openstack_servers[].{name: name, private_v4: private_v4}') }}" + + when: agents | length > 0 and agents[0].private_v4 == '' or servers | length > 1 and servers[0].private_v4 == '' + + +- name: update inventory in project + template: + src: hosts.j2 + dest: hosts + +- name: create inventory folders + file: + path: "{{ item }}" + state: directory + loop: + - group_vars + - group_vars/all + - group_vars/master + - group_vars/servers + - group_vars/agents + +- name: update all inventory + template: + src: all.yml.j2 + dest: group_vars/all/infrastructure.yml + +- name: update master inventory + template: + src: master.yml.j2 + dest: group_vars/master/infrastructure.yml + +- name: update servers inventory + template: + src: servers.yml.j2 + dest: group_vars/servers/infrastructure.yml + +- name: update agents inventory + template: + src: agents.yml.j2 + dest: group_vars/agents/infrastructure.yml diff --git a/roles/create_infrastructure/tasks/keypair.yml b/roles/create_infrastructure/tasks/keypair.yml new file mode 100644 index 0000000000000000000000000000000000000000..4e1f3b23bea1f5c2430350c515d8353845e4d737 --- /dev/null +++ b/roles/create_infrastructure/tasks/keypair.yml @@ -0,0 +1,16 @@ +- name: ssh key pair + openstack.cloud.keypair: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ ssh_key_name }}" + state: "{{ state }}" + register: ssh_key + +# - debug: +# var: ssh_key + +- name: store private key locally + copy: + dest: "{{ ssh_key_name }}.pem" + content: "{{ ssh_key.key.private_key }}" + when: state == 'present' and ssh_key.key.private_key != none \ No newline at end of file diff --git a/roles/create_infrastructure/tasks/lb_members.yml b/roles/create_infrastructure/tasks/lb_members.yml new file mode 100644 index 0000000000000000000000000000000000000000..35b6d60559c919c6c5eac74a08a5e72db9c051c8 --- /dev/null +++ b/roles/create_infrastructure/tasks/lb_members.yml @@ -0,0 +1,79 @@ +- name: wait for loadbalancer + async_status: + jid: "{{ _create_loadbalancer.ansible_job_id }}" + register: loadbalancer_result + until: loadbalancer_result.finished + delay: 1 + retries: 600 + +- name: store loadbalancer ip + set_fact: + new_loadbalancer_ip: "{{ loadbalancer_result.loadbalancer.public_vip_address | default(loadbalancer_ip) }}" + +- name: add listener + openstack.cloud.lb_listener: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ loadbalancer_name }}-listener-{{ item }}" + loadbalancer: "{{ loadbalancer_name }}" + protocol: TCP + protocol_port: "{{ item }}" + state: "{{ state }}" + loop: + - 6443 + - 9345 + - 80 + - 443 + +- name: add pool + openstack.cloud.lb_pool: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ loadbalancer_name }}-pool-{{ item }}" + listener: "{{ loadbalancer_name }}-listener-{{ item }}" + protocol: TCP + lb_algorithm: ROUND_ROBIN + state: "{{ state }}" + loop: + - 6443 + - 9345 + - 80 + - 443 + +- name: add health monitor + openstack.cloud.lb_health_monitor: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + pool: "{{ loadbalancer_name }}-pool-{{ item }}" + name: "{{ loadbalancer_name }}-pool-{{ item }}-healthmonitor" + delay: '20' + max_retries: '5' + max_retries_down: '5' + resp_timeout: '10' + type: TCP + loop: + - 6443 + - 9345 + - 80 + - 443 + + +- set_fact: + pool_members: + - { pool: 6443, port: 6443, vm: "{{ servers + [ master ] }}" } + - { pool: 9345, port: 9345, vm: "{{ servers + [ master ] }}" } + - { pool: 80, port: 32080, vm: "{{ servers + [ master ] + agents }}" } + - { pool: 443, port: 32443, vm: "{{ servers + [ master ] + agents }}" } + +- name: add members + openstack.cloud.lb_member: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ loadbalancer_name }}-pool-{{ item.0.pool }}-{{ item.1.name }}" + pool: "{{ loadbalancer_name }}-pool-{{ item.0.pool }}" + address: "{{ item.1.private_v4 }}" + protocol_port: "{{ item.0.port }}" + with_subelements: + - "{{ pool_members }}" + - vm + diff --git a/roles/create_infrastructure/tasks/loadbalancer.yml b/roles/create_infrastructure/tasks/loadbalancer.yml new file mode 100644 index 0000000000000000000000000000000000000000..f1a17f10a6939349e768bd7ae1f3a139f52f1a8c --- /dev/null +++ b/roles/create_infrastructure/tasks/loadbalancer.yml @@ -0,0 +1,13 @@ +- name: loadbalancer + openstack.cloud.loadbalancer: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ loadbalancer_name }}" + vip_subnet: "{{ subnet_name }}" + auto_public_ip: yes + public_network: public + state: "{{ state }}" + register: _create_loadbalancer + async: 600 + poll: 0 + diff --git a/roles/create_infrastructure/tasks/main.yml b/roles/create_infrastructure/tasks/main.yml new file mode 100644 index 0000000000000000000000000000000000000000..a8cdc75fd06224466376aaf702cc2ba5e4774b41 --- /dev/null +++ b/roles/create_infrastructure/tasks/main.yml @@ -0,0 +1,82 @@ +- block: + - name: network + include_tasks: network.yml + args: + apply: + tags: + - network + tags: + - network + + - name: loadbalancer + include_tasks: loadbalancer.yml + args: + apply: + tags: + - loadbalancer + tags: + - loadbalancer + + - name: security groups + include_tasks: security_groups.yml + args: + apply: + tags: + - security_groups + tags: + - security_groups + + - name: keypair + include_tasks: keypair.yml + args: + apply: + tags: + - keypair + tags: + - keypair + + - name: vm + include_tasks: vm.yml + args: + apply: + tags: + - vm + tags: + - vm + + - name: lb elements + include_tasks: lb_members.yml + args: + apply: + tags: + - loadbalancer + tags: + - loadbalancer + + - name: inventory + include_tasks: inventory.yml + + when: state == 'present' + +- block: + - name: loadbalancer + include_tasks: loadbalancer.yml + + - name: vm + include_tasks: vm.yml + + - name: security groups + include_tasks: security_groups.yml + + - name: network + include_tasks: network.yml + + - name: delete inventory + file: + path: "{{ item }}" + state: absent + loop: + - group_vars + - hosts + + when: state == 'absent' \ No newline at end of file diff --git a/roles/create_infrastructure/tasks/network.yml b/roles/create_infrastructure/tasks/network.yml new file mode 100644 index 0000000000000000000000000000000000000000..3ce82b7278e2935484d856162b6a5aae54091246 --- /dev/null +++ b/roles/create_infrastructure/tasks/network.yml @@ -0,0 +1,38 @@ +- name: delete router + openstack.cloud.router: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ router_name }}" + state: "{{ state }}" + when: state == 'absent' + +- name: network + openstack.cloud.network: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ network_name }}" + state: "{{ state }}" + +- name: subnet + openstack.cloud.subnet: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + network_name: "{{ network_name }}" + name: "{{ subnet_name }}" + cidr: "{{ cidr }}" + state: "{{ state }}" + register: subnet_result + +- set_fact: + new_subnet_id: "{{ subnet_result.subnet.id }}" + when: state == 'present' + +- name: router + openstack.cloud.router: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ router_name }}" + network: public + interfaces: + - "{{ network_name }}" + when: state == 'present' diff --git a/roles/create_infrastructure/tasks/security_groups.yml b/roles/create_infrastructure/tasks/security_groups.yml new file mode 100644 index 0000000000000000000000000000000000000000..de43f1f9cf464edd981c94f01ca4f9b13e956851 --- /dev/null +++ b/roles/create_infrastructure/tasks/security_groups.yml @@ -0,0 +1,29 @@ +- name: security group + openstack.cloud.security_group: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "{{ security_group }}" + state: "{{ state }}" + +- name: rules + openstack.cloud.security_group_rule: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + security_group: "{{ security_group }}" + protocol: "{{ item.protocol }}" + port_range_min: "{{ item.min }}" + port_range_max: "{{ item.max }}" + remote_ip_prefix: "{{ item.cidr }}" + loop: + - { protocol: TCP, min: 2379, max: 2380, cidr: "{{ cidr }}" } # etcd + - { protocol: TCP, min: 6443, max: 6443, cidr: "{{ cidr }}" } # Kubernetes API + - { protocol: TCP, min: 9345, max: 9345, cidr: "{{ cidr }}" } # RKE2 API + - { protocol: TCP, min: 10250, max: 10250, cidr: "{{ cidr }}" } # kubelet metrics + - { protocol: TCP, min: 30000, max: 32767, cidr: "{{ cidr }}" } # NodePort port range + - { protocol: UDP, min: 8472, max: 8472, cidr: "{{ cidr }}" } # flannel + - { protocol: TCP, min: 4240, max: 4240, cidr: "{{ cidr }}" } # cni-health + - { protocol: TCP, min: 22, max: 22, cidr: "0.0.0.0/0" } # SSH + ignore_errors: yes + async: 60 + poll: 0 + when: state == 'present' \ No newline at end of file diff --git a/roles/create_infrastructure/tasks/vm.yml b/roles/create_infrastructure/tasks/vm.yml new file mode 100644 index 0000000000000000000000000000000000000000..5d957f1105aea43b2299c3ca1fb90d84765522f3 --- /dev/null +++ b/roles/create_infrastructure/tasks/vm.yml @@ -0,0 +1,110 @@ +- name: master VM + openstack.cloud.server: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "rke2-{{ cluster_name }}-master" + boot_from_volume: yes + terminate_volume: yes + volume_size: "{{ server_volume_size }}" + network: "{{ network_name }}" + key_name: "{{ ssh_key_name }}" + flavor: "{{ server_flavor }}" + image: "{{ image }}" + security_groups: + - "{{ security_group }}" + delete_fip: yes + floating_ip_pools: + - public + state: "{{ state }}" + userdata: |- + #cloud-config + package_update: true + package_upgrade: true + register: _create_master + async: 600 + poll: 0 + +- name: server VMs + openstack.cloud.server: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "rke2-{{ cluster_name }}-server-{{ item }}" + boot_from_volume: yes + terminate_volume: yes + volume_size: "{{ server_volume_size }}" + network: "{{ network_name }}" + key_name: "{{ ssh_key_name }}" + flavor: "{{ server_flavor }}" + image: "{{ image }}" + security_groups: + - "{{ security_group }}" + auto_ip: no + state: "{{ state }}" + userdata: |- + #cloud-config + package_update: true + package_upgrade: true + loop: "{{ range(1, server_count, 1) | list }}" + register: _create_servers + async: 600 + poll: 0 + +- name: agent VM + openstack.cloud.server: + auth: "{{ openstack_auth }}" + auth_type: "{{ openstack_auth_type }}" + name: "rke2-{{ cluster_name }}-agent-{{ item }}" + boot_from_volume: yes + terminate_volume: yes + volume_size: "{{ agent_volume_size }}" + network: "{{ network_name }}" + key_name: "{{ ssh_key_name }}" + flavor: "{{ agent_flavor }}" + image: "{{ image }}" + security_groups: + - "{{ security_group }}" + auto_ip: no + state: "{{ state }}" + userdata: |- + #cloud-config + package_update: true + package_upgrade: true + loop: "{{ range(0, agent_count, 1) | list }}" + register: _create_agents + async: 600 + poll: 0 + +- name: wait for master vm + async_status: + jid: "{{ _create_master.ansible_job_id }}" + register: master_result + until: master_result.finished + delay: 1 + retries: 600 + +- block: + - name: wait for server vms + async_status: + jid: "{{ item.ansible_job_id }}" + register: servers_result + until: servers_result.finished + delay: 1 + retries: 600 + loop: "{{ _create_servers.results }}" + +- block: + - name: wait for agents vm + async_status: + jid: "{{ item.ansible_job_id }}" + register: agents_result + until: agents_result.finished + delay: 1 + retries: 600 + loop: "{{ _create_agents.results }}" + +- set_fact: + master: "{{ master_result | community.general.json_query('server.{name: name, private_v4: private_v4, public_v4: public_v4}') }}" + servers: "{{ servers_result.results | community.general.json_query('[].server.{name: name, private_v4: private_v4}') }}" + agents: "{{ agents_result.results | community.general.json_query('[].server.{name: name, private_v4: private_v4}') }}" + new_master_floating_ip: "{{ master_result.server.public_v4 }}" + when: state == 'present' \ No newline at end of file diff --git a/roles/create_infrastructure/templates/agents.yml.j2 b/roles/create_infrastructure/templates/agents.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..ab719f07ce7a59f19d307272bc38bd5ccf3d71c2 --- /dev/null +++ b/roles/create_infrastructure/templates/agents.yml.j2 @@ -0,0 +1,6 @@ +{% raw %} +ansible_user: "ubuntu" +ansible_ssh_private_key_file: "{{ ssh_key_file }}" +ansible_ssh_common_args: "-o ProxyCommand='ssh -q ubuntu@{{ master_floating_ip }} -o StrictHostKeyChecking=no -i {{ ssh_key_file }} -W %h:%p' -o StrictHostKeyChecking=no" +node_type: agent +{% endraw %} \ No newline at end of file diff --git a/roles/create_infrastructure/templates/all.yml.j2 b/roles/create_infrastructure/templates/all.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..f64dd209613ca942fae0b1863dcd8452fdbeaedd --- /dev/null +++ b/roles/create_infrastructure/templates/all.yml.j2 @@ -0,0 +1,5 @@ +loadbalancer_ip: {{ new_loadbalancer_ip }} + +master_floating_ip: {{ new_master_floating_ip }} + +subnet_id: {{ new_subnet_id }} \ No newline at end of file diff --git a/roles/create_infrastructure/templates/hosts.j2 b/roles/create_infrastructure/templates/hosts.j2 new file mode 100644 index 0000000000000000000000000000000000000000..a68715697be9a9af5e8a276883997c1f55e963e7 --- /dev/null +++ b/roles/create_infrastructure/templates/hosts.j2 @@ -0,0 +1,16 @@ +[master] +{{ master.name }} ansible_host="{{ master.public_v4 }}" + +[servers] +{% if servers is defined and servers | length > 0 %} +{% for server in servers %} +{{ server.name }} ansible_host="{{ server.private_v4 }}" +{% endfor %} +{% endif %} + +[agents] +{% if agents is defined and agents | length > 0 %} +{% for agent in agents %} +{{ agent.name }} ansible_host="{{ agent.private_v4 }}" +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/roles/create_infrastructure/templates/master.yml.j2 b/roles/create_infrastructure/templates/master.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..39c75b6d782e08911cdff64b46a2f2cf49b8c414 --- /dev/null +++ b/roles/create_infrastructure/templates/master.yml.j2 @@ -0,0 +1,6 @@ +{% raw %} +ansible_user: "ubuntu" +ansible_ssh_private_key_file: "{{ ssh_key_file }}" +ansible_ssh_common_args: "-o StrictHostKeyChecking=no" +node_type: server +{% endraw %} \ No newline at end of file diff --git a/roles/create_infrastructure/templates/servers.yml.j2 b/roles/create_infrastructure/templates/servers.yml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..562da525422fe35ca273ebb83cc4dff70d6fbbbb --- /dev/null +++ b/roles/create_infrastructure/templates/servers.yml.j2 @@ -0,0 +1,6 @@ +{% raw %} +ansible_user: "ubuntu" +ansible_ssh_private_key_file: "{{ ssh_key_file }}" +ansible_ssh_common_args: "-o ProxyCommand='ssh -q ubuntu@{{ master_floating_ip }} -o StrictHostKeyChecking=no -i {{ ssh_key_file }} -W %h:%p' -o StrictHostKeyChecking=no" +node_type: server +{% endraw %} \ No newline at end of file diff --git a/roles/infrastructure/default/main.yml b/roles/infrastructure/default/main.yml deleted file mode 100644 index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0000000000000000000000000000000000000000 diff --git a/roles/infrastructure/tasks/keypair.yml b/roles/infrastructure/tasks/keypair.yml deleted file mode 100644 index 1093d3e1e62bbd684f59db50443c54e056b8dbef..0000000000000000000000000000000000000000 --- a/roles/infrastructure/tasks/keypair.yml +++ /dev/null @@ -1,6 +0,0 @@ -- name: create ssh key pair - openstack.cloud.keypair: - cloud: "{{ cloud }}" - state: present - name: "{{ ssh_key_name }}" - register: ssh_key \ No newline at end of file diff --git a/roles/infrastructure/tasks/loadbalancer.yml b/roles/infrastructure/tasks/loadbalancer.yml deleted file mode 100644 index 807852438c4041891722fc7e2af02357c3537ed0..0000000000000000000000000000000000000000 --- a/roles/infrastructure/tasks/loadbalancer.yml +++ /dev/null @@ -1,119 +0,0 @@ -- name: create loadbalancer - openstack.cloud.loadbalancer: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}" - vip_subnet: "{{ network_name }}" - -- name: add listener 6443 - openstack.cloud.lb_listener: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}-listener-6443" - loadbalancer: "{{ loadbalancer_name }}" - protocol: TCP - protocol_port: 6443 - -- name: add listener 9345 - openstack.cloud.lb_listener: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}-listener-9345" - loadbalancer: "{{ loadbalancer_name }}" - protocol: TCP - protocol_port: 9345 - -- name: add pool 6443 - openstack.cloud.lb_pool: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}-pool-6443" - loadbalancer: "{{ loadbalancer_name }}" - protocol: TCP - lb_algorithm: ROUND_ROBIN - -- name: add pool 9345 - openstack.cloud.lb_pool: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}-pool-9345" - loadbalancer: "{{ loadbalancer_name }}" - protocol: TCP - lb_algorithm: ROUND_ROBIN - -- name: add members 6443 - openstack.cloud.lb_member: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}-pool-6443-{{ inventory_hostname }}" - pool: "{{ loadbalancer_name }}-pool-6443" - address: "{{ ansible primary ip }}" - protocol_port: 6443 - loop: - -- name: add members 9345 - openstack.cloud.lb_member: - cloud: "{{ cloud }}" - name: "{{ loadbalancer_name }}-pool-9345-{{ inventory_hostname }}" - pool: "{{ loadbalancer_name }}-pool-9345" - address: "{{ ansible primary ip }}" - protocol_port: 9345 - loop: - - -- name: add health monitor 6443 - openstack.cloud.lb_health_monitor: - cloud: "{{ cloud }}" - expected_codes: '200' - max_retries_down: '4' - pool: "{{ loadbalancer_name }}-pool-6443" - name: "{{ loadbalancer_name }}-pool-6443-healthmonitor" - delay: '20' - max_retries: '5' - resp_timeout: '10' - type: TCP - -- name: add health monitor 9345 - openstack.cloud.lb_health_monitor: - cloud: "{{ cloud }}" - expected_codes: '200' - max_retries_down: '4' - pool: "{{ loadbalancer_name }}-pool-9345" - name: "{{ loadbalancer_name }}-pool-9345-healthmonitor" - delay: '20' - max_retries: '5' - resp_timeout: '10' - type: TCP - - - -# resource "openstack_lb_member_v2" "members_k8s_mgmt_6443" { -# count = var.rancher.count_mgmt_nodes -# address = openstack_compute_instance_v2.rancher_mgmt[count.index].network[0].fixed_ip_v4 -# pool_id = openstack_lb_pool_v2.k8s_pool_6443.id -# protocol_port = 6443 -# subnet_id = openstack_networking_subnet_v2.k8s_mgmt_subnet.id -# depends_on = [ -# openstack_networking_subnet_v2.k8s_mgmt_subnet, -# openstack_lb_pool_v2.k8s_pool_6443, -# openstack_compute_instance_v2.rancher_mgmt -# ] -# } - -# resource "openstack_lb_member_v2" "members_k8s_mgmt_9345" { -# count = var.rancher.count_mgmt_nodes -# address = openstack_compute_instance_v2.rancher_mgmt[count.index].network[0].fixed_ip_v4 -# pool_id = openstack_lb_pool_v2.k8s_pool_9345.id -# protocol_port = 9345 -# subnet_id = openstack_networking_subnet_v2.k8s_mgmt_subnet.id -# depends_on = [ openstack_networking_subnet_v2.k8s_mgmt_subnet, openstack_lb_pool_v2.k8s_pool_9345, openstack_compute_instance_v2.rancher_mgmt] -# } - -# resource "openstack_lb_monitor_v2" "monitor_k8s_mgmt_6443" { -# pool_id = openstack_lb_pool_v2.k8s_pool_6443.id -# type = "TCP" -# delay = 20 -# timeout = 10 -# max_retries = 5 -# } -# resource "openstack_lb_monitor_v2" "monitor_k8s_mgmt_9345" { -# pool_id = openstack_lb_pool_v2.k8s_pool_9345.id -# type = "TCP" -# delay = 20 -# timeout = 10 -# max_retries = 5 -# } \ No newline at end of file diff --git a/roles/infrastructure/tasks/network.yml b/roles/infrastructure/tasks/network.yml deleted file mode 100644 index 039f427d36ddea71cd0197acf2a54005e608035b..0000000000000000000000000000000000000000 --- a/roles/infrastructure/tasks/network.yml +++ /dev/null @@ -1,20 +0,0 @@ -- name: create network - openstack.cloud.network: - cloud: "{{ cloud }}" - name: "{{ network_name }}" - -- name: create subnet - openstack.cloud.subnet: - cloud: "{{ cloud }}" - network_name: "{{ network_name }}" - name: "{{ subnet_name }}" - cidr: "{{ cidr }}" - dns_nameservers: "{{ nameserver }}" - -- name: create router - openstack.cloud.router: - cloud: "{{ cloud }}" - name: "{{ router_name }}" - network: public - interfaces: - - "{{ network_name }}" \ No newline at end of file diff --git a/roles/infrastructure/tasks/security_groups.yml b/roles/infrastructure/tasks/security_groups.yml deleted file mode 100644 index 6f5bae3a35849d0b7affe5f2807ea1c3546f53fb..0000000000000000000000000000000000000000 --- a/roles/infrastructure/tasks/security_groups.yml +++ /dev/null @@ -1,268 +0,0 @@ -# # Creating Openstack security groups -# resource "openstack_networking_secgroup_v2" "k8s_secgroup" { -# name = "sg-os-k8s-sbx" -# description = "k8s security group" -# } - -# # Creating Openstack security group rule for etcd 2379-2380 -# resource "openstack_networking_secgroup_rule_v2" "etcd" { -# description = "etcd" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 2379 -# port_range_max = 2380 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for k8s-api 6443 -# resource "openstack_networking_secgroup_rule_v2" "k8s-api-mgmt" { -# description = "k8s-api" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 6443 -# port_range_max = 6443 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } -# resource "openstack_networking_secgroup_rule_v2" "k8s-api-agent" { -# description = "k8s-api" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 6443 -# port_range_max = 6443 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for rke2-api 9345 -# resource "openstack_networking_secgroup_rule_v2" "rke2-api-mgmt" { -# description = "rke2-api" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 9345 -# port_range_max = 9345 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } -# resource "openstack_networking_secgroup_rule_v2" "rke2-api-agent" { -# description = "rke2-api" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 9345 -# port_range_max = 9345 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - - - -# ######## CNI - - -# # Creating Openstack security group rule for vxlan -# resource "openstack_networking_secgroup_rule_v2" "vxlan-mgmt" { -# description = "vxlan-mgmt" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "udp" -# port_range_min = 8472 -# port_range_max = 8472 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for vxlan -# resource "openstack_networking_secgroup_rule_v2" "vxlan-agent" { -# description = "vxlan-agent" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "udp" -# port_range_min = 8472 -# port_range_max = 8472 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - - -# # Creating Openstack security group rule for vxlan -# resource "openstack_networking_secgroup_rule_v2" "cni-health-tcp-mgmt" { -# description = "cni-health-tcp-mgmt" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 4240 -# port_range_max = 4240 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for vxlan -# resource "openstack_networking_secgroup_rule_v2" "cni-health-tcp-agent" { -# description = "cni-health-tcp-agent" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 4240 -# port_range_max = 4240 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for vxlan -# resource "openstack_networking_secgroup_rule_v2" "cni-health-icmp-mgmt" { -# description = "cni-health-icmp-mgmt" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "icmp" -# port_range_min = 8 -# port_range_max = 8 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for vxlan -# resource "openstack_networking_secgroup_rule_v2" "cni-health-icmp-agent" { -# description = "cni-health-icmp-agent" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "icmp" -# port_range_min = 8 -# port_range_max = 8 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - - - -# ########### - - - - -# # Creating Openstack security group rule for kubelet metrics -# resource "openstack_networking_secgroup_rule_v2" "kubelet-metrics-api-mgmt" { -# description = "metrics-api-mgmt" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 10250 -# port_range_max = 10250 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for kubelet metrics -# resource "openstack_networking_secgroup_rule_v2" "kubelet-metrics-api-agent" { -# description = "metrics-api-agent" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 10250 -# port_range_max = 10250 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } -# # Creating Openstack security group rule for NodePort port range -# resource "openstack_networking_secgroup_rule_v2" "nodeport-range-mgmt" { -# description = "nodeports-mgmt" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 30000 -# port_range_max = 32767 -# remote_ip_prefix = var.network.k8s_mgmt_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for NodePort port range -# resource "openstack_networking_secgroup_rule_v2" "nodeport-range-agent" { -# description = "nodeports-agent" -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 30000 -# port_range_max = 32767 -# remote_ip_prefix = var.network.k8s_agent_subnet_cidr -# #remote_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for https 8443 -# resource "openstack_networking_secgroup_rule_v2" "rancher-ui" { -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 8443 -# port_range_max = 8443 -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for https 443 -# resource "openstack_networking_secgroup_rule_v2" "http" { -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 80 -# port_range_max = 80 -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for https 443 -# resource "openstack_networking_secgroup_rule_v2" "https" { -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 443 -# port_range_max = 443 -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } - -# # Creating Openstack security group rule for ssh 22 -# resource "openstack_networking_secgroup_rule_v2" "ssh" { -# direction = "ingress" -# ethertype = "IPv4" -# protocol = "tcp" -# port_range_min = 22 -# port_range_max = 22 -# remote_ip_prefix = var.network.ssh_access_subnet_cidr -# security_group_id = openstack_networking_secgroup_v2.k8s_secgroup.id -# depends_on = [openstack_networking_secgroup_v2.k8s_secgroup] -# } \ No newline at end of file diff --git a/roles/infrastructure/tasks/vm.yml b/roles/infrastructure/tasks/vm.yml deleted file mode 100644 index 0583776951e709588bd1624bd47293a0b33cf651..0000000000000000000000000000000000000000 --- a/roles/infrastructure/tasks/vm.yml +++ /dev/null @@ -1,43 +0,0 @@ -- name: create server vms - openstack.cloud.server: - cloud: "{{ cloud }}" - name: "{{ name }}-server-{{ item }}" - boot_from_volume: yes - terminate_volume: yes - volume_size: "{{ server_volume_size }}" - network: "{{ network_name }}" - key_name: "{{ ssh_key_name }}" - flavor: "{{ server_flavor }}" - image: "{{ image }}" - loop: "{{ range(0, server_count, 1) | list }}" - register: servers - -- name: add servers to inventory - add_host: - ansible_host: "{{ item.server.private_v4 }}" - ansible_user: ubuntu - group: server - name: "{{ item.server.name }}" - loop: "{{ servers }}" - -- name: create agent vms - openstack.cloud.server: - cloud: "{{ cloud }}" - name: "{{ name }}-agent-{{ item }}" - boot_from_volume: yes - terminate_volume: yes - volume_size: "{{ agent_volume_size }}" - network: "{{ network_name }}" - key_name: "{{ ssh_key_name }}" - flavor: "{{ agent_flavor }}" - image: "{{ image }}" - loop: "{{ range(0, agent_count, 1) | list }}" - register: agents - -- name: add agents to inventory - add_host: - ansible_host: "{{ item.server.private_v4 }}" - ansible_user: ubuntu - group: agent - name: "{{ item.server.name }}" - loop: "{{ agents }}" diff --git a/roles/rke2/defaults/main.yml b/roles/rke2/defaults/main.yml index 44ba27cd014f57d7adc2d11e6f5bbc8fae438658..9923fff709e96664d7fe1a5f8442ec6fdc29bcae 100644 --- a/roles/rke2/defaults/main.yml +++ b/roles/rke2/defaults/main.yml @@ -1,10 +1,14 @@ -server: https://mgmtlb.k8s.example:9345 -master: mgmt-1 +domain: +server: "https://{{ domain }}:9345" + tls_san: - - "mgmtlb.k8s.example" - - "another.k8s.example" -server_node_taints: - - "CriticalAddonsOnly=true:NoExecute" -grafana_password: "PASSWORD" -rancher_ui_dns: "ui.k8s.example" -letsEncrypt_admin_mail: "test@test.com" \ No newline at end of file + - "{{ domain }}" + +node_taints: [] +node_labels: [] + +grafana_password: +rancher_ui_dns: +letsEncrypt_admin_mail: + +rke2_channel: stable \ No newline at end of file diff --git a/roles/rke2/handlers/main.yml b/roles/rke2/handlers/main.yml index d3f55bb45f7335fc2dd1d264bc33081b18918340..b5a2ba62aa46a0d50acc9e5c426fa6225d1a94d6 100644 --- a/roles/rke2/handlers/main.yml +++ b/roles/rke2/handlers/main.yml @@ -1,18 +1,6 @@ ---- - -# restart rke2 configuration -- name: restart_rke2 +- name: restart rke2 service: - name: rke2-server - enabled: yes - masked: no - state: restarted - daemon_reload: yes - -# restart rke2 configuration -- name: restart_rke2_agent - service: - name: rke2-agent + name: "rke2-{{ node_type }}" enabled: yes masked: no state: restarted diff --git a/roles/rke2/tasks/add_cloud_config.yml b/roles/rke2/tasks/add_cloud_config.yml deleted file mode 100644 index 27e08f816152515974b61eb84bcfb1ff78146631..0000000000000000000000000000000000000000 --- a/roles/rke2/tasks/add_cloud_config.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- - -- name: render cloud-config template to variable - set_fact: - cloud_config: "{{ lookup('template', 'cloud-config.j2') }}" - -- name: add cloud config - template: - src: deploy-openstack-cloud-config-assets.j2 - dest: /var/lib/rancher/rke2/server/manifests/deploy-openstack-cloud-config-assets.yaml - - -- name: download openstack manifests - get_url: - url: "{{ item.url }}" - dest: '/var/lib/rancher/rke2/server/manifests/{{ item.dest }}' - loop: - - { dest: 'cloud-controller-manager-roles.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-roles.yaml'} - - { dest: 'cloud-controller-manager-role-bindings.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml'} - - { dest: 'openstack-cloud-controller-manager-ds.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml'} - - { dest: 'cinder-csi-controllerplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml'} - - { dest: 'cinder-csi-controllerplugin.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml'} - - { dest: 'cinder-csi-nodeplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml'} - - { dest: 'cinder-csi-nodeplugin.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml'} - - { dest: 'csi-cinder-driver.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/csi-cinder-driver.yaml'} - - { dest: 'snapshot.storage.k8s.io_volumesnapshotclasses.yaml', url: 'https://github.com/kubernetes-csi/external-snapshotter/raw/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml'} - - { dest: 'snapshot.storage.k8s.io_volumesnapshotcontents.yaml', url: 'https://github.com/kubernetes-csi/external-snapshotter/raw/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml'} - - { dest: 'snapshot.storage.k8s.io_volumesnapshots.yaml', url: 'https://github.com/kubernetes-csi/external-snapshotter/raw/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml'} - - { dest: 'rbac-snapshot-controller.yaml', url: 'https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml'} - - { dest: 'setup-snapshot-controller.yaml', url: 'https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml'} - #- { dest: 'manila-csi-controllerplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/manila-csi-plugin/csi-controllerplugin-rbac.yaml'} - #- { dest: 'manila-csi-nodeplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/manila-csi-plugin/csi-nodeplugin-rbac.yaml'} - #- { dest: 'manila-csidriver.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/manila-csi-plugin/csidriver.yaml'} - - -- name: Replace Master Selector openstack-cloud-controller-manager-ds.yaml - ansible.builtin.replace: - path: /var/lib/rancher/rke2/server/manifests/openstack-cloud-controller-manager-ds.yaml - regexp: 'node-role\.kubernetes\.io\/master: ""' - replace: 'node-role.kubernetes.io/master: "true"' \ No newline at end of file diff --git a/roles/rke2/tasks/config_rke2.yml b/roles/rke2/tasks/config_rke2.yml new file mode 100644 index 0000000000000000000000000000000000000000..dc936f2b5b65dd4a54147e0ad70a1130142228ec --- /dev/null +++ b/roles/rke2/tasks/config_rke2.yml @@ -0,0 +1,33 @@ +- name: create token + delegate_to: localhost + run_once: yes + set_fact: + token: "{{ lookup('community.general.random_string', length=129, special=False) }}" + when: token is not defined + +- name: store token + delegate_to: localhost + become: no + run_once: yes + copy: + dest: group_vars/all/token.yml + content: |- + token: {{ token }} + +- name: read token + include_vars: group_vars/all/token.yml + +- name: rke2 config + template: + src: config.yaml.j2 + dest: /etc/rancher/rke2/config.yaml + notify: + - restart rke2 + +- name: start RKE2 + service: + name: "rke2-{{ node_type }}" + enabled: yes + masked: no + state: started + daemon_reload: yes \ No newline at end of file diff --git a/roles/rke2/tasks/configure_rke2_master.yml b/roles/rke2/tasks/configure_rke2_master.yml deleted file mode 100644 index a81145af33a5a50ac5a614bc11c3d15f4a7ad0a9..0000000000000000000000000000000000000000 --- a/roles/rke2/tasks/configure_rke2_master.yml +++ /dev/null @@ -1,49 +0,0 @@ ---- - -# config -- name: rke2 config - template: - src: rke2_conf.j2 - dest: /etc/rancher/rke2/config.yaml - notify: - - restart_rke2 - -# helm charts -- name: ensure all helm template files are rendered - template: - src: 'helm/{{ item.template }}.j2' - dest: '/var/lib/rancher/rke2/server/manifests/{{ item.template }}.yaml' - loop: - #- { template: 'conf-rke2-canal'} - - { template: 'conf-nginx-ingress'} - - { template: 'deploy-grafana'} - - { template: 'deploy-cert-manager'} - - { template: 'deploy-rancher-ui'} - - { template: 'deploy-rke2-cilium'} - - { template: 'rke2-coredns-config'} - notify: - - restart_rke2 - - -# start rke2 server -- name: start_rke2 - service: - name: rke2-server - enabled: yes - masked: no - state: started - -# slurp token -- name: Wait until the token is present before continuing - wait_for: - path: /var/lib/rancher/rke2/server/node-token - -- name: Load token - slurp: - src: "/var/lib/rancher/rke2/server/node-token" - register: slurped_token - -- name: Decode token and store as fact at dummy master_host with host variable - add_host: - name: "MASTER_HOST" - token: "{{ slurped_token.content | b64decode | trim }}" \ No newline at end of file diff --git a/roles/rke2/tasks/helm.yml b/roles/rke2/tasks/helm.yml new file mode 100644 index 0000000000000000000000000000000000000000..8d047a88cb9274342d27c87c9287b9adfb46b859 --- /dev/null +++ b/roles/rke2/tasks/helm.yml @@ -0,0 +1,19 @@ +- name: copy helm template files + template: + src: 'helm/{{ item.template }}.j2' + dest: '/var/lib/rancher/rke2/server/manifests/{{ item.template }}.yaml' + loop: + - { template: 'deploy-openstack-ccm'} + - { template: 'deploy-openstack-cinder'} + # - { template: 'deploy-openstack-manila'} + - { template: 'deploy-cephfs'} + # - { template: 'deploy-nfs'} + # - { template: 'deploy-grafana'} + # - { template: 'deploy-cert-manager'} + # - { template: 'deploy-rancher-ui'} + # - { template: 'deploy-rke2-cilium'} + - { template: 'config-rke2-canal'} + - { template: 'config-nginx-ingress'} + # - { template: 'config-rke2-coredns'} + notify: + - restart rke2 \ No newline at end of file diff --git a/roles/rke2/tasks/install_rke2.yml b/roles/rke2/tasks/install_rke2.yml new file mode 100644 index 0000000000000000000000000000000000000000..5cab01d94a1d4423d490757dbd448d911d66a215 --- /dev/null +++ b/roles/rke2/tasks/install_rke2.yml @@ -0,0 +1,19 @@ +- name: download RKE2 install script + get_url: + url: https://get.rke2.io + dest: /tmp/rke2.sh + mode: '0755' + when: not rke2_installed.stat.exists + +- name: Install RKE2 + command: "/tmp/rke2.sh" + args: + creates: /usr/local/bin/rke2 + environment: + INSTALL_RKE2_CHANNEL: "{{ rke2_channel }}" + INSTALL_RKE2_TYPE: "{{ node_type }}" + +- name: remove RKE2 install script + file: + path: /tmp/rke2.sh + state: absent diff --git a/roles/rke2/tasks/kubeconfig.yml b/roles/rke2/tasks/kubeconfig.yml new file mode 100644 index 0000000000000000000000000000000000000000..75bed5da27d09adbc92e82ad72f28c93a6057000 --- /dev/null +++ b/roles/rke2/tasks/kubeconfig.yml @@ -0,0 +1,17 @@ +- name: wait for kubeconfig + wait_for: + path: /etc/rancher/rke2/rke2.yaml + +- name: fetch kubeconfig from master + ansible.builtin.fetch: + src: /etc/rancher/rke2/rke2.yaml + dest: kubeconfig.yaml + flat: yes + +- name: replace endpoint in kubeconfig + delegate_to: localhost + become: no + ansible.builtin.replace: + path: kubeconfig.yaml + regexp: '^(\s+server: ).*' + replace: '\1https://{{ domain }}:6443' diff --git a/roles/rke2/tasks/main.yml b/roles/rke2/tasks/main.yml index 2ff0b10f3781a72ce6789a18ef42fff3874b3d9d..d65d0db0143fba04652bc4d9edea5435de2cbb63 100644 --- a/roles/rke2/tasks/main.yml +++ b/roles/rke2/tasks/main.yml @@ -1,47 +1,21 @@ ---- -- include_tasks: add_basics.yml +- stat: + path: /usr/local/bin/rke2 + register: rke2_installed +- block: + - include_tasks: setup_host.yml + - include_tasks: install_rke2.yml + when: not rke2_installed.stat.exists and state != 'absent' -- name: configure, and start RKE2 - when: "'kubemgmt' in group_names" - block: +- include_tasks: helm.yml + when: "state != 'absent' and 'master' in group_names" - - include_tasks: install_rke2_server.yml +- include_tasks: config_rke2.yml + when: state != 'absent' - - name: configure, and start RKE2 Master - when: inventory_hostname == master - block: +- include_tasks: kubeconfig.yml + when: state != 'absent' and 'master' in group_names - - include_tasks: configure_rke2_master.yml - - include_tasks: add_cloud_config.yml - - - -## just when token is ready -- name: configure, and start RKE2 Slave - when: inventory_hostname != master - block: - - - name: rke2 config - when: "'kubemgmt' in group_names" - throttle: 1 - template: - src: rke2_conf.j2 - dest: /etc/rancher/rke2/config.yaml - notify: - - restart_rke2 - - - - name: configure, and start RKE2 AGENT - when: "'kubeagents' in group_names" - block: - - - include_tasks: install_rke2_agent.yml - - - name: rke2 config - throttle: 1 - template: - src: rke2_conf.j2 - dest: /etc/rancher/rke2/config.yaml - notify: - - restart_rke2_agent +- name: uninstall rke2 + command: rke2-uninstall.sh + when: rke2_installed.stat.exists and state == 'absent' \ No newline at end of file diff --git a/roles/infrastructure/tasks/setup_vm.yml b/roles/rke2/tasks/setup_host.yml similarity index 54% rename from roles/infrastructure/tasks/setup_vm.yml rename to roles/rke2/tasks/setup_host.yml index 80aae142200292b2907f87daca1f143a3fd1a43e..221922ba5aa56fbde9ebd27374b20d9937afbec8 100644 --- a/roles/infrastructure/tasks/setup_vm.yml +++ b/roles/rke2/tasks/setup_host.yml @@ -1,24 +1,21 @@ - - - name: Update the /etc/hosts file with node name - tags: etchostsupdate lineinfile: dest: "/etc/hosts" - regexp: ".*\t{{ hostvars[item]['ansible_hostname']}}\t{{ hostvars[item]['ansible_hostname']}}" - line: "{{ hostvars[item]['ansible_default_ipv4']['address'] }}\t{{ hostvars[item]['ansible_hostname']}}\t{{ hostvars[item]['ansible_hostname']}}" + regexp: ".*\t{{ hostvars[item]['inventory_hostname']}}" + line: "{{ hostvars[item]['ansible_host'] }}\t{{ hostvars[item]['inventory_hostname']}}" state: present backup: yes - register: etchostsupdate - when: ansible_hostname != "{{ item }}" or ansible_hostname == "{{ item }}" - with_items: "{{groups['all']}}" + loop: "{{ groups['all'] }}" - name: add kernel params template: - src: kernel_params_conf.j2 + src: kernel_params.conf.j2 dest: /etc/sysctl.d/rke2_kernel_params.conf + register: kernel_params - name: load kernel params shell: sysctl --system + when: kernel_params.changed - name: add group - etcd group: @@ -40,4 +37,20 @@ file: path: /etc/rancher/rke2 state: directory - recurse: yes \ No newline at end of file + recurse: yes + +- name: update package cache + apt: + update_cache: yes + +- name: upgrade packages + apt: + name: "*" + state: latest + register: update_packages + +- name: reboot vm + throttle: 1 + reboot: + reboot_timeout: 300 + when: update_packages.changed \ No newline at end of file diff --git a/roles/rke2/templates/all.yml b/roles/rke2/templates/all.yml new file mode 100644 index 0000000000000000000000000000000000000000..7288a0061b07a6478a1a9d046782878acdd527d0 --- /dev/null +++ b/roles/rke2/templates/all.yml @@ -0,0 +1,3 @@ +token: {{ token }} + +server: "https://{{ domain }}:9345" diff --git a/roles/rke2/templates/cloud-config.j2 b/roles/rke2/templates/cloud-config.j2 deleted file mode 100644 index ed67ae38f11da5688a90abb544a61d9309fcb1bf..0000000000000000000000000000000000000000 --- a/roles/rke2/templates/cloud-config.j2 +++ /dev/null @@ -1,12 +0,0 @@ -[global] -auth-url={{openstack_auth_url}} -application-credential-id=2c38a96c670a44bc8011720a16d9c9fd -application-credential-secret=2Y6pIxUzLfI8N7I0HlbLbpBKS-QvEgXFUzXs7jxMEgHxLFjbk8hA_AQpE24tkGR3ZYcSieOEM4RwPDcMABcuWw -region=RegionOne - -[LoadBalancer] -use-octavia=true -subnet-id=b88e8883-3a9c-4225-8e21-23f61a2022b9 - -[BlockStorage] -#bs-version=v2 \ No newline at end of file diff --git a/roles/rke2/templates/config.yaml.j2 b/roles/rke2/templates/config.yaml.j2 new file mode 100644 index 0000000000000000000000000000000000000000..2ab69109e52171d05ce9a86edfc59b04ce26fbca --- /dev/null +++ b/roles/rke2/templates/config.yaml.j2 @@ -0,0 +1,37 @@ +node-ip: "{{ ansible_default_ipv4.address }}" +node-name: "{{ ansible_hostname }}" + +token: "{{ token }}" +{% if 'master' not in group_names %} +server: "{{ server }}" +{% endif %} + +profile: "cis-1.5" +resolv-conf: "/run/systemd/resolve/resolv.conf" # systemd-resolved +debug: true +cloud-provider-name: "external" + +{% if node_type == 'server' %} +{# disable: rke2-canal #} +write-kubeconfig-mode: "0644" +{% if tls_san is defined and tls_san | length > 0 %} +tls-san: +{% for san in tls_san %} + - {{ san }} +{% endfor %} +{% endif %} +{% endif %} + +{% if node_taints is defined and node_taints | length > 0 %} +node-taint: +{% for item in node_taints %} + - {{ item }} +{% endfor %} +{% endif %} + +{% if node_labels is defined and node_labels | length > 0 %} +node-label: +{% for item in node_labels %} + - {{ item }} +{% endfor %} +{% endif %} \ No newline at end of file diff --git a/roles/rke2/templates/deploy-openstack-cloud-config-assets.j2 b/roles/rke2/templates/deploy-openstack-cloud-config-assets.j2 deleted file mode 100644 index 36b33fc65f186c4538796b1dc365ca121470b188..0000000000000000000000000000000000000000 --- a/roles/rke2/templates/deploy-openstack-cloud-config-assets.j2 +++ /dev/null @@ -1,16 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: cloud-config - namespace: kube-system -type: Opaque -data: - cloud.conf: {{ cloud_config | string | b64encode }} - ---- - -apiVersion: storage.k8s.io/v1 -kind: StorageClass -metadata: - name: csi-sc-cinderplugin -provisioner: cinder.csi.openstack.org \ No newline at end of file diff --git a/roles/rke2/templates/helm/conf-nginx-ingress.j2 b/roles/rke2/templates/helm/config-nginx-ingress.j2 similarity index 67% rename from roles/rke2/templates/helm/conf-nginx-ingress.j2 rename to roles/rke2/templates/helm/config-nginx-ingress.j2 index 60b8e5d7ad1bead4ad040df0fa1d155dc8be173f..a730ded72442ac1ce3549db7ad985173b0a178c0 100644 --- a/roles/rke2/templates/helm/conf-nginx-ingress.j2 +++ b/roles/rke2/templates/helm/config-nginx-ingress.j2 @@ -1,4 +1,3 @@ ---- apiVersion: helm.cattle.io/v1 kind: HelmChartConfig metadata: @@ -13,11 +12,13 @@ spec: upstream-keepalive-timeout: 3600 valuesContent: |- controller: - kind: DaemonSet - daemonset: - useHostPort: true image: repository: k8s.gcr.io/ingress-nginx/controller - tag: "v0.45.0" + tag: "v0.48.1" config: - use-forwarded-headers: "true" \ No newline at end of file + use-forwarded-headers: "true" + service: + type: NodePort + nodePorts: + http: 32080 + https: 32443 \ No newline at end of file diff --git a/roles/rke2/templates/helm/conf-rke2-canal.j2 b/roles/rke2/templates/helm/config-rke2-canal.j2 similarity index 100% rename from roles/rke2/templates/helm/conf-rke2-canal.j2 rename to roles/rke2/templates/helm/config-rke2-canal.j2 diff --git a/roles/rke2/templates/helm/rke2-coredns-config.j2 b/roles/rke2/templates/helm/config-rke2-coredns.j2 similarity index 100% rename from roles/rke2/templates/helm/rke2-coredns-config.j2 rename to roles/rke2/templates/helm/config-rke2-coredns.j2 diff --git a/roles/rke2/templates/helm/deploy-cephfs.j2 b/roles/rke2/templates/helm/deploy-cephfs.j2 new file mode 100644 index 0000000000000000000000000000000000000000..fe7d1a80230e040aeed3f65988f4928cd02899bb --- /dev/null +++ b/roles/rke2/templates/helm/deploy-cephfs.j2 @@ -0,0 +1,10 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: cephfs-csi + namespace: kube-system +spec: + repo: https://ceph.github.io/csi-charts + chart: ceph-csi-cephfs + version: 3.3.1 +# valuesContent: |- diff --git a/roles/rke2/templates/helm/deploy-cert-manager.j2 b/roles/rke2/templates/helm/deploy-cert-manager.j2 index 6102c59501501e1d86bbbb9a90d54ec16c373b5e..25b4576f77e355487a99603fdb523616f79599b7 100644 --- a/roles/rke2/templates/helm/deploy-cert-manager.j2 +++ b/roles/rke2/templates/helm/deploy-cert-manager.j2 @@ -1,4 +1,3 @@ ---- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: diff --git a/roles/rke2/templates/helm/deploy-grafana.j2 b/roles/rke2/templates/helm/deploy-grafana.j2 index 516a9cf5e5f5caa6e52f9939c0e4a8db3affdf25..97f1d80dc91697bda4fdd05d92ee1980e0bf0280 100644 --- a/roles/rke2/templates/helm/deploy-grafana.j2 +++ b/roles/rke2/templates/helm/deploy-grafana.j2 @@ -1,4 +1,3 @@ ---- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: @@ -8,7 +7,7 @@ spec: chart: stable/grafana #targetNamespace: monitoring set: - adminPassword: "{{grafana_password}}" + adminPassword: "{{ grafana_password }}" valuesContent: |- image: tag: master diff --git a/roles/rke2/templates/helm/deploy-openstack-ccm.j2 b/roles/rke2/templates/helm/deploy-openstack-ccm.j2 new file mode 100644 index 0000000000000000000000000000000000000000..234918112c50869cbb1265c9759e5a4628de7f90 --- /dev/null +++ b/roles/rke2/templates/helm/deploy-openstack-ccm.j2 @@ -0,0 +1,51 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: openstack-ccm + namespace: kube-system +spec: + # repo: https://kubernetes.github.io/cloud-provider-openstack + chart: http://s146.dl.hpc.tuwien.ac.at/openstack-cloud-controller-manager-1.1.2.tgz + # version: 1.0.1 + bootstrap: True + valuesContent: |- + cloudConfig: + global: + auth-url: {{ openstack_auth.auth_url }} + application-credential-id: {{ openstack_auth.application_credential_id }} + application-credential-secret: {{ openstack_auth.application_credential_secret }} + region: {{ openstack_region_name }} + loadBalancer: + subnet-id: {{ subnet_id }} + floating-network-id: {{ floating_network_id }} + +{% if router_id is defined %} + route: + router-id: {{ router_id }} +{% endif %} + + tolerations: + - key: node.cloudprovider.kubernetes.io/uninitialized + value: "true" + effect: NoSchedule + - key: node-role.kubernetes.io/master + value: "true" + effect: NoSchedule + + nodeSelector: + node-role.kubernetes.io/control-plane: "true" + + # serviceMonitor: + # enabled: "true" + + {# livenessProbe: + httpGet: + path: /metrics + port: 10258 #} + + controllerExtraArgs: |- + - --cluster-name=rke2-{{ cluster_name }} + + resources: + requests: + cpu: 200m \ No newline at end of file diff --git a/roles/rke2/templates/helm/deploy-openstack-cinder-csi.j2 b/roles/rke2/templates/helm/deploy-openstack-cinder-csi.j2 deleted file mode 100644 index 0b92d6227f8a57b79298c7530c066231d006829a..0000000000000000000000000000000000000000 --- a/roles/rke2/templates/helm/deploy-openstack-cinder-csi.j2 +++ /dev/null @@ -1,11 +0,0 @@ -apiVersion: helm.cattle.io/v1 -kind: HelmChart -metadata: - name: cinder-csi - namespace: kube-system -spec: - repo: https://rke2-charts.rancher.io - chart: cinder-csi - bootstrap: true - valuesContent: |- - cilium: {} \ No newline at end of file diff --git a/roles/rke2/templates/helm/deploy-openstack-cinder.j2 b/roles/rke2/templates/helm/deploy-openstack-cinder.j2 new file mode 100644 index 0000000000000000000000000000000000000000..354a29416a9d85c49c947d2d1c2df405a6aef03c --- /dev/null +++ b/roles/rke2/templates/helm/deploy-openstack-cinder.j2 @@ -0,0 +1,25 @@ +apiVersion: helm.cattle.io/v1 +kind: HelmChart +metadata: + name: openstack-cinder-csi + namespace: kube-system +spec: + repo: https://kubernetes.github.io/cloud-provider-openstack + chart: openstack-cinder-csi + version: 1.3.8 + valuesContent: |- + secret: + enabled: true + create: true + name: cinder-csi-cloud-config + data: + cloud-config: |- + [Global] + auth-url={{ openstack_auth.auth_url }} + application-credential-id={{ openstack_auth.application_credential_id }} + application-credential-secret={{ openstack_auth.application_credential_secret }} + region={{ openstack_region_name }} + + storageClass: + delete: + isDefault: true diff --git a/roles/rke2/templates/helm/deploy-rancher-ui.j2 b/roles/rke2/templates/helm/deploy-rancher-ui.j2 index b995d910f8fda805ca473c5d48b0c720409822ed..c36482cacf46ecf5697c8e10f8fc7e02b2bbcbf9 100644 --- a/roles/rke2/templates/helm/deploy-rancher-ui.j2 +++ b/roles/rke2/templates/helm/deploy-rancher-ui.j2 @@ -1,4 +1,3 @@ ---- apiVersion: helm.cattle.io/v1 kind: HelmChart metadata: diff --git a/roles/rke2/templates/kernel_params_conf.j2 b/roles/rke2/templates/kernel_params.conf.j2 similarity index 100% rename from roles/rke2/templates/kernel_params_conf.j2 rename to roles/rke2/templates/kernel_params.conf.j2 diff --git a/roles/rke2/templates/rke2_conf.j2 b/roles/rke2/templates/rke2_conf.j2 deleted file mode 100644 index 8a30a477fa1aaa163e13bcd89159ab794c6b2761..0000000000000000000000000000000000000000 --- a/roles/rke2/templates/rke2_conf.j2 +++ /dev/null @@ -1,53 +0,0 @@ -# all rke2 instances - -node-ip: "{{ ansible_default_ipv4.address }}" -#node-name: "{{ ansible_default_ipv4.address }}" -profile: "cis-1.5" -resolv-conf: "/etc/resolv.conf" -debug: true - -# server slaves and agents only -{% if inventory_hostname != master %} -server: "{{ server }}" -token: "{{ hostvars['MASTER_HOST']['token'] }}" -{% endif %} - -# servers only -{% if 'kubemgmt' in group_names %} -cloud-provider-name: "external" -disable: rke2-canal -write-kubeconfig-mode: "0644" -tls-san: - - {{ rancher_ui_dns }} -{% for item in tls_san %} - - {{ item }} -{% endfor %} -node-taint: -{% for item in mgmt_node_taints %} - - {{ item }} -{% endfor %} -node-label: -{% for item in mgmt_node_labels %} - - {{ item }} -{% endfor %} -{% endif %} - -# agents only -{% if 'kubeagents' in group_names %} -node-label: -{% for item in agent_node_labels %} - - {{ item }} -{% endfor %} -{% endif %} - - - -#cloud-provider-config: "???/etc/kubernetes/vsphere.conf" -## maybe? -#private-registry: "/etc/rancher/rke2/registries.yaml" -#disable: -# - "rke2-ingress-nginx" -#container-runtime-endpoint: "/run/containerd/containerd.sock" -#node-label: -# - "foo=bar" -# - "something=amazing" \ No newline at end of file diff --git a/roles/rke2_agent/tasks/install_rke2_agent.yml b/roles/rke2_agent/tasks/install_rke2_agent.yml deleted file mode 100644 index 0f1c6c6509bdb10fb59376d4e0848bd68eee9dc3..0000000000000000000000000000000000000000 --- a/roles/rke2_agent/tasks/install_rke2_agent.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -- name: Install rke2 agent - shell: curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=latest INSTALL_RKE2_TYPE="agent" sh - \ No newline at end of file diff --git a/roles/rke2_server/tasks/install_rke2_server.yml b/roles/rke2_server/tasks/install_rke2_server.yml deleted file mode 100644 index 3de052ca6bd0ff3426ccdb051c953ce0409a73f6..0000000000000000000000000000000000000000 --- a/roles/rke2_server/tasks/install_rke2_server.yml +++ /dev/null @@ -1,4 +0,0 @@ ---- - -- name: Install rke2 server - shell: curl -sfL https://get.rke2.io | INSTALL_RKE2_CHANNEL=latest sh - \ No newline at end of file