Initial commit

roles/infrastructure/tasks/keypair.yml

+- name: create ssh key pair
+  openstack.cloud.keypair:
+    cloud: "{{ cloud }}"
+    state: present
+    name: "{{ ssh_key_name }}"
+  register: ssh_key
\ No newline at end of file

roles/infrastructure/tasks/loadbalancer.yml

+- name: create loadbalancer
+  openstack.cloud.loadbalancer:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}"
+    vip_subnet: "{{ network_name }}"
+
+- name: add listener 6443
+  openstack.cloud.lb_listener:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}-listener-6443"
+    loadbalancer: "{{ loadbalancer_name }}"
+    protocol: TCP
+    protocol_port: 6443
+
+- name: add listener 9345
+  openstack.cloud.lb_listener:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}-listener-9345"
+    loadbalancer: "{{ loadbalancer_name }}"
+    protocol: TCP
+    protocol_port: 9345
+
+- name: add pool 6443
+  openstack.cloud.lb_pool:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}-pool-6443"
+    loadbalancer: "{{ loadbalancer_name }}"
+    protocol: TCP
+    lb_algorithm: ROUND_ROBIN
+
+- name: add pool 9345
+  openstack.cloud.lb_pool:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}-pool-9345"
+    loadbalancer: "{{ loadbalancer_name }}"
+    protocol: TCP
+    lb_algorithm: ROUND_ROBIN
+
+- name: add members 6443
+  openstack.cloud.lb_member:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}-pool-6443-{{ inventory_hostname }}"
+    pool: "{{ loadbalancer_name }}-pool-6443"
+    address: "{{ ansible primary ip }}"
+    protocol_port: 6443
+  loop: 
+
+- name: add members 9345
+  openstack.cloud.lb_member:
+    cloud: "{{ cloud }}"
+    name: "{{ loadbalancer_name }}-pool-9345-{{ inventory_hostname }}"
+    pool: "{{ loadbalancer_name }}-pool-9345"
+    address: "{{ ansible primary ip }}"
+    protocol_port: 9345
+  loop: 
+
+
+- name: add health monitor 6443
+  openstack.cloud.lb_health_monitor:
+    cloud: "{{ cloud }}"
+    expected_codes: '200'
+    max_retries_down: '4'
+    pool: "{{ loadbalancer_name }}-pool-6443"
+    name: "{{ loadbalancer_name }}-pool-6443-healthmonitor"
+    delay: '20'
+    max_retries: '5'
+    resp_timeout: '10'
+    type: TCP
+
+- name: add health monitor 9345
+  openstack.cloud.lb_health_monitor:
+    cloud: "{{ cloud }}"
+    expected_codes: '200'
+    max_retries_down: '4'
+    pool: "{{ loadbalancer_name }}-pool-9345"
+    name: "{{ loadbalancer_name }}-pool-9345-healthmonitor"
+    delay: '20'
+    max_retries: '5'
+    resp_timeout: '10'
+    type: TCP
+
+
+
+# resource "openstack_lb_member_v2" "members_k8s_mgmt_6443" {
+#     count           = var.rancher.count_mgmt_nodes
+#     address         = openstack_compute_instance_v2.rancher_mgmt[count.index].network[0].fixed_ip_v4
+#     pool_id         = openstack_lb_pool_v2.k8s_pool_6443.id
+#     protocol_port   = 6443
+#     subnet_id       = openstack_networking_subnet_v2.k8s_mgmt_subnet.id
+#     depends_on = [
+#         openstack_networking_subnet_v2.k8s_mgmt_subnet,
+#         openstack_lb_pool_v2.k8s_pool_6443,
+#         openstack_compute_instance_v2.rancher_mgmt
+#     ]
+# }
+
+# resource "openstack_lb_member_v2" "members_k8s_mgmt_9345" {
+#     count           = var.rancher.count_mgmt_nodes
+#     address         = openstack_compute_instance_v2.rancher_mgmt[count.index].network[0].fixed_ip_v4
+#     pool_id         = openstack_lb_pool_v2.k8s_pool_9345.id
+#     protocol_port   = 9345
+#     subnet_id       = openstack_networking_subnet_v2.k8s_mgmt_subnet.id
+#     depends_on = [ openstack_networking_subnet_v2.k8s_mgmt_subnet, openstack_lb_pool_v2.k8s_pool_9345, openstack_compute_instance_v2.rancher_mgmt]
+# }
+
+# resource "openstack_lb_monitor_v2" "monitor_k8s_mgmt_6443" {
+#   pool_id     = openstack_lb_pool_v2.k8s_pool_6443.id
+#   type        = "TCP"
+#   delay       = 20
+#   timeout     = 10
+#   max_retries = 5
+# }
+# resource "openstack_lb_monitor_v2" "monitor_k8s_mgmt_9345" {
+#   pool_id     = openstack_lb_pool_v2.k8s_pool_9345.id
+#   type        = "TCP"
+#   delay       = 20
+#   timeout     = 10
+#   max_retries = 5
+# }
\ No newline at end of file

roles/infrastructure/tasks/network.yml

+- name: create network
+  openstack.cloud.network:
+    cloud: "{{ cloud }}"
+    name: "{{ network_name }}"
+
+- name: create subnet
+  openstack.cloud.subnet:
+    cloud: "{{ cloud }}" 
+    network_name: "{{ network_name }}"
+    name: "{{ subnet_name }}"
+    cidr: "{{ cidr }}"
+    dns_nameservers: "{{ nameserver }}"
+    
+- name: create router
+  openstack.cloud.router:
+    cloud: "{{ cloud }}"
+    name: "{{ router_name }}"
+    network: public
+    interfaces:
+      - "{{ network_name }}"
\ No newline at end of file

roles/infrastructure/tasks/setup_vm.yml

+
+
+- name: Update the /etc/hosts file with node name
+  tags: etchostsupdate
+  lineinfile:
+    dest: "/etc/hosts"
+    regexp: ".*\t{{ hostvars[item]['ansible_hostname']}}\t{{ hostvars[item]['ansible_hostname']}}"
+    line: "{{ hostvars[item]['ansible_default_ipv4']['address'] }}\t{{ hostvars[item]['ansible_hostname']}}\t{{ hostvars[item]['ansible_hostname']}}"
+    state: present
+    backup: yes
+  register: etchostsupdate
+  when: ansible_hostname != "{{ item }}" or ansible_hostname == "{{ item }}"
+  with_items: "{{groups['all']}}"
+
+- name: add kernel params
+  template:
+    src: kernel_params_conf.j2
+    dest: /etc/sysctl.d/rke2_kernel_params.conf
+
+- name: load kernel params
+  shell: sysctl --system
+
+- name: add group - etcd 
+  group:
+    name: etcd
+    state: present
+
+- name: add user - etcd
+  user:
+    name: etcd
+    group: etcd
+
+- name: Ensure /var/lib/rancher/rke2/server/manifests
+  file:
+    path: /var/lib/rancher/rke2/server/manifests
+    state: directory
+    recurse: yes
+
+- name: Ensure /etc/rancher/rke2
+  file:
+    path: /etc/rancher/rke2
+    state: directory
+    recurse: yes
\ No newline at end of file

roles/infrastructure/tasks/vm.yml

+- name: create server vms
+  openstack.cloud.server:
+    cloud: "{{ cloud }}"
+    name: "{{ name }}-server-{{ item }}"
+    boot_from_volume: yes
+    terminate_volume: yes
+    volume_size: "{{ server_volume_size }}"
+    network: "{{ network_name }}"
+    key_name: "{{ ssh_key_name }}"
+    flavor: "{{ server_flavor }}"
+    image: "{{ image }}"
+  loop: "{{ range(0, server_count, 1) | list }}"
+  register: servers
+
+- name: add servers to inventory
+  add_host:
+    ansible_host: "{{ item.server.private_v4 }}"
+    ansible_user: ubuntu
+    group: server
+    name: "{{ item.server.name }}"
+  loop: "{{ servers }}"
+
+- name: create agent vms
+  openstack.cloud.server:
+    cloud: "{{ cloud }}"
+    name: "{{ name }}-agent-{{ item }}"
+    boot_from_volume: yes
+    terminate_volume: yes
+    volume_size: "{{ agent_volume_size }}"
+    network: "{{ network_name }}"
+    key_name: "{{ ssh_key_name }}"
+    flavor: "{{ agent_flavor }}"
+    image: "{{ image }}"
+  loop: "{{ range(0, agent_count, 1) | list }}"
+  register: agents
+
+- name: add agents to inventory
+  add_host:
+    ansible_host: "{{ item.server.private_v4 }}"
+    ansible_user: ubuntu
+    group: agent
+    name: "{{ item.server.name }}"
+  loop: "{{ agents }}"

roles/rke2/defaults/main.yml

+server: https://mgmtlb.k8s.example:9345
+master: mgmt-1
+tls_san:
+  - "mgmtlb.k8s.example"
+  - "another.k8s.example"
+server_node_taints:
+  - "CriticalAddonsOnly=true:NoExecute"
+grafana_password: "PASSWORD"
+rancher_ui_dns: "ui.k8s.example"
+letsEncrypt_admin_mail: "test@test.com"
\ No newline at end of file

roles/rke2/handlers/main.yml

+---
+
+# restart rke2 configuration
+- name: restart_rke2
+  service:
+    name: rke2-server
+    enabled: yes
+    masked: no
+    state: restarted
+    daemon_reload: yes
+
+# restart rke2 configuration
+- name: restart_rke2_agent
+  service:
+    name: rke2-agent
+    enabled: yes
+    masked: no
+    state: restarted
+    daemon_reload: yes
\ No newline at end of file

roles/rke2/tasks/add_cloud_config.yml

+---
+
+- name: render cloud-config template to variable
+  set_fact:
+    cloud_config: "{{ lookup('template', 'cloud-config.j2') }}"
+
+- name: add cloud config
+  template:
+    src: deploy-openstack-cloud-config-assets.j2
+    dest: /var/lib/rancher/rke2/server/manifests/deploy-openstack-cloud-config-assets.yaml
+
+
+- name: download openstack manifests
+  get_url:
+    url: "{{ item.url }}"
+    dest: '/var/lib/rancher/rke2/server/manifests/{{ item.dest }}'
+  loop:
+      - { dest: 'cloud-controller-manager-roles.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-roles.yaml'}
+      - { dest: 'cloud-controller-manager-role-bindings.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/master/manifests/controller-manager/cloud-controller-manager-role-bindings.yaml'}
+      - { dest: 'openstack-cloud-controller-manager-ds.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/controller-manager/openstack-cloud-controller-manager-ds.yaml'}
+      - { dest: 'cinder-csi-controllerplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-controllerplugin-rbac.yaml'}
+      - { dest: 'cinder-csi-controllerplugin.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-controllerplugin.yaml'}
+      - { dest: 'cinder-csi-nodeplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-nodeplugin-rbac.yaml'}
+      - { dest: 'cinder-csi-nodeplugin.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/cinder-csi-nodeplugin.yaml'}
+      - { dest: 'csi-cinder-driver.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/cinder-csi-plugin/csi-cinder-driver.yaml'}
+      - { dest: 'snapshot.storage.k8s.io_volumesnapshotclasses.yaml', url: 'https://github.com/kubernetes-csi/external-snapshotter/raw/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotclasses.yaml'}
+      - { dest: 'snapshot.storage.k8s.io_volumesnapshotcontents.yaml', url: 'https://github.com/kubernetes-csi/external-snapshotter/raw/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshotcontents.yaml'}
+      - { dest: 'snapshot.storage.k8s.io_volumesnapshots.yaml', url: 'https://github.com/kubernetes-csi/external-snapshotter/raw/master/client/config/crd/snapshot.storage.k8s.io_volumesnapshots.yaml'}
+      - { dest: 'rbac-snapshot-controller.yaml', url: 'https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/rbac-snapshot-controller.yaml'}
+      - { dest: 'setup-snapshot-controller.yaml', url: 'https://raw.githubusercontent.com/kubernetes-csi/external-snapshotter/master/deploy/kubernetes/snapshot-controller/setup-snapshot-controller.yaml'}
+      #- { dest: 'manila-csi-controllerplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/manila-csi-plugin/csi-controllerplugin-rbac.yaml'}
+      #- { dest: 'manila-csi-nodeplugin-rbac.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/manila-csi-plugin/csi-nodeplugin-rbac.yaml'}
+      #- { dest: 'manila-csidriver.yaml', url: 'https://raw.githubusercontent.com/kubernetes/cloud-provider-openstack/v1.21.1/manifests/manila-csi-plugin/csidriver.yaml'}
+
+
+- name: Replace Master Selector openstack-cloud-controller-manager-ds.yaml
+  ansible.builtin.replace:
+    path: /var/lib/rancher/rke2/server/manifests/openstack-cloud-controller-manager-ds.yaml
+    regexp: 'node-role\.kubernetes\.io\/master: ""'
+    replace: 'node-role.kubernetes.io/master: "true"'
\ No newline at end of file

roles/rke2/tasks/configure_rke2_master.yml

+---
+
+# config
+- name: rke2 config
+  template:
+    src: rke2_conf.j2
+    dest: /etc/rancher/rke2/config.yaml
+  notify:
+    - restart_rke2
+
+# helm charts
+- name: ensure all helm template files are rendered
+  template:
+    src: 'helm/{{ item.template }}.j2'
+    dest: '/var/lib/rancher/rke2/server/manifests/{{ item.template }}.yaml'
+  loop:
+      #- { template: 'conf-rke2-canal'}
+      - { template: 'conf-nginx-ingress'}
+      - { template: 'deploy-grafana'}
+      - { template: 'deploy-cert-manager'}
+      - { template: 'deploy-rancher-ui'}
+      - { template: 'deploy-rke2-cilium'}
+      - { template: 'rke2-coredns-config'}
+  notify:
+    - restart_rke2
+
+
+# start rke2 server
+- name: start_rke2
+  service:
+    name: rke2-server
+    enabled: yes
+    masked: no
+    state: started
+
+# slurp token
+- name: Wait until the token is present before continuing
+  wait_for:
+    path: /var/lib/rancher/rke2/server/node-token
+
+- name: Load token
+  slurp:
+    src: "/var/lib/rancher/rke2/server/node-token"
+  register: slurped_token
+
+- name: Decode token and store as fact at dummy master_host with host variable
+  add_host:
+    name: "MASTER_HOST"
+    token: "{{ slurped_token.content | b64decode | trim }}"
\ No newline at end of file

roles/rke2/tasks/main.yml

+---
+- include_tasks: add_basics.yml
+
+
+- name: configure, and start RKE2
+  when: "'kubemgmt' in group_names"
+  block:
+
+    - include_tasks: install_rke2_server.yml
+
+    - name: configure, and start RKE2 Master
+      when: inventory_hostname == master
+      block:
+
+        - include_tasks: configure_rke2_master.yml
+        - include_tasks: add_cloud_config.yml
+
+
+
+## just when token is ready
+- name: configure, and start RKE2 Slave
+  when: inventory_hostname != master
+  block:
+
+    - name: rke2 config
+      when: "'kubemgmt' in group_names"
+      throttle: 1
+      template:
+        src: rke2_conf.j2
+        dest: /etc/rancher/rke2/config.yaml
+      notify: 
+        - restart_rke2
+
+
+    - name: configure, and start RKE2 AGENT
+      when: "'kubeagents' in group_names"
+      block:
+      
+        - include_tasks: install_rke2_agent.yml
+        
+        - name: rke2 config
+          throttle: 1
+          template:
+            src: rke2_conf.j2
+            dest: /etc/rancher/rke2/config.yaml
+          notify: 
+            - restart_rke2_agent

roles/rke2/templates/cloud-config.j2

+[global]
+auth-url={{openstack_auth_url}}
+application-credential-id=2c38a96c670a44bc8011720a16d9c9fd
+application-credential-secret=2Y6pIxUzLfI8N7I0HlbLbpBKS-QvEgXFUzXs7jxMEgHxLFjbk8hA_AQpE24tkGR3ZYcSieOEM4RwPDcMABcuWw
+region=RegionOne
+
+[LoadBalancer]
+use-octavia=true
+subnet-id=b88e8883-3a9c-4225-8e21-23f61a2022b9
+
+[BlockStorage]
+#bs-version=v2
\ No newline at end of file

roles/rke2/templates/deploy-openstack-cloud-config-assets.j2

+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloud-config
+  namespace: kube-system
+type: Opaque
+data:
+  cloud.conf: {{  cloud_config  | string | b64encode }}
+
+---
+
+apiVersion: storage.k8s.io/v1
+kind: StorageClass
+metadata:
+  name: csi-sc-cinderplugin
+provisioner: cinder.csi.openstack.org
\ No newline at end of file

roles/rke2/templates/helm/conf-nginx-ingress.j2

+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+
+spec:
+  set:
+    controller:
+      config:
+        entries: |
+          upstream-keepalive-timeout: 3600
+  valuesContent: |-
+    controller:
+      kind: DaemonSet
+      daemonset:
+        useHostPort: true
+      image:
+        repository: k8s.gcr.io/ingress-nginx/controller
+        tag: "v0.45.0"
+      config: 
+        use-forwarded-headers: "true"
\ No newline at end of file

roles/rke2/templates/helm/conf-rke2-canal.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-canal
+  namespace: kube-system
+spec:
+    valuesContent: |-
+        calico:
+            vethuMTU: 1400
+            networkingBackend: "vxlan"
+            masquerade: false
\ No newline at end of file

roles/rke2/templates/helm/deploy-cert-manager.j2

+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cert-manager
+  namespace: kube-system
+spec:
+  repo: https://charts.jetstack.io
+  chart: cert-manager
+  #targetNamespace: cert-manager
+  version: v1.3.1
+  set:
+    installCRDs: "true"
\ No newline at end of file

roles/rke2/templates/helm/deploy-grafana.j2

+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: grafana
+  namespace: kube-system
+spec:
+  chart: stable/grafana
+  #targetNamespace: monitoring
+  set:
+    adminPassword: "{{grafana_password}}"
+  valuesContent: |-
+    image:
+      tag: master
+    env:
+      GF_EXPLORE_ENABLED: true
+    adminUser: admin
+    sidecar:
+      datasources:
+        enabled: true
\ No newline at end of file

roles/rke2/templates/helm/deploy-openstack-cinder-csi.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cinder-csi
+  namespace: kube-system
+spec:
+  repo: https://rke2-charts.rancher.io
+  chart: cinder-csi
+  bootstrap: true
+  valuesContent: |-
+    cilium: {}
\ No newline at end of file

roles/rke2/templates/helm/deploy-rancher-ui.j2

+---
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: rancher
+  namespace: kube-system
+spec:
+  repo: https://releases.rancher.com/server-charts/latest
+  chart: rancher
+  version: 2.5.9-rc2
+  #targetNamespace: cattle-system
+  set:
+    hostname: "{{rancher_ui_dns}}"
+    letsEncrypt.email: "{{letsEncrypt_admin_mail}}"
+    ingress.tls.source: "letsEncrypt"
\ No newline at end of file