.vscode/settings.json

+{
+    "yaml.schemas": {
+        "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible.json#/$defs/tasks": "file:///home/ewimmer/workspace/datalab/ansible/collections/datalab.osranger/collections/ansible_collections/datalab/osrancher/roles/create_infrastructure/tasks/inventory.yml"
+    }
+}
\ No newline at end of file

MANIFEST.json

+{
+ "collection_info": {
+  "namespace": "datalab",
+  "name": "osrancher",
+  "version": "1.0.0",
+  "authors": [
+   "Thomas Weber <thomas.weber@wu.ac.at>",
+   "Elias Wimmer <elias.wimmer@tuwien.ac.at>"
+  ],
+  "readme": "README.md",
+  "tags": [],
+  "description": "Setup a RKE2 Kubernetes Cluster on OpenStack",
+  "license": [
+   "GPL-2.0-or-later"
+  ],
+  "license_file": null,
+  "dependencies": {},
+  "repository": "https://gitlab.tuwien.ac.at/ADLS/infrastructure/adls.osrancher",
+  "documentation": "https://gitlab.tuwien.ac.at/ADLS/infrastructure/adls.osrancher",
+  "homepage": "https://gitlab.tuwien.ac.at/ADLS/infrastructure/adls.osrancher",
+  "issues": "http://example.com/issue/tracker"
+ },
+ "format": 1
+}
\ No newline at end of file

README.md

@@ -4,7 +4,7 @@ This collection contains roles for setup of a Kubernetes cluster with OpenStack
 
 ## Install
 ```
-ansible-galaxy collection install git+ssh://git@gitlab.tuwien.ac.at:822/ADLS/infrastructure/adls.osrancher.git
+ansible-galaxy collection install git+ssh://git@gitlab.tuwien.ac.at:822/datalab/ansible/collections/datalab.osrancher.git
 ```
 
 ## Usage
@@ -16,31 +16,29 @@ Use roles inside a Ansible playbook
   vars: 
     cluster_name: test
   roles:
-    adls.osrancher.create_infrastructure
+    datalab.osrancher.create_infrastructure
 
 # install and configure RKE2
 - hosts: all
   vars: 
   roles:
-    adls.osrancher.rke2    
+    datalab.osrancher.rke2    
 ```
 
 ## Variables
 
-| Variable           | Default                              | Description                                                                                 |
-| ------------------ | ------------------------------------ | ------------------------------------------------------------------------------------------- |
-| cluster_name       |                                      | Name of the RKE2 cluster                                                                    |
-| server_count       | 3                                    | Number of RKE2 worker VMs                                                                   |
-| agent_count        | 3                                    | Number of RKE2 server VMs                                                                   |
-| server_flavor      | m1a.large                            | Server VM flavor                                                                            |
-| agent_flavor       | m1a.xlarge                           | Worker VM flavor                                                                            |
-| server_volume_size | 50                                   | Volume size (GB) for server VM                                                              |
-| agent_volume_size  | 100                                  | Volume size (GB) for worker VM                                                              |
-| image              | 1fe615f0-9dad-447d-bf54-9071defafb77 | ID for OpenStack VM image                                                                   |
-| domain             |                                      | DNS-Entry for loadbalancer IP                                                               |
-| node_taints        |                                      | Node taints for RKE2 node                                                                   |
-| node_labels        |                                      | Node labels for RKE2 node                                                                   |
-| rke2_channel       | stable                               | RKE3 version channel                                                                        |
-| state              | present                              | Flag for setup (`present`) or removing (`absent`) RKE3 cluster                              |
-| registry_mirrors   | {}                                   | [rke2-docs](https://docs.rke2.io/install/containerd_registry_configuration/#mirrors)        |
-| manifests          | {}                                   | [defualts](collections/ansible_collections/adls/osrancher/roles/rke2/defaults/main.yml#L58) |
+| Variable | Default | Description |
+| ----------- | ----------- | ----------- |
+| cluster_name |  | Name of the RKE2 cluster |
+| server_count | 3 | Number of RKE2 worker VMs |
+| agent_count | 3 | Number of RKE2 server VMs |
+| server_flavor | m1a.large | Server VM flavor |
+| agent_flavor | m1a.xlarge | Worker VM flavor |
+| server_volume_size | 50 | Volume size (GB) for server VM |
+| agent_volume_size | 100 | Volume size (GB) for worker VM |
+| image | 1fe615f0-9dad-447d-bf54-9071defafb77 | ID for OpenStack VM image |
+| domain | | DNS-Entry for loadbalancer IP |
+| node_taints | | Node taints for RKE2 node |
+| node_labels | | Node labels for RKE2 node |
+| rke2_channel | stable | RKE3 version channel |
+| state | present | Flag for setup (`present`) or removing (`absent`) RKE3 cluster |

roles/create_infrastructure/defaults/main.yml

@@ -3,7 +3,7 @@ ssh_key_name: "rke2-{{ cluster_name }}-ssh-key"
 ssh_key_file: "{{ ssh_keys_dir }}/{{ ssh_key_name }}.pem"
 network_name: "rke2-{{ cluster_name }}"
 subnet_name: "rke2-{{ cluster_name }}"
-cidr: 192.168.13.0/24
+cidr: 10.0.0.0/24
 router_name: "rke2-{{ cluster_name }}"
 
 server_volume_size: 50

roles/create_infrastructure/templates/agents.yml.j2

+{% raw %}
+ansible_user: "ubuntu"
+ansible_ssh_private_key_file: "{{ ssh_key_file }}"
+ansible_ssh_common_args: "-o ProxyCommand='ssh -q ubuntu@{{ master_floating_ip }} -o StrictHostKeyChecking=no -i {{ ssh_key_file }} -W %h:%p' -o StrictHostKeyChecking=no"
+node_type: agent
+{% endraw %}
\ No newline at end of file

roles/create_infrastructure/templates/all.yml.j2

+loadbalancer_ip: {{ new_loadbalancer_ip }}
+
+master_floating_ip: {{ new_master_floating_ip }}
+
+subnet_id: {{ new_subnet_id }}
+
+ssh_key_file: {{ ssh_key_file }}
\ No newline at end of file

roles/create_infrastructure/templates/master.yml.j2

+{% raw %}
+ansible_user: "ubuntu"
+ansible_ssh_private_key_file: "{{ ssh_key_file }}"
+ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+node_type: server
+{% endraw %}
\ No newline at end of file

roles/create_infrastructure/templates/servers.yml.j2

+{% raw %}
+ansible_user: "ubuntu"
+ansible_ssh_private_key_file: "{{ ssh_key_file }}"
+ansible_ssh_common_args: "-o ProxyCommand='ssh -q ubuntu@{{ master_floating_ip }} -o StrictHostKeyChecking=no -i {{ ssh_key_file }} -W %h:%p' -o StrictHostKeyChecking=no"
+node_type: server
+{% endraw %}
\ No newline at end of file

roles/rke2/tasks/helm.yml

+- name: copy helm template files
+  ansible.builtin.template:
+    src: 'helm/{{ item.template }}.j2'
+    dest: '/var/lib/rancher/rke2/server/manifests/{{ item.template }}.yaml'
+  loop:
+      - { template: 'deploy-openstack-ccm'}
+      - { template: 'deploy-openstack-cinder'}
+      # - { template: 'deploy-openstack-manila'}
+      - { template: 'deploy-cephfs'}
+      # - { template: 'deploy-nfs'}
+      # - { template: 'deploy-grafana'}
+      # - { template: 'deploy-cert-manager'}
+      # - { template: 'deploy-rancher-ui'}
+      # - { template: 'deploy-rke2-cilium'}
+      - { template: 'config-rke2-canal'}
+      - { template: 'config-nginx-ingress'}
+      # - { template: 'config-rke2-coredns'}
+  notify:
+    - restart rke2
\ No newline at end of file

roles/rke2/tasks/kubeconfig.yml

+- name: wait for kubeconfig
+  ansible.builtin.wait_for:
+    path: /etc/rancher/rke2/rke2.yaml
+
+- name: fetch kubeconfig from master
+  ansible.builtin.fetch:
+    src: /etc/rancher/rke2/rke2.yaml
+    dest: kubeconfig.yaml
+    flat: yes
+
+- name: replace endpoint in kubeconfig
+  delegate_to: localhost
+  become: no
+  ansible.builtin.replace:
+    path: kubeconfig.yaml
+    regexp: '^(\s+server: ).*'
+    replace: '\1https://{{ domain }}:6443'

roles/rke2/templates/all.yml

+token: {{ token }}
+
+server: "https://{{ domain }}:9345"

roles/rke2/templates/helm/config-nginx-ingress.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-ingress-nginx
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    controller:
+      config: 
+        use-forwarded-headers: "true"
+        proxy-body-size: 64m
+    tolerations:
+      - effect: NoSchedule
\ No newline at end of file

roles/rke2/templates/helm/config-rke2-canal.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-canal
+  namespace: kube-system
+spec:
+    valuesContent: |-
+      calico:
+        vethuMTU: 1400
+        networkingBackend: "vxlan"
+        masquerade: false
\ No newline at end of file

roles/rke2/templates/helm/config-rke2-coredns.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+  name: rke2-coredns
+  namespace: kube-system
+spec:
+  valuesContent: |-
+    nodeSelector:
+      role: "agent"

roles/rke2/templates/helm/deploy-cephfs.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cephfs-csi
+  namespace: kube-system
+spec:
+  repo: https://ceph.github.io/csi-charts
+  chart: ceph-csi-cephfs
+  version: {{ ceph_csi_version}}
+  valuesContent: |-
+    tolerations:
+      - effect: NoSchedule
+    csiConfig:
+     - clusterID: "{{ ceph_cluster_id }}"
+       monitors:
+{% for mon in ceph_monitors %}
+         - {{ mon }}
+{% endfor %}
\ No newline at end of file

roles/rke2/templates/helm/deploy-cert-manager.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: cert-manager
+  namespace: kube-system
+spec:
+  repo: https://charts.jetstack.io
+  chart: cert-manager
+  #targetNamespace: cert-manager
+  version: v1.3.1
+  set:
+    installCRDs: "true"
\ No newline at end of file

roles/rke2/templates/helm/deploy-grafana.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: grafana
+  namespace: kube-system
+spec:
+  chart: stable/grafana
+  #targetNamespace: monitoring
+  set:
+    adminPassword: "{{ grafana_password }}"
+  valuesContent: |-
+    image:
+      tag: master
+    env:
+      GF_EXPLORE_ENABLED: true
+    adminUser: admin
+    sidecar:
+      datasources:
+        enabled: true
\ No newline at end of file

roles/rke2/templates/helm/deploy-openstack-ccm.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: openstack-ccm
+  namespace: kube-system
+spec:
+  repo: https://kubernetes.github.io/cloud-provider-openstack
+  chart: openstack-cloud-controller-manager
+  version: {{ openstack_ccm_version }}
+  bootstrap: True
+  valuesContent: |-
+    cloudConfig:
+      global:
+        auth-url: {{ openstack_auth.auth_url }}
+        application-credential-id: {{ openstack_auth.application_credential_id }}
+        application-credential-secret: {{ openstack_auth.application_credential_secret }}
+        region: {{ openstack_region_name }}
+      loadBalancer:
+        subnet-id: {{ subnet_id }}
+        floating-network-id: {{ floating_network_id }}
+
+{% if router_id is defined %}      
+      route:
+        router-id: {{ router_id }}
+{% endif %}
+
+    tolerations:
+      - key: node.cloudprovider.kubernetes.io/uninitialized
+        value: "true"
+        effect: NoSchedule
+      - key: node-role.kubernetes.io/master
+        value: "true"
+        effect: NoSchedule
+      - key: CriticalAddonsOnly
+        value: "true"
+        effect: NoExecute
+
+    nodeSelector:
+      node-role.kubernetes.io/control-plane: "true"
+
+    # serviceMonitor:
+    #   enabled: "true"
+
+    {# livenessProbe:
+      httpGet:
+        path: /metrics
+        port: 10258 #}
+
+    controllerExtraArgs: |-
+      - --cluster-name=rke2-{{ cluster_name }}
+
+    resources:
+      requests:
+        cpu: 200m
\ No newline at end of file

roles/rke2/templates/helm/deploy-openstack-cinder.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: openstack-cinder-csi
+  namespace: kube-system
+spec:
+  repo: https://kubernetes.github.io/cloud-provider-openstack
+  chart: openstack-cinder-csi
+  version: {{ cinder_csi_version }}
+  valuesContent: |-
+    secret:
+      enabled: true
+      create: true
+      name: cinder-csi-cloud-config
+      data:
+        cloud.conf: |-
+          [Global]
+          auth-url={{ openstack_auth.auth_url }}
+          application-credential-id={{ openstack_auth.application_credential_id }}
+          application-credential-secret={{ openstack_auth.application_credential_secret }}
+          region={{ openstack_region_name }}
+
+    storageClass:
+      delete:
+        isDefault: true

roles/rke2/templates/helm/deploy-rancher-ui.j2

+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+  name: rancher
+  namespace: kube-system
+spec:
+  repo: https://releases.rancher.com/server-charts/latest
+  chart: rancher
+  version: 2.5.9-rc2
+  #targetNamespace: cattle-system
+  set:
+    hostname: "{{rancher_ui_dns}}"
+    letsEncrypt.email: "{{letsEncrypt_admin_mail}}"
+    ingress.tls.source: "letsEncrypt"
\ No newline at end of file