diff --git a/.vscode/settings.json b/.vscode/settings.json
new file mode 100644
index 0000000000000000000000000000000000000000..12268f11c70ebcb4ce8387e7220e40ada34cabc0
--- /dev/null
+++ b/.vscode/settings.json
@@ -0,0 +1,5 @@
+{
+ "yaml.schemas": {
+ "https://raw.githubusercontent.com/ansible/schemas/main/f/ansible.json#/$defs/tasks": "file:///home/ewimmer/workspace/datalab/ansible/collections/datalab.osranger/collections/ansible_collections/datalab/osrancher/roles/create_infrastructure/tasks/inventory.yml"
+ }
+}
\ No newline at end of file
diff --git a/MANIFEST.json b/MANIFEST.json
new file mode 100644
index 0000000000000000000000000000000000000000..93d9dc2a05e2a175fcf484c7d62fe0ebbee18947
--- /dev/null
+++ b/MANIFEST.json
@@ -0,0 +1,24 @@
+{
+ "collection_info": {
+ "namespace": "datalab",
+ "name": "osrancher",
+ "version": "1.0.0",
+ "authors": [
+ "Thomas Weber <thomas.weber@wu.ac.at>",
+ "Elias Wimmer <elias.wimmer@tuwien.ac.at>"
+ ],
+ "readme": "README.md",
+ "tags": [],
+ "description": "Setup a RKE2 Kubernetes Cluster on OpenStack",
+ "license": [
+ "GPL-2.0-or-later"
+ ],
+ "license_file": null,
+ "dependencies": {},
+ "repository": "https://gitlab.tuwien.ac.at/ADLS/infrastructure/adls.osrancher",
+ "documentation": "https://gitlab.tuwien.ac.at/ADLS/infrastructure/adls.osrancher",
+ "homepage": "https://gitlab.tuwien.ac.at/ADLS/infrastructure/adls.osrancher",
+ "issues": "http://example.com/issue/tracker"
+ },
+ "format": 1
+}
\ No newline at end of file
diff --git a/README.md b/README.md
index 0e9896b4acb7f89e28621b8a085efcec8c7c954c..b4e4d8b5111a10454d8b9d1dc30152ddd3903ca1 100644
--- a/README.md
+++ b/README.md
@@ -4,7 +4,7 @@ This collection contains roles for setup of a Kubernetes cluster with OpenStack
## Install
```
-ansible-galaxy collection install git+ssh://git@gitlab.tuwien.ac.at:822/ADLS/infrastructure/adls.osrancher.git
+ansible-galaxy collection install git+ssh://git@gitlab.tuwien.ac.at:822/datalab/ansible/collections/datalab.osrancher.git
```
## Usage
@@ -16,31 +16,29 @@ Use roles inside a Ansible playbook
vars:
cluster_name: test
roles:
- adls.osrancher.create_infrastructure
+ datalab.osrancher.create_infrastructure
# install and configure RKE2
- hosts: all
vars:
roles:
- adls.osrancher.rke2
+ datalab.osrancher.rke2
```
## Variables
-| Variable | Default | Description |
-| ------------------ | ------------------------------------ | ------------------------------------------------------------------------------------------- |
-| cluster_name | | Name of the RKE2 cluster |
-| server_count | 3 | Number of RKE2 worker VMs |
-| agent_count | 3 | Number of RKE2 server VMs |
-| server_flavor | m1a.large | Server VM flavor |
-| agent_flavor | m1a.xlarge | Worker VM flavor |
-| server_volume_size | 50 | Volume size (GB) for server VM |
-| agent_volume_size | 100 | Volume size (GB) for worker VM |
-| image | 1fe615f0-9dad-447d-bf54-9071defafb77 | ID for OpenStack VM image |
-| domain | | DNS-Entry for loadbalancer IP |
-| node_taints | | Node taints for RKE2 node |
-| node_labels | | Node labels for RKE2 node |
-| rke2_channel | stable | RKE3 version channel |
-| state | present | Flag for setup (`present`) or removing (`absent`) RKE3 cluster |
-| registry_mirrors | {} | [rke2-docs](https://docs.rke2.io/install/containerd_registry_configuration/#mirrors) |
-| manifests | {} | [defualts](collections/ansible_collections/adls/osrancher/roles/rke2/defaults/main.yml#L58) |
+| Variable | Default | Description |
+| ----------- | ----------- | ----------- |
+| cluster_name | | Name of the RKE2 cluster |
+| server_count | 3 | Number of RKE2 worker VMs |
+| agent_count | 3 | Number of RKE2 server VMs |
+| server_flavor | m1a.large | Server VM flavor |
+| agent_flavor | m1a.xlarge | Worker VM flavor |
+| server_volume_size | 50 | Volume size (GB) for server VM |
+| agent_volume_size | 100 | Volume size (GB) for worker VM |
+| image | 1fe615f0-9dad-447d-bf54-9071defafb77 | ID for OpenStack VM image |
+| domain | | DNS-Entry for loadbalancer IP |
+| node_taints | | Node taints for RKE2 node |
+| node_labels | | Node labels for RKE2 node |
+| rke2_channel | stable | RKE3 version channel |
+| state | present | Flag for setup (`present`) or removing (`absent`) RKE3 cluster |
diff --git a/roles/create_infrastructure/defaults/main.yml b/roles/create_infrastructure/defaults/main.yml
index d797c97c72e2f4cfdd1fd616f48e12153bea7ae1..aa247cb6f8ab11a80f8870a95aba7b09ac93c763 100644
--- a/roles/create_infrastructure/defaults/main.yml
+++ b/roles/create_infrastructure/defaults/main.yml
@@ -3,7 +3,7 @@ ssh_key_name: "rke2-{{ cluster_name }}-ssh-key"
ssh_key_file: "{{ ssh_keys_dir }}/{{ ssh_key_name }}.pem"
network_name: "rke2-{{ cluster_name }}"
subnet_name: "rke2-{{ cluster_name }}"
-cidr: 192.168.13.0/24
+cidr: 10.0.0.0/24
router_name: "rke2-{{ cluster_name }}"
server_volume_size: 50
diff --git a/roles/create_infrastructure/templates/agents.yml.j2 b/roles/create_infrastructure/templates/agents.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..ab719f07ce7a59f19d307272bc38bd5ccf3d71c2
--- /dev/null
+++ b/roles/create_infrastructure/templates/agents.yml.j2
@@ -0,0 +1,6 @@
+{% raw %}
+ansible_user: "ubuntu"
+ansible_ssh_private_key_file: "{{ ssh_key_file }}"
+ansible_ssh_common_args: "-o ProxyCommand='ssh -q ubuntu@{{ master_floating_ip }} -o StrictHostKeyChecking=no -i {{ ssh_key_file }} -W %h:%p' -o StrictHostKeyChecking=no"
+node_type: agent
+{% endraw %}
\ No newline at end of file
diff --git a/roles/create_infrastructure/templates/all.yml.j2 b/roles/create_infrastructure/templates/all.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..ce804d51e8080994961b1a1b533968d4a33c425c
--- /dev/null
+++ b/roles/create_infrastructure/templates/all.yml.j2
@@ -0,0 +1,7 @@
+loadbalancer_ip: {{ new_loadbalancer_ip }}
+
+master_floating_ip: {{ new_master_floating_ip }}
+
+subnet_id: {{ new_subnet_id }}
+
+ssh_key_file: {{ ssh_key_file }}
\ No newline at end of file
diff --git a/roles/create_infrastructure/templates/master.yml.j2 b/roles/create_infrastructure/templates/master.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..39c75b6d782e08911cdff64b46a2f2cf49b8c414
--- /dev/null
+++ b/roles/create_infrastructure/templates/master.yml.j2
@@ -0,0 +1,6 @@
+{% raw %}
+ansible_user: "ubuntu"
+ansible_ssh_private_key_file: "{{ ssh_key_file }}"
+ansible_ssh_common_args: "-o StrictHostKeyChecking=no"
+node_type: server
+{% endraw %}
\ No newline at end of file
diff --git a/roles/create_infrastructure/templates/servers.yml.j2 b/roles/create_infrastructure/templates/servers.yml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..562da525422fe35ca273ebb83cc4dff70d6fbbbb
--- /dev/null
+++ b/roles/create_infrastructure/templates/servers.yml.j2
@@ -0,0 +1,6 @@
+{% raw %}
+ansible_user: "ubuntu"
+ansible_ssh_private_key_file: "{{ ssh_key_file }}"
+ansible_ssh_common_args: "-o ProxyCommand='ssh -q ubuntu@{{ master_floating_ip }} -o StrictHostKeyChecking=no -i {{ ssh_key_file }} -W %h:%p' -o StrictHostKeyChecking=no"
+node_type: server
+{% endraw %}
\ No newline at end of file
diff --git a/roles/rke2/tasks/helm.yml b/roles/rke2/tasks/helm.yml
new file mode 100644
index 0000000000000000000000000000000000000000..7a63b155fa61721b7dd456a6eecb18417e782be7
--- /dev/null
+++ b/roles/rke2/tasks/helm.yml
@@ -0,0 +1,19 @@
+- name: copy helm template files
+ ansible.builtin.template:
+ src: 'helm/{{ item.template }}.j2'
+ dest: '/var/lib/rancher/rke2/server/manifests/{{ item.template }}.yaml'
+ loop:
+ - { template: 'deploy-openstack-ccm'}
+ - { template: 'deploy-openstack-cinder'}
+ # - { template: 'deploy-openstack-manila'}
+ - { template: 'deploy-cephfs'}
+ # - { template: 'deploy-nfs'}
+ # - { template: 'deploy-grafana'}
+ # - { template: 'deploy-cert-manager'}
+ # - { template: 'deploy-rancher-ui'}
+ # - { template: 'deploy-rke2-cilium'}
+ - { template: 'config-rke2-canal'}
+ - { template: 'config-nginx-ingress'}
+ # - { template: 'config-rke2-coredns'}
+ notify:
+ - restart rke2
\ No newline at end of file
diff --git a/roles/rke2/tasks/kubeconfig.yml b/roles/rke2/tasks/kubeconfig.yml
new file mode 100644
index 0000000000000000000000000000000000000000..5ff31ab483e4694ae361037fb0747e149cb14a5f
--- /dev/null
+++ b/roles/rke2/tasks/kubeconfig.yml
@@ -0,0 +1,17 @@
+- name: wait for kubeconfig
+ ansible.builtin.wait_for:
+ path: /etc/rancher/rke2/rke2.yaml
+
+- name: fetch kubeconfig from master
+ ansible.builtin.fetch:
+ src: /etc/rancher/rke2/rke2.yaml
+ dest: kubeconfig.yaml
+ flat: yes
+
+- name: replace endpoint in kubeconfig
+ delegate_to: localhost
+ become: no
+ ansible.builtin.replace:
+ path: kubeconfig.yaml
+ regexp: '^(\s+server: ).*'
+ replace: '\1https://{{ domain }}:6443'
diff --git a/roles/rke2/templates/all.yml b/roles/rke2/templates/all.yml
new file mode 100644
index 0000000000000000000000000000000000000000..7288a0061b07a6478a1a9d046782878acdd527d0
--- /dev/null
+++ b/roles/rke2/templates/all.yml
@@ -0,0 +1,3 @@
+token: {{ token }}
+
+server: "https://{{ domain }}:9345"
diff --git a/roles/rke2/templates/helm/config-nginx-ingress.j2 b/roles/rke2/templates/helm/config-nginx-ingress.j2
new file mode 100644
index 0000000000000000000000000000000000000000..571f68e234f014e1de8cf9f2b25b138b0c338bf8
--- /dev/null
+++ b/roles/rke2/templates/helm/config-nginx-ingress.j2
@@ -0,0 +1,13 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+ name: rke2-ingress-nginx
+ namespace: kube-system
+spec:
+ valuesContent: |-
+ controller:
+ config:
+ use-forwarded-headers: "true"
+ proxy-body-size: 64m
+ tolerations:
+ - effect: NoSchedule
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/config-rke2-canal.j2 b/roles/rke2/templates/helm/config-rke2-canal.j2
new file mode 100644
index 0000000000000000000000000000000000000000..0fade2f5523f0eb70843a0d925334204f786493a
--- /dev/null
+++ b/roles/rke2/templates/helm/config-rke2-canal.j2
@@ -0,0 +1,11 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+ name: rke2-canal
+ namespace: kube-system
+spec:
+ valuesContent: |-
+ calico:
+ vethuMTU: 1400
+ networkingBackend: "vxlan"
+ masquerade: false
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/config-rke2-coredns.j2 b/roles/rke2/templates/helm/config-rke2-coredns.j2
new file mode 100644
index 0000000000000000000000000000000000000000..c6b8a671dc5ed1de854f2e60201536b32ecee12c
--- /dev/null
+++ b/roles/rke2/templates/helm/config-rke2-coredns.j2
@@ -0,0 +1,9 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChartConfig
+metadata:
+ name: rke2-coredns
+ namespace: kube-system
+spec:
+ valuesContent: |-
+ nodeSelector:
+ role: "agent"
diff --git a/roles/rke2/templates/helm/deploy-cephfs.j2 b/roles/rke2/templates/helm/deploy-cephfs.j2
new file mode 100644
index 0000000000000000000000000000000000000000..be935457cd4c327146dc0471c797b7a809e6880b
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-cephfs.j2
@@ -0,0 +1,18 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: cephfs-csi
+ namespace: kube-system
+spec:
+ repo: https://ceph.github.io/csi-charts
+ chart: ceph-csi-cephfs
+ version: {{ ceph_csi_version}}
+ valuesContent: |-
+ tolerations:
+ - effect: NoSchedule
+ csiConfig:
+ - clusterID: "{{ ceph_cluster_id }}"
+ monitors:
+{% for mon in ceph_monitors %}
+ - {{ mon }}
+{% endfor %}
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/deploy-cert-manager.j2 b/roles/rke2/templates/helm/deploy-cert-manager.j2
new file mode 100644
index 0000000000000000000000000000000000000000..25b4576f77e355487a99603fdb523616f79599b7
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-cert-manager.j2
@@ -0,0 +1,12 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: cert-manager
+ namespace: kube-system
+spec:
+ repo: https://charts.jetstack.io
+ chart: cert-manager
+ #targetNamespace: cert-manager
+ version: v1.3.1
+ set:
+ installCRDs: "true"
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/deploy-grafana.j2 b/roles/rke2/templates/helm/deploy-grafana.j2
new file mode 100644
index 0000000000000000000000000000000000000000..97f1d80dc91697bda4fdd05d92ee1980e0bf0280
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-grafana.j2
@@ -0,0 +1,19 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: grafana
+ namespace: kube-system
+spec:
+ chart: stable/grafana
+ #targetNamespace: monitoring
+ set:
+ adminPassword: "{{ grafana_password }}"
+ valuesContent: |-
+ image:
+ tag: master
+ env:
+ GF_EXPLORE_ENABLED: true
+ adminUser: admin
+ sidecar:
+ datasources:
+ enabled: true
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/deploy-openstack-ccm.j2 b/roles/rke2/templates/helm/deploy-openstack-ccm.j2
new file mode 100644
index 0000000000000000000000000000000000000000..709c1e841d9bb950bf74c74e4b0f827dc5c9decc
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-openstack-ccm.j2
@@ -0,0 +1,54 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: openstack-ccm
+ namespace: kube-system
+spec:
+ repo: https://kubernetes.github.io/cloud-provider-openstack
+ chart: openstack-cloud-controller-manager
+ version: {{ openstack_ccm_version }}
+ bootstrap: True
+ valuesContent: |-
+ cloudConfig:
+ global:
+ auth-url: {{ openstack_auth.auth_url }}
+ application-credential-id: {{ openstack_auth.application_credential_id }}
+ application-credential-secret: {{ openstack_auth.application_credential_secret }}
+ region: {{ openstack_region_name }}
+ loadBalancer:
+ subnet-id: {{ subnet_id }}
+ floating-network-id: {{ floating_network_id }}
+
+{% if router_id is defined %}
+ route:
+ router-id: {{ router_id }}
+{% endif %}
+
+ tolerations:
+ - key: node.cloudprovider.kubernetes.io/uninitialized
+ value: "true"
+ effect: NoSchedule
+ - key: node-role.kubernetes.io/master
+ value: "true"
+ effect: NoSchedule
+ - key: CriticalAddonsOnly
+ value: "true"
+ effect: NoExecute
+
+ nodeSelector:
+ node-role.kubernetes.io/control-plane: "true"
+
+ # serviceMonitor:
+ # enabled: "true"
+
+ {# livenessProbe:
+ httpGet:
+ path: /metrics
+ port: 10258 #}
+
+ controllerExtraArgs: |-
+ - --cluster-name=rke2-{{ cluster_name }}
+
+ resources:
+ requests:
+ cpu: 200m
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/deploy-openstack-cinder.j2 b/roles/rke2/templates/helm/deploy-openstack-cinder.j2
new file mode 100644
index 0000000000000000000000000000000000000000..c7c45a34f8c60a9cb26f993b5a671d149ccf35d2
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-openstack-cinder.j2
@@ -0,0 +1,25 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: openstack-cinder-csi
+ namespace: kube-system
+spec:
+ repo: https://kubernetes.github.io/cloud-provider-openstack
+ chart: openstack-cinder-csi
+ version: {{ cinder_csi_version }}
+ valuesContent: |-
+ secret:
+ enabled: true
+ create: true
+ name: cinder-csi-cloud-config
+ data:
+ cloud.conf: |-
+ [Global]
+ auth-url={{ openstack_auth.auth_url }}
+ application-credential-id={{ openstack_auth.application_credential_id }}
+ application-credential-secret={{ openstack_auth.application_credential_secret }}
+ region={{ openstack_region_name }}
+
+ storageClass:
+ delete:
+ isDefault: true
diff --git a/roles/rke2/templates/helm/deploy-rancher-ui.j2 b/roles/rke2/templates/helm/deploy-rancher-ui.j2
new file mode 100644
index 0000000000000000000000000000000000000000..c36482cacf46ecf5697c8e10f8fc7e02b2bbcbf9
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-rancher-ui.j2
@@ -0,0 +1,14 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: rancher
+ namespace: kube-system
+spec:
+ repo: https://releases.rancher.com/server-charts/latest
+ chart: rancher
+ version: 2.5.9-rc2
+ #targetNamespace: cattle-system
+ set:
+ hostname: "{{rancher_ui_dns}}"
+ letsEncrypt.email: "{{letsEncrypt_admin_mail}}"
+ ingress.tls.source: "letsEncrypt"
\ No newline at end of file
diff --git a/roles/rke2/templates/helm/deploy-rke2-cilium.j2 b/roles/rke2/templates/helm/deploy-rke2-cilium.j2
new file mode 100644
index 0000000000000000000000000000000000000000..aa6dea3b6df997a2ecb81c31197c7e71ae964c4a
--- /dev/null
+++ b/roles/rke2/templates/helm/deploy-rke2-cilium.j2
@@ -0,0 +1,11 @@
+apiVersion: helm.cattle.io/v1
+kind: HelmChart
+metadata:
+ name: rke2-cilium
+ namespace: kube-system
+spec:
+ repo: https://rke2-charts.rancher.io
+ chart: rke2-cilium
+ bootstrap: true
+ valuesContent: |-
+ cilium: {}
\ No newline at end of file
diff --git a/roles/rke2/templates/registries.yaml.j2 b/roles/rke2/templates/registries.yaml.j2
new file mode 100644
index 0000000000000000000000000000000000000000..ee263fc31b14d8f3294dc92a81def873ec142ab1
--- /dev/null
+++ b/roles/rke2/templates/registries.yaml.j2
@@ -0,0 +1,4 @@
+mirrors:
+ docker.io:
+ endpoint:
+ - "s210.dl.hpc.tuwien.ac.at/cache"
\ No newline at end of file