From 37c83f1af7d8e4db4c81d0fd6cbbd709d5b1d5d1 Mon Sep 17 00:00:00 2001
From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at>
Date: Mon, 13 Jan 2025 18:41:26 +0100
Subject: [PATCH 1/5] =?UTF-8?q?=F0=9F=93=82=20Global:=20restructure=20file?=
 =?UTF-8?q?s?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* docker related files moved under `docker`
* remove obsolete version tag in docker compose files
---
 docker-compose.base.yml                              | 12 +++++-------
 docker-compose.yml                                   |  7 +++----
 .../dashboards}/config/opensearch_dashboards.yml     |  0
 {metricbeat => docker/metricbeat}/metricbeat.yml     |  0
 {opensearch => docker/opensearch}/init-security.sh   |  0
 {opensearch => docker/opensearch}/opensearch.yml     |  0
 .../opensearch}/security/internal_users.template.yml |  0
 .../opensearch}/security/tenants.yml                 |  0
 8 files changed, 8 insertions(+), 11 deletions(-)
 rename {dashboards => docker/dashboards}/config/opensearch_dashboards.yml (100%)
 rename {metricbeat => docker/metricbeat}/metricbeat.yml (100%)
 rename {opensearch => docker/opensearch}/init-security.sh (100%)
 rename {opensearch => docker/opensearch}/opensearch.yml (100%)
 rename {opensearch => docker/opensearch}/security/internal_users.template.yml (100%)
 rename {opensearch => docker/opensearch}/security/tenants.yml (100%)

diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index 7c5f901..d8cee61 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   os-node:
     image: opensearchproject/opensearch:2
@@ -25,10 +23,11 @@ services:
       - os-node-1:/usr/share/opensearch/data
       # NOTE: We map every file separately so that we don't overwrite the files created
       # in the same directory from opensearch.
-      - ./opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/internal_users.template.yml:ro
-      - ./opensearch/security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml:ro
-      - ./opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro
-      - ./opensearch/init-security.sh:/usr/share/opensearch/init-security.sh:ro
+      - ./docker/opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/internal_users.template.yml:ro
+      - ./docker/opensearch/security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
+      - ./docker/opensearch/security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml:ro
+      - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro
+      - ./docker/opensearch/init-security.sh:/usr/share/opensearch/init-security.sh:ro
       # internal ssl files
       - ./ssl/node1-crt.pem:/usr/share/opensearch/config/node-crt.pem:ro
       - ./ssl/node1-key.pem:/usr/share/opensearch/config/node-key.pem:ro
@@ -38,4 +37,3 @@ services:
       # ssl for external communication
       - ./ssl/cluster-crt.pem:/usr/share/opensearch/config/cluster-crt.pem:ro
       - ./ssl/cluster-key.pem:/usr/share/opensearch/config/cluster-key.pem:ro
-
diff --git a/docker-compose.yml b/docker-compose.yml
index 94c8cab..1b0d09e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,3 @@
-version: '3'
-
 services:
   os-node-1:
     extends:
@@ -33,7 +31,7 @@ services:
     ports:
       - ${DASHBOARDS_PORT:-443}:5601
     volumes:
-      - ./dashboards/config/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro
+      - ./docker/dashboards/config/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro
       - ./ssl/dashboards-crt.pem:/usr/share/opensearch-dashboards/config/crt.pem:ro
       - ./ssl/dashboards-key.pem:/usr/share/opensearch-dashboards/config/key.pem:ro
       - ./ssl/root-ca-crt.pem:/usr/share/opensearch-dashboards/config/root-ca.pem:ro
@@ -58,7 +56,7 @@ services:
       - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro
       - /proc:/hostfs/proc:ro
       - /:/hostfs:ro
-      - ./metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
+      - ./docker/metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro
     depends_on:
       - os-node-1
 
@@ -67,5 +65,6 @@ volumes:
   os-node-2:
   metricbeat-data:
 
+
 networks:
   default:
diff --git a/dashboards/config/opensearch_dashboards.yml b/docker/dashboards/config/opensearch_dashboards.yml
similarity index 100%
rename from dashboards/config/opensearch_dashboards.yml
rename to docker/dashboards/config/opensearch_dashboards.yml
diff --git a/metricbeat/metricbeat.yml b/docker/metricbeat/metricbeat.yml
similarity index 100%
rename from metricbeat/metricbeat.yml
rename to docker/metricbeat/metricbeat.yml
diff --git a/opensearch/init-security.sh b/docker/opensearch/init-security.sh
similarity index 100%
rename from opensearch/init-security.sh
rename to docker/opensearch/init-security.sh
diff --git a/opensearch/opensearch.yml b/docker/opensearch/opensearch.yml
similarity index 100%
rename from opensearch/opensearch.yml
rename to docker/opensearch/opensearch.yml
diff --git a/opensearch/security/internal_users.template.yml b/docker/opensearch/security/internal_users.template.yml
similarity index 100%
rename from opensearch/security/internal_users.template.yml
rename to docker/opensearch/security/internal_users.template.yml
diff --git a/opensearch/security/tenants.yml b/docker/opensearch/security/tenants.yml
similarity index 100%
rename from opensearch/security/tenants.yml
rename to docker/opensearch/security/tenants.yml
-- 
GitLab


From 9b1450be28cb9eda671b022ae27f0b04703c3920 Mon Sep 17 00:00:00 2001
From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at>
Date: Fri, 17 Jan 2025 13:54:46 +0100
Subject: [PATCH 2/5] =?UTF-8?q?=F0=9F=9A=80=20CI/CD:=20create=20pipeline?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

* deployment to staging and production environments
* set pull policy to always for services, as major versions are pinned
---
 .gitlab-ci.yml          | 26 +++++++++++++++++++
 .gitlab-ci/deploy.sh    | 57 +++++++++++++++++++++++++++++++++++++++++
 docker-compose.base.yml |  1 +
 docker-compose.yml      |  1 +
 scripts/dump-vars.sh    | 28 ++++++++++++++++++++
 5 files changed, 113 insertions(+)
 create mode 100644 .gitlab-ci.yml
 create mode 100755 .gitlab-ci/deploy.sh
 create mode 100755 scripts/dump-vars.sh

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..b656386
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,26 @@
+# GitLab CI/CD configuration
+
+stages:
+  - staging
+  - production
+
+deploy-staging:
+  stage: staging
+  environment:
+    name: staging
+    url: ${DEPLOYMENT_HOST}
+  script:
+    - ./.gitlab-ci/deploy.sh
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+
+deploy-production:
+  stage: production
+  environment:
+    name: production
+    url: ${DEPLOYMENT_HOST}
+  script:
+    - ./.gitlab-ci/deploy.sh
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "main"'
+      when: manual
diff --git a/.gitlab-ci/deploy.sh b/.gitlab-ci/deploy.sh
new file mode 100755
index 0000000..e18ef3c
--- /dev/null
+++ b/.gitlab-ci/deploy.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+#
+# Deployment script for CRDM Logs.
+# Intended for execution by the GitLab Runner.
+
+set -euo pipefail
+
+# Ensure required environment variables are set.
+# DEPLOYMENT_HOST denotes the SSH TARGET. Can be only ONE IP or DNS name.
+if [ -z "$DEPLOYMENT_HOST" ]; then
+    echo "[ERROR] Make sure DEPLOYMENT_HOST is set." >&2
+    exit 1
+fi
+
+# ----------------------- #
+#    Repository checks    #
+# ----------------------- #
+DEPLOYMENT_PATH="${DEPLOYMENT_PATH:-crdm-logging-setup}"
+SSH_TARGET="${DEPLOYMENT_USER:-logs}@${DEPLOYMENT_HOST}"
+
+# SSH to target, initialize repo if it doesn't exist.
+# Otherwise, tear down old deployment and pull changes.
+ssh "${SSH_TARGET}" "bash" <<- EOF
+set -eu pipefail
+mkdir -p "${DEPLOYMENT_PATH}" && cd "${DEPLOYMENT_PATH}"
+
+if git status > /dev/null 2>&1; then
+    echo "[INFO] Pulling origin."
+    git pull "${CI_REPOSITORY_URL}"
+else
+    echo "[INFO] Git repository doesn't exist, cloning and setting up."
+    git clone "${CI_REPOSITORY_URL}" . > /dev/null 2>&1
+fi
+
+echo -n "[INFO] " && git checkout "${CI_COMMIT_BRANCH}"
+EOF
+
+# ----------------------- #
+#    .env file checks     #
+# ----------------------- #
+# Dump variables and transfer the file.
+echo "[INFO] Dumping variables to .env file."
+./scripts/dump-vars.sh ".env"
+scp ".env" "${SSH_TARGET}:${DEPLOYMENT_PATH}/.env"
+
+# SSH again, check if ssl directory is empty: setup or rerun.
+ssh "${SSH_TARGET}" "bash" <<- EOF
+set -eu pipefail
+cd "${DEPLOYMENT_PATH}"
+if [ $(find "./ssl" -type f ! -name '.gitkeep' | wc -l) -eq 0 ]; then
+    echo "[INFO] SSL directory only contains .gitkeep. Running setup.sh to create certificates."
+    ./scripts/setup.sh
+else
+    echo "[INFO] SSL directory is not empty. Skipping certificate setup."
+    docker compose up -d
+fi
+EOF
diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index d8cee61..9bf5d4e 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -1,6 +1,7 @@
 services:
   os-node:
     image: opensearchproject/opensearch:2
+    pull_policy: "always"
     restart: "unless-stopped"
     environment:
       - DISABLE_INSTALL_DEMO_CONFIG=true
diff --git a/docker-compose.yml b/docker-compose.yml
index 1b0d09e..b5fbcbc 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -21,6 +21,7 @@ services:
 
   os-dashboards:
     image: opensearchproject/opensearch-dashboards:2
+    pull_policy: "always"
     restart: "unless-stopped"
     environment:
       # note: the dashboards entrypoint performs translations of naming for the env vars
diff --git a/scripts/dump-vars.sh b/scripts/dump-vars.sh
new file mode 100755
index 0000000..fb3aaba
--- /dev/null
+++ b/scripts/dump-vars.sh
@@ -0,0 +1,28 @@
+#!/bin/sh
+#
+# Dump environment variables to a file.
+#
+# This script is intended to be run by the Gitlab Runner, to dump all Gitlab
+# variables to the secrets file.
+#
+
+# The user can optionally specify the output file
+OUTPUT_FILE="${1:-.env}"
+
+# Delete the old file if it exists, and create one with secure permissions
+rm -f "${OUTPUT_FILE}"
+
+# Set 'rwx' only for user
+umask 077
+touch "${OUTPUT_FILE}"
+
+# Dump each exported variable
+# According to the current setup:
+# DASHBOARDS_ prefix for variables relevant to OpenSearch Dashboards.
+# METRICBEAT_ prefix for variables relevant to Metricbeat.
+# OPENSEARCH_ prefix for variables relevant to OpenSearch nodes.
+while read -r var; do
+    echo $var >> "${OUTPUT_FILE}"
+done <<- EOF
+    $(env | grep -E "^DASHBOARDS_.*|^METRICBEAT_.*|^OPENSEARCH_.*")
+EOF
-- 
GitLab


From c47ad38260ef457f580671b8c351e670b5783de3 Mon Sep 17 00:00:00 2001
From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at>
Date: Mon, 20 Jan 2025 15:22:21 +0100
Subject: [PATCH 3/5] =?UTF-8?q?=E2=9A=96=EF=B8=8F=20Update=20License=20inf?=
 =?UTF-8?q?ormation?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 LICENSE | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/LICENSE b/LICENSE
index bfc4758..6b4a86f 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,6 +1,6 @@
 MIT License
 
-Copyright (c) 2022 TU Wien
+Copyright (c) 2022-2025 TU Wien
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
-- 
GitLab


From c554226227b71737d33d12370ada1eb2997c51a2 Mon Sep 17 00:00:00 2001
From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at>
Date: Thu, 23 Jan 2025 21:05:40 +0100
Subject: [PATCH 4/5] =?UTF-8?q?=F0=9F=9B=A0=EF=B8=8F=20Services:=20add=20C?=
 =?UTF-8?q?ertbot=20with=20auto=20SSL=20renewal?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 .gitignore              |   3 +
 docker-compose.yml      |  18 +++++
 scripts/dump-vars.sh    |   2 +-
 scripts/generate-ssl.sh | 170 ++++++++++++++++++++++++++++++++++++----
 4 files changed, 175 insertions(+), 18 deletions(-)

diff --git a/.gitignore b/.gitignore
index f4fa38d..0051758 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,6 @@
 # ssl keys and certificates
 ssl/*
 !ssl/.gitkeep
+
+# Certbot files
+docker/certbot/*
diff --git a/docker-compose.yml b/docker-compose.yml
index b5fbcbc..3799671 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -61,6 +61,24 @@ services:
     depends_on:
       - os-node-1
 
+  certbot:
+    image: certbot/certbot
+    restart: "unless-stopped"
+    ports:
+      - 80:80
+    volumes:
+      - ./docker/certbot/conf:/etc/letsencrypt
+      - ./docker/certbot/www:/var/www/certbot
+      - ./ssl:/ssl
+    entrypoint: >
+      /bin/sh -c '
+        trap exit TERM;
+        while :; do
+          certbot renew --standalone --noninteractive --preferred-challenges http-01 --http-01-port 80;
+          sleep ${CERTBOT_SLEEP_INTERVAL:-12h} & wait $${!};
+        done;
+      '
+
 volumes:
   os-node-1:
   os-node-2:
diff --git a/scripts/dump-vars.sh b/scripts/dump-vars.sh
index fb3aaba..d54d0f0 100755
--- a/scripts/dump-vars.sh
+++ b/scripts/dump-vars.sh
@@ -24,5 +24,5 @@ touch "${OUTPUT_FILE}"
 while read -r var; do
     echo $var >> "${OUTPUT_FILE}"
 done <<- EOF
-    $(env | grep -E "^DASHBOARDS_.*|^METRICBEAT_.*|^OPENSEARCH_.*")
+    $(env | grep -E "^CERTBOT_.*|^DASHBOARDS_.*|^METRICBEAT_.*|^OPENSEARCH_.*")
 EOF
diff --git a/scripts/generate-ssl.sh b/scripts/generate-ssl.sh
index 1e14f06..18edf44 100755
--- a/scripts/generate-ssl.sh
+++ b/scripts/generate-ssl.sh
@@ -1,17 +1,44 @@
 #!/bin/bash
 #
-# script for generating SSL keys and certificates
+# Script for generating SSL keys and certificates.
+#
 
 set -euo pipefail
 
+# The guard below ensures, in most cases, that the script is executed from the correct directory (project directory).
+# So it should be safe to assume that we can extract the path from the parent of the 'scripts' directory.
+PROJECT_DIR="$(dirname "$(dirname "$(realpath "$0")")")"
+
+# ---------------- #
+# Execution checks #
+# ---------------- #
 if [[ -d "ssl" ]]; then
     cd "ssl"
 else
-    echo >&2 "error: this script needs to be executed from the project directory!"
+    echo >&2 "[ERROR] This script needs to be executed from the project directory!"
     exit 1
 fi
 
-# common certificate details
+# --------------------- #
+# Environment variables #
+# --------------------- #
+ENV_FILE="${PROJECT_DIR}/.env"
+
+# Check if the .env file exists and source it
+if [[ -f "${ENV_FILE}" ]]; then
+    echo "[INFO] Detected .env file, sourcing it."
+    source "${ENV_FILE}"
+fi
+
+# Set hostnames for Dashboards and Opensearch nodes.
+# They are determining whether the setup is in a local or live environment.
+DASHBOARDS_HOSTNAME="${DASHBOARDS_HOSTNAME:-localhost}"
+OPENSEARCH_HOSTNAME="${OPENSEARCH_HOSTNAME:-localhost}"
+
+# --------------------- #
+# Utilities definitions #
+# --------------------- #
+# Common certificate details
 organization="TU Wien"
 org_unit="Center for Research Data Management"
 country="AT"
@@ -19,35 +46,35 @@ state="Vienna"
 locality="Vienna"
 
 function generate_certificate {
-    # generate keypair
+    # Generate keypair
     local cert_name="${1}"
     local common_name="${2}"
 
-    # generate the ssl files
+    # Generate the ssl files
     subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=${common_name}"
     openssl genrsa -out "${cert_name}-key-temp.pem" 2048
     openssl pkcs8 -inform PEM -outform PEM -in "${cert_name}-key-temp.pem" -topk8 -nocrypt -v1 PBE-SHA1-3DES -out "${cert_name}-key.pem"
     openssl req -new -key "${cert_name}-key.pem" -subj "${subject}" -out "${cert_name}.csr"
     openssl x509 -req -in "${cert_name}.csr" -CA root-ca-crt.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out "${cert_name}-crt.pem" -days 365
 
-    # clean up temporary files
+    # Clean up temporary files
     rm "${cert_name}-key-temp.pem" "${cert_name}.csr"
 }
 
 function generate_if_not_user_defined {
-    # generate ssl keypair if it wasn't explicitly set by an operator
+    # Generate ssl keypair if it wasn't explicitly set by an operator
     local cert_name="${1}"
     local common_name="${2}"
     local key_file="${cert_name}-key.pem"
     local cert_file="${cert_name}-crt.pem"
 
     if [[ -f "${key_file}" && ! -L "${key_file}" && -f "${cert_file}" && ! -L "${cert_file}" ]]; then
-        # if both files exist, we assume they were placed here by the operator
-        echo "info: keeping keypair for ${cert_name}."
+        # If both files exist, we assume they were placed here by the operator
+        echo "[INFO] Keeping keypair for ${cert_name}."
 
     elif [[ ! -e "${key_file}" || -L "${key_file}" || ! -e "${cert_file}" || -L "${cert_file}" ]]; then
-        # if the files are symlinks, we assume they were auto-generated by the script
-        # similar if they don't exist
+        # If the files are symlinks, we assume they were auto-generated by the script.
+        # Similarly if they don't exist.
         unlink "${key_file}" &>/dev/null || true
         unlink "${cert_file}" &>/dev/null || true
         generate_certificate "${cert_name}-gen" "${common_name}"
@@ -55,13 +82,52 @@ function generate_if_not_user_defined {
         ln -s "${cert_name}-gen-crt.pem" "${cert_file}"
 
     else
-        # maybe the files got turned into directories by the bind mounts?
-        echo "info: something weird is going on with the files ${cert_name}-{key,crt}.pem"
+        # Maybe the files got turned into directories by the bind mounts?
+        echo "[INFO] Something weird is going on with the files ${cert_name}-{key,crt}.pem"
         echo ">     please investigate manually."
     fi
 }
 
+function is_local_setup {
+    local hostname="${1}"
+    [[ "${hostname}" == "localhost" ]]
+}
+
+function link_certbot_certificates {
+    # Symlink Certbot certificates
+    local cert_name="${1}"
+    local hostname="${2}"
+
+    # Spin up again certbot container and change directory permissions to make certificates readable.
+    # Symlinks won't work otherwise; 'live' directory has 700 permissions with root ownership.
+    docker compose up -d certbot
+    eval "docker compose exec -u root certbot sh -c \"
+        chmod 0755 /etc/letsencrypt/live /etc/letsencrypt/archive &&
+        chmod -R +r /etc/letsencrypt/archive/* /etc/letsencrypt/live/*
+    \""
+
+    if [[ $? -ne 0 ]]; then
+        echo "[ERROR] Failed to adjust permissions for Let's Encrypt files."
+        exit 1
+    fi
+    echo "[INFO] Permissions adjusted for Let's Encrypt directories."
+
+
+    echo "[INFO] Linking Certbot certificates for ${hostname} to ${cert_name}."
+    local ssl_path="${PROJECT_DIR}/docker/certbot/conf/live/${hostname}"
+
+    if [[ -d "${ssl_path}" ]]; then
+        ln -sf "${ssl_path}/privkey.pem" "${cert_name}-key.pem"
+        ln -sf "${ssl_path}/fullchain.pem" "${cert_name}-crt.pem"
+    else
+        echo "[ERROR] Certbot certificates for ${hostname} not found. Ensure Certbot has generated them."
+        exit 1
+    fi
+}
 
+# ------------ #
+# Run sequence #
+# ------------ #
 # Root CA
 subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=ROOT" 
 openssl genrsa -out root-ca-key.pem 2048
@@ -76,8 +142,78 @@ generate_certificate "node1" "os-node-1"
 # Node 2 cert (for internal communication)
 generate_certificate "node2" "os-node-2"
 
-# Dashboards cert (for outside communication)
-generate_if_not_user_defined "dashboards" "${OPENSEARCH_HOSTNAME:-localhost}"
+# Determine external SSL certificates generation
+if is_local_setup "${DASHBOARDS_HOSTNAME}" && is_local_setup "${OPENSEARCH_HOSTNAME}"; then
+    # REST HTTPS cert (for outside communication)
+    generate_if_not_user_defined "cluster" "${OPENSEARCH_HOSTNAME}"
+
+    echo "[INFO] Detected local setup, generating local certificates for Dashboards and Cluster."
+    # Dashboards cert (for outside communication)
+    generate_if_not_user_defined "dashboards" "${DASHBOARDS_HOSTNAME}"
+
+else
+    echo "[INFO] Detected live setup, generating Let's Encrypt certificates for Dashboards and Cluster."
+    CERTBOT_EMAIL="${CERTBOT_EMAIL:-}"
+    CERTBOT_RSA_KEY_SIZE="${CERTBOT_RSA_KEY_SIZE:-4096}"
+    CERTBOT_STAGING="${CERTBOT_STAGING:-}"
+
+    # Select appropriate email argument.
+    if [[ -z "${CERTBOT_EMAIL}" ]]; then
+        email_arg="--register-unsafely-without-email"
+    else
+        email_arg="--email ${CERTBOT_EMAIL}"
+    fi
+
+    # Determine certbot staging argument.
+    if [[ -z "${CERTBOT_STAGING}" ]] || [[ ! "${CERTBOT_STAGING}" =~ ^[01]$ ]]; then
+        echo "[WARN] Invalid or unset value for 'CERTBOT_STAGING'. Setting it to 1 (testing mode)."
+        CERTBOT_STAGING="1"
+    fi
+    if [[ "${CERTBOT_STAGING}" != "0" ]]; then staging_arg="--staging"; fi
+
+    # Define common certbot command
+    certbot_cmd="certbot certonly ${staging_arg:-} ${email_arg} --rsa-key-size ${CERTBOT_RSA_KEY_SIZE} --standalone --agree-tos --force-renewal"
+
+    # Check if Dashboards and Cluster point to the same hostname.
+    if [[ "${DASHBOARDS_HOSTNAME}" == "${OPENSEARCH_HOSTNAME}" ]]; then
+        echo "[INFO] Hostnames for Dashboards and Cluster are the same. Generating a single certificate for both."
+        LIVE_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${DASHBOARDS_HOSTNAME}/fullchain.pem"
+
+        if [[ -L "${LIVE_CRT_PATH}" ]]; then
+            echo "[WARN] Certificates for ${DASHBOARDS_HOSTNAME} already exist. Skipping generation."
+
+        else
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${DASHBOARDS_HOSTNAME}\" certbot"
+            # Symlink Certbot certificates
+            link_certbot_certificates "dashboards" "${DASHBOARDS_HOSTNAME}"
+            link_certbot_certificates "cluster" "${OPENSEARCH_HOSTNAME}"
+        fi
+
+    else
+        echo "[INFO] Hostnames for are different. Generating separate certificates for each."
+        LIVE_DASHBOARDS_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${DASHBOARDS_HOSTNAME}/fullchain.pem"
+        LIVE_OPENSEARCH_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${OPENSEARCH_HOSTNAME}/fullchain.pem"
+
+        # Attempt to generate certificate for DASHBOARDS_HOSTNAME.
+        if [[ -L "${LIVE_DASHBOARDS_CRT_PATH}" ]]; then
+            echo "[WARN] Certificates for ${DASHBOARDS_HOSTNAME} already exist. Skipping generation."
+
+        else
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${DASHBOARDS_HOSTNAME}\" certbot"
+            link_certbot_certificates "dashboards" "${DASHBOARDS_HOSTNAME}"
+        fi
+
+        # Attempt to generate certificate for OPENSEARCH_HOSTNAME.
+        if [[ -L "${LIVE_OPENSEARCH_CRT_PATH}" ]]; then
+            echo "[WARN] Certificates for ${OPENSEARCH_HOSTNAME} already exist. Skipping generation."
+
+        else
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${OPENSEARCH_HOSTNAME}\" certbot"
+            link_certbot_certificates "cluster" "${OPENSEARCH_HOSTNAME}"
+        fi
+    fi
+fi
 
-# REST HTTPS cert (for outside communication)
-generate_if_not_user_defined "cluster" "${DASHBOARDS_HOSTNAME:-localhost}"
+# Ensure read permissions to all files in the 'ssl' directory because 'opensearch' user must be able to read them in the container.
+# This is especially important for the *-key.pem files (private keys).
+chmod -R +r "${PROJECT_DIR}/ssl"
-- 
GitLab


From 732019ed05852a8a3a07d4b9f2e1e9bc67cd5814 Mon Sep 17 00:00:00 2001
From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at>
Date: Thu, 6 Feb 2025 19:05:03 +0100
Subject: [PATCH 5/5] =?UTF-8?q?=F0=9F=94=A7=20Setup:=20adjust=20for=20Open?=
 =?UTF-8?q?Search=20v2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker-compose.base.yml                       |  12 +-
 docker-compose.yml                            |   8 +-
 docker/opensearch/init-security.sh            |  76 +++++++-----
 docker/opensearch/opensearch.yml              |   1 +
 .../security/internal_users.template.yml      |  15 ++-
 docker/opensearch/security/tenants.yml        |   1 -
 scripts/generate-ssl.sh                       | 108 +++++++++++-------
 7 files changed, 136 insertions(+), 85 deletions(-)

diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index 9bf5d4e..9fc7407 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -21,14 +21,14 @@ services:
         soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
         hard: 65536
     volumes:
-      - os-node-1:/usr/share/opensearch/data
-      # NOTE: We map every file separately so that we don't overwrite the files created
-      # in the same directory from opensearch.
-      - ./docker/opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/internal_users.template.yml:ro
-      - ./docker/opensearch/security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
-      - ./docker/opensearch/security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml:ro
+      - os-node-1-config:/usr/share/opensearch/config
+      - os-node-1-data:/usr/share/opensearch/data
       - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro
       - ./docker/opensearch/init-security.sh:/usr/share/opensearch/init-security.sh:ro
+      # NOTE: In this section we map every configuration file separately so that
+      # we don't overwrite the files created in the same directory from OpenSearch.
+      - ./docker/opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/opensearch-security/internal_users.template.yml
+      - ./docker/opensearch/security/tenants.yml:/usr/share/opensearch/config/opensearch-security/tenants.yml
       # internal ssl files
       - ./ssl/node1-crt.pem:/usr/share/opensearch/config/node-crt.pem:ro
       - ./ssl/node1-key.pem:/usr/share/opensearch/config/node-key.pem:ro
diff --git a/docker-compose.yml b/docker-compose.yml
index 3799671..e3132a8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,7 +44,10 @@ services:
     # Beats versions newer than 7.12.x are not supported by OpenSearch
     image: docker.elastic.co/beats/metricbeat-oss:7.12.1
     restart: "unless-stopped"
-    user: root
+    user: ${METRICBEAT_CONTAINER_UID:-1000}
+    # add docker gid from the host so that docker.sock can be accessed
+    group_add:
+      - ${METRICBEAT_DOCKER_GID:-999}
     environment:
       - METRICBEAT_INDEX_INFIX=${METRICBEAT_INDEX_INFIX:-dev}
       - METRICBEAT_INTERVAL=${METRICBEAT_INTERVAL:-10s}
@@ -80,7 +83,8 @@ services:
       '
 
 volumes:
-  os-node-1:
+  os-node-1-config:
+  os-node-1-data:
   os-node-2:
   metricbeat-data:
 
diff --git a/docker/opensearch/init-security.sh b/docker/opensearch/init-security.sh
index 987542f..4fb551f 100755
--- a/docker/opensearch/init-security.sh
+++ b/docker/opensearch/init-security.sh
@@ -1,42 +1,64 @@
 #!/bin/bash
 #
-# update the internal_users via 'securityadmin.sh' to set their passwords
-# this should only be called on a new cluster
+# Update the internal_users via 'securityadmin.sh' to set their passwords.
+# This should only be called on a new cluster.
+#
+# Usage: docker compose run os-node-1 ./init-security.sh
 #
-# usage: docker compose run os-node-1 ./init-security.sh
 
 set -euo pipefail
 
+# Define common paths in the opensearch docker container.
+security_config_path="./config/opensearch-security"
+security_tools_path="./plugins/opensearch-security/tools"
 
-# get the passwords from the env vars, and hash them
-sec_plugin_path="./plugins/opensearch-security"
+# Get the passwords from the env vars.
 admin_pw="${OPENSEARCH_ADMIN_PASSWORD:?admin password required}"
 kibanaserver_pw="${OPENSEARCH_KIBANASERVER_PASSWORD:?kibana server password required}"
 logginguser_pw="${OPENSEARCH_LOGGINGUSER_PASSWORD:?logging user password required}"
 
-admin_hash=$(${sec_plugin_path}/tools/hash.sh -p "${admin_pw}" | tr " " "\n" | tail -n 1)
-dashboards_hash=$(${sec_plugin_path}/tools/hash.sh -p "${kibanaserver_pw}" | tr " " "\n" | tail -n 1)
-logging_hash=$(${sec_plugin_path}/tools/hash.sh -p "${logginguser_pw}" | tr " " "\n" | tail -n 1)
+# Hash the passwords using the security plugin's hash script.
+admin_hash=$(${security_tools_path}/hash.sh -p "${admin_pw}" | awk '{print $NF}')
+dashboards_hash=$(${security_tools_path}/hash.sh -p "${kibanaserver_pw}" | awk '{print $NF}')
+logging_hash=$(${security_tools_path}/hash.sh -p "${logginguser_pw}" | awk '{print $NF}')
+
+internal_users_file="${security_config_path}/internal_users.yml"
+internal_users_template="${security_config_path}/internal_users.template.yml"
+
+# Create internal_users.yml from the template.
+# This is important to prevent docker mount issues.
+cp "${internal_users_template}" "${internal_users_file}"
+echo "[INFO] Updating password hashes."
+
+# Create a temporary file before modifying.
+cp "${internal_users_file}" "${internal_users_file}.tmp"
 
-# write the passwords to 'internal_users.yml'
-# (the lines to be replaced are marked with comments)
-# this is because the '-rev' flag for 'securityadmin.sh' does not seem to be working for 'internal_users.yml'
-source="./config/internal_users.template.yml"
-target="${sec_plugin_path}/securityconfig/internal_users.yml"
-sed -e "s|^\(\s\+hash:\).*\s\+# ADMIN_PW|\1 ${admin_hash}|" \
-    -e "s|^\(\s\+hash:\).*\s\+# DASHBOARDS_PW|\1 ${dashboards_hash}|" \
-    -e "s|^\(\s\+hash:\).*\s\+# LOGGING_PW|\1 ${logging_hash}|" \
-    "${source}" > "${target}"
+# Add an autogenerated comment and remove the original 1st and 2nd lines.
+{
+    echo "# ------------------------------------------------- #"
+    echo "# Auto-generated from 'internal_users.template.yml' #"
+    echo "# ------------------------------------------------- #"
+    sed '1d;2d' "${internal_users_file}"
+} > "${internal_users_file}.tmp"
 
+# Update the password hashes while preserving all other content.
+sed -E \
+    -e "s|^( *hash: )[^\s]+(  # ADMIN_PW)|\1${admin_hash}\2|" \
+    -e "s|^( *hash: )[^\s]+(  # DASHBOARDS_PW)|\1${dashboards_hash}\2|" \
+    -e "s|^( *hash: )[^\s]+(  # LOGGING_PW)|\1${logging_hash}\2|" \
+    "${internal_users_file}.tmp" > "${internal_users_file}"
 
-# start opensearch (securityadmin.sh needs the security index)
+# Manual cleanup as we persist configuration (see 'docker-compose.base.yml').
+rm "${internal_users_file}.tmp"
+
+# Start OpenSearch (securityadmin.sh needs the security index).
 opensearch -Eplugins.security.allow_default_init_securityindex=true -Ediscovery.type=single-node &
 os_pid=$!
 retries=10
 
-# wait until opensearch is ready to accept connections
+# Wait until OpenSearch is ready to accept connections.
 while [[ "${retries}" -gt 0 ]] ; do
-    echo "waiting for opensearch to be up... $retries"
+    echo "[INFO] Waiting for opensearch to be up. $retries"
     sleep 5s
     if curl -ksu "admin:admin" "https://localhost:9200"; then
       break
@@ -45,23 +67,21 @@ while [[ "${retries}" -gt 0 ]] ; do
 done
 
 if [[ "${retries}" -eq 0 ]]; then
-  echo &>2 "error: exceeded maximum number of retries!"
+  echo &>2 "[ERROR] Exceeded maximum number of retries!"
   exit 1
 fi
 
-# wait for opensearch to be healthy
+# Wait for opensearch to be healthy.
 curl -ku "admin:admin" "https://localhost:9200/_cluster/health?wait_for_status=yellow"
 
-
-# securityadmin needs admin certificates or a keystore to operate
-"${sec_plugin_path}/tools/securityadmin.sh" \
+# The securityadmin script needs admin certificates or a keystore to operate.
+"${security_tools_path}/securityadmin.sh" \
   -rev -icl -nhnv \
   -cacert ./config/root-ca.pem \
   -cert ./config/admin-crt.pem \
   -key ./config/admin-key.pem \
-  -cd "${sec_plugin_path}/securityconfig"
-
+  -cd "${security_config_path}"
 
-# shut down opensearch again
+# Shut down opensearch again.
 sleep 5s
 kill ${os_pid}
diff --git a/docker/opensearch/opensearch.yml b/docker/opensearch/opensearch.yml
index ce4348e..81ede48 100644
--- a/docker/opensearch/opensearch.yml
+++ b/docker/opensearch/opensearch.yml
@@ -26,6 +26,7 @@ plugins.security.allow_default_init_securityindex: false
 plugins.security.ssl.http.enabled: true
 plugins.security.ssl.http.pemkey_filepath: cluster-key.pem
 plugins.security.ssl.http.pemcert_filepath: cluster-crt.pem
+plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
 
 # transport layer TLS, for inter-node communication
 plugins.security.ssl.transport.pemkey_filepath: node-key.pem
diff --git a/docker/opensearch/security/internal_users.template.yml b/docker/opensearch/security/internal_users.template.yml
index ce5731e..1d72ad3 100644
--- a/docker/opensearch/security/internal_users.template.yml
+++ b/docker/opensearch/security/internal_users.template.yml
@@ -1,13 +1,16 @@
----
-# This is the internal user database (for initial setup)
+# This file serves as a template.
+# Contents will be written to the 'internal_users.yml' file.
+#
+# This is the internal user database (for initial setup).
 #
 # The hash value is a bcrypt hash and can be generated with
-# './plugins/opensearch-security/tools/hash.sh' (in an OS node container)
+# './plugins/opensearch-security/tools/hash.sh' (in an OS node container).
 #
-# note: reserved users can't be changed, except for with 'securityadmin.sh'
+# NOTE: Reserved users can't be changed, except for with 'securityadmin.sh'.
 #
-# pre-defined roles can be seen in the documentation:
+# Pre-defined roles can be seen in the documentation:
 # https://opensearch.org/docs/latest/security-plugin/access-control/users-roles/#predefined-roles
+#
 
 _meta:
   type: "internalusers"
@@ -24,7 +27,7 @@ admin:
 kibanaserver:
   hash: "$2y$12$LuNY.MSV7dGzyy85c8v1oOxIRlP0TFEk6qgpGRYHZW0Qk0YVc1qdm"  # DASHBOARDS_PW
   reserved: true
-  description: "Machine account for the OpenSeaerch Dashboards server"
+  description: "Machine account for the OpenSearch Dashboards server"
 
 # the 'logging_user' with the 'logstash' role has permissions to manage the 'logstash-*' indices
 logging_user:
diff --git a/docker/opensearch/security/tenants.yml b/docker/opensearch/security/tenants.yml
index 60f6100..7d923c9 100644
--- a/docker/opensearch/security/tenants.yml
+++ b/docker/opensearch/security/tenants.yml
@@ -1,4 +1,3 @@
----
 _meta:
   type: "tenants"
   config_version: 2
diff --git a/scripts/generate-ssl.sh b/scripts/generate-ssl.sh
index 18edf44..363e8d4 100755
--- a/scripts/generate-ssl.sh
+++ b/scripts/generate-ssl.sh
@@ -7,7 +7,7 @@ set -euo pipefail
 
 # The guard below ensures, in most cases, that the script is executed from the correct directory (project directory).
 # So it should be safe to assume that we can extract the path from the parent of the 'scripts' directory.
-PROJECT_DIR="$(dirname "$(dirname "$(realpath "$0")")")"
+project_dir="$(dirname "$(dirname "$(realpath "$0")")")"
 
 # ---------------- #
 # Execution checks #
@@ -22,18 +22,22 @@ fi
 # --------------------- #
 # Environment variables #
 # --------------------- #
-ENV_FILE="${PROJECT_DIR}/.env"
+env_file="${project_dir}/.env"
 
 # Check if the .env file exists and source it
-if [[ -f "${ENV_FILE}" ]]; then
+if [[ -f "${env_file}" ]]; then
     echo "[INFO] Detected .env file, sourcing it."
-    source "${ENV_FILE}"
+    source "${env_file}"
 fi
 
 # Set hostnames for Dashboards and Opensearch nodes.
 # They are determining whether the setup is in a local or live environment.
-DASHBOARDS_HOSTNAME="${DASHBOARDS_HOSTNAME:-localhost}"
-OPENSEARCH_HOSTNAME="${OPENSEARCH_HOSTNAME:-localhost}"
+dashboards_hostname="${DASHBOARDS_HOSTNAME:-localhost}"
+opensearch_hostname="${OPENSEARCH_HOSTNAME:-localhost}"
+
+certbot_email="${CERTBOT_EMAIL:-}"
+certbot_rsa_key_size="${CERTBOT_RSA_KEY_SIZE:-4096}"
+certbot_staging="${CERTBOT_STAGING:-}"
 
 # --------------------- #
 # Utilities definitions #
@@ -112,9 +116,8 @@ function link_certbot_certificates {
     fi
     echo "[INFO] Permissions adjusted for Let's Encrypt directories."
 
-
     echo "[INFO] Linking Certbot certificates for ${hostname} to ${cert_name}."
-    local ssl_path="${PROJECT_DIR}/docker/certbot/conf/live/${hostname}"
+    local ssl_path="${project_dir}/docker/certbot/conf/live/${hostname}"
 
     if [[ -d "${ssl_path}" ]]; then
         ln -sf "${ssl_path}/privkey.pem" "${cert_name}-key.pem"
@@ -125,11 +128,29 @@ function link_certbot_certificates {
     fi
 }
 
+function append_letsencrypt_ca() {
+    local hostname="$1"
+    local ca_chain="${project_dir}/docker/certbot/conf/live/${hostname}/chain.pem"
+    local root_ca="${project_dir}/ssl/root-ca-crt.pem"
+
+    # Append Let's Encrypt CA to root-ca-crt.pem only if it's not already present.
+    # OpenSearch needs it to recognize the new certificates.
+    chain_cert_content=$(cat "$ca_chain")
+    root_ca_content=$(cat "$root_ca")
+
+    if [[ "$root_ca_content" == *"$chain_cert_content"* ]]; then
+        :
+    else
+        echo -e "\n$chain_cert_content" >> "$root_ca"  # Append correctly formatted certificate.
+        echo "[INFO] Appending Let's Encrypt CA to 'root-ca-crt.pem'."
+    fi
+}
+
 # ------------ #
 # Run sequence #
 # ------------ #
 # Root CA
-subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=ROOT" 
+subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=ROOT"
 openssl genrsa -out root-ca-key.pem 2048
 openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "${subject}" -out root-ca-crt.pem -days 365
 
@@ -143,77 +164,80 @@ generate_certificate "node1" "os-node-1"
 generate_certificate "node2" "os-node-2"
 
 # Determine external SSL certificates generation
-if is_local_setup "${DASHBOARDS_HOSTNAME}" && is_local_setup "${OPENSEARCH_HOSTNAME}"; then
+if is_local_setup "${dashboards_hostname}" || is_local_setup "${opensearch_hostname}"; then
+    echo "[INFO] Detected local setup, generating local certificates for Dashboards and Cluster."
+
     # REST HTTPS cert (for outside communication)
-    generate_if_not_user_defined "cluster" "${OPENSEARCH_HOSTNAME}"
+    generate_if_not_user_defined "cluster" "${opensearch_hostname}"
 
-    echo "[INFO] Detected local setup, generating local certificates for Dashboards and Cluster."
     # Dashboards cert (for outside communication)
-    generate_if_not_user_defined "dashboards" "${DASHBOARDS_HOSTNAME}"
+    generate_if_not_user_defined "dashboards" "${dashboards_hostname}"
 
 else
     echo "[INFO] Detected live setup, generating Let's Encrypt certificates for Dashboards and Cluster."
-    CERTBOT_EMAIL="${CERTBOT_EMAIL:-}"
-    CERTBOT_RSA_KEY_SIZE="${CERTBOT_RSA_KEY_SIZE:-4096}"
-    CERTBOT_STAGING="${CERTBOT_STAGING:-}"
 
     # Select appropriate email argument.
-    if [[ -z "${CERTBOT_EMAIL}" ]]; then
+    if [[ -z "${certbot_email}" ]]; then
         email_arg="--register-unsafely-without-email"
     else
-        email_arg="--email ${CERTBOT_EMAIL}"
+        email_arg="--email ${certbot_email}"
     fi
 
     # Determine certbot staging argument.
-    if [[ -z "${CERTBOT_STAGING}" ]] || [[ ! "${CERTBOT_STAGING}" =~ ^[01]$ ]]; then
+    if [[ -z "${certbot_staging}" ]] || [[ ! "${certbot_staging}" =~ ^[01]$ ]]; then
         echo "[WARN] Invalid or unset value for 'CERTBOT_STAGING'. Setting it to 1 (testing mode)."
-        CERTBOT_STAGING="1"
+        certbot_staging="1"
     fi
-    if [[ "${CERTBOT_STAGING}" != "0" ]]; then staging_arg="--staging"; fi
+    if [[ "${certbot_staging}" != "0" ]]; then staging_arg="--staging"; fi
 
     # Define common certbot command
-    certbot_cmd="certbot certonly ${staging_arg:-} ${email_arg} --rsa-key-size ${CERTBOT_RSA_KEY_SIZE} --standalone --agree-tos --force-renewal"
+    certbot_cmd="certbot certonly ${staging_arg:-} ${email_arg} --rsa-key-size ${certbot_rsa_key_size} --standalone --agree-tos --force-renewal"
+
+    ssl_base_path="${project_dir}/docker/certbot/conf/live"
 
     # Check if Dashboards and Cluster point to the same hostname.
-    if [[ "${DASHBOARDS_HOSTNAME}" == "${OPENSEARCH_HOSTNAME}" ]]; then
+    if [[ "${dashboards_hostname}" == "${opensearch_hostname}" ]]; then
         echo "[INFO] Hostnames for Dashboards and Cluster are the same. Generating a single certificate for both."
-        LIVE_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${DASHBOARDS_HOSTNAME}/fullchain.pem"
+        live_crt_path="${ssl_base_path}/${dashboards_hostname}/fullchain.pem"
 
-        if [[ -L "${LIVE_CRT_PATH}" ]]; then
-            echo "[WARN] Certificates for ${DASHBOARDS_HOSTNAME} already exist. Skipping generation."
+        if [[ -L "${live_crt_path}" ]]; then
+            echo "[WARN] Certificates for ${dashboards_hostname} already exist. Skipping generation."
 
         else
-            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${DASHBOARDS_HOSTNAME}\" certbot"
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${dashboards_hostname}\" certbot"
             # Symlink Certbot certificates
-            link_certbot_certificates "dashboards" "${DASHBOARDS_HOSTNAME}"
-            link_certbot_certificates "cluster" "${OPENSEARCH_HOSTNAME}"
+            link_certbot_certificates "dashboards" "${dashboards_hostname}"
+            link_certbot_certificates "cluster" "${opensearch_hostname}"
+            append_letsencrypt_ca "${dashboards_hostname}"
         fi
 
     else
         echo "[INFO] Hostnames for are different. Generating separate certificates for each."
-        LIVE_DASHBOARDS_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${DASHBOARDS_HOSTNAME}/fullchain.pem"
-        LIVE_OPENSEARCH_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${OPENSEARCH_HOSTNAME}/fullchain.pem"
+        live_dashboards_crt_path="${ssl_base_path}/${dashboards_hostname}/fullchain.pem"
+        live_opensearch_crt_path="${ssl_base_path}/${opensearch_hostname}/fullchain.pem"
 
-        # Attempt to generate certificate for DASHBOARDS_HOSTNAME.
-        if [[ -L "${LIVE_DASHBOARDS_CRT_PATH}" ]]; then
-            echo "[WARN] Certificates for ${DASHBOARDS_HOSTNAME} already exist. Skipping generation."
+        # Attempt to generate certificate for dashboards_hostname.
+        if [[ -L "${live_dashboards_crt_path}" ]]; then
+            echo "[WARN] Certificates for ${dashboards_hostname} already exist. Skipping generation."
 
         else
-            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${DASHBOARDS_HOSTNAME}\" certbot"
-            link_certbot_certificates "dashboards" "${DASHBOARDS_HOSTNAME}"
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${dashboards_hostname}\" certbot"
+            link_certbot_certificates "dashboards" "${dashboards_hostname}"
+            append_letsencrypt_ca "${dashboards_hostname}"
         fi
 
-        # Attempt to generate certificate for OPENSEARCH_HOSTNAME.
-        if [[ -L "${LIVE_OPENSEARCH_CRT_PATH}" ]]; then
-            echo "[WARN] Certificates for ${OPENSEARCH_HOSTNAME} already exist. Skipping generation."
+        # Attempt to generate certificate for opensearch_hostname.
+        if [[ -L "${live_opensearch_crt_path}" ]]; then
+            echo "[WARN] Certificates for ${opensearch_hostname} already exist. Skipping generation."
 
         else
-            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${OPENSEARCH_HOSTNAME}\" certbot"
-            link_certbot_certificates "cluster" "${OPENSEARCH_HOSTNAME}"
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${opensearch_hostname}\" certbot"
+            link_certbot_certificates "cluster" "${opensearch_hostname}"
+            append_letsencrypt_ca "${opensearch_hostname}"
         fi
     fi
 fi
 
 # Ensure read permissions to all files in the 'ssl' directory because 'opensearch' user must be able to read them in the container.
 # This is especially important for the *-key.pem files (private keys).
-chmod -R +r "${PROJECT_DIR}/ssl"
+chmod -R +r "${project_dir}/ssl"
-- 
GitLab