From 732019ed05852a8a3a07d4b9f2e1e9bc67cd5814 Mon Sep 17 00:00:00 2001
From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at>
Date: Thu, 6 Feb 2025 19:05:03 +0100
Subject: [PATCH] =?UTF-8?q?=F0=9F=94=A7=20Setup:=20adjust=20for=20OpenSear?=
 =?UTF-8?q?ch=20v2?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

---
 docker-compose.base.yml                       |  12 +-
 docker-compose.yml                            |   8 +-
 docker/opensearch/init-security.sh            |  76 +++++++-----
 docker/opensearch/opensearch.yml              |   1 +
 .../security/internal_users.template.yml      |  15 ++-
 docker/opensearch/security/tenants.yml        |   1 -
 scripts/generate-ssl.sh                       | 108 +++++++++++-------
 7 files changed, 136 insertions(+), 85 deletions(-)

diff --git a/docker-compose.base.yml b/docker-compose.base.yml
index 9bf5d4e..9fc7407 100644
--- a/docker-compose.base.yml
+++ b/docker-compose.base.yml
@@ -21,14 +21,14 @@ services:
         soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems
         hard: 65536
     volumes:
-      - os-node-1:/usr/share/opensearch/data
-      # NOTE: We map every file separately so that we don't overwrite the files created
-      # in the same directory from opensearch.
-      - ./docker/opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/internal_users.template.yml:ro
-      - ./docker/opensearch/security/internal_users.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/internal_users.yml
-      - ./docker/opensearch/security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml:ro
+      - os-node-1-config:/usr/share/opensearch/config
+      - os-node-1-data:/usr/share/opensearch/data
       - ./docker/opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro
       - ./docker/opensearch/init-security.sh:/usr/share/opensearch/init-security.sh:ro
+      # NOTE: In this section we map every configuration file separately so that
+      # we don't overwrite the files created in the same directory from OpenSearch.
+      - ./docker/opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/opensearch-security/internal_users.template.yml
+      - ./docker/opensearch/security/tenants.yml:/usr/share/opensearch/config/opensearch-security/tenants.yml
       # internal ssl files
       - ./ssl/node1-crt.pem:/usr/share/opensearch/config/node-crt.pem:ro
       - ./ssl/node1-key.pem:/usr/share/opensearch/config/node-key.pem:ro
diff --git a/docker-compose.yml b/docker-compose.yml
index 3799671..e3132a8 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,7 +44,10 @@ services:
     # Beats versions newer than 7.12.x are not supported by OpenSearch
     image: docker.elastic.co/beats/metricbeat-oss:7.12.1
     restart: "unless-stopped"
-    user: root
+    user: ${METRICBEAT_CONTAINER_UID:-1000}
+    # add docker gid from the host so that docker.sock can be accessed
+    group_add:
+      - ${METRICBEAT_DOCKER_GID:-999}
     environment:
       - METRICBEAT_INDEX_INFIX=${METRICBEAT_INDEX_INFIX:-dev}
       - METRICBEAT_INTERVAL=${METRICBEAT_INTERVAL:-10s}
@@ -80,7 +83,8 @@ services:
       '
 
 volumes:
-  os-node-1:
+  os-node-1-config:
+  os-node-1-data:
   os-node-2:
   metricbeat-data:
 
diff --git a/docker/opensearch/init-security.sh b/docker/opensearch/init-security.sh
index 987542f..4fb551f 100755
--- a/docker/opensearch/init-security.sh
+++ b/docker/opensearch/init-security.sh
@@ -1,42 +1,64 @@
 #!/bin/bash
 #
-# update the internal_users via 'securityadmin.sh' to set their passwords
-# this should only be called on a new cluster
+# Update the internal_users via 'securityadmin.sh' to set their passwords.
+# This should only be called on a new cluster.
+#
+# Usage: docker compose run os-node-1 ./init-security.sh
 #
-# usage: docker compose run os-node-1 ./init-security.sh
 
 set -euo pipefail
 
+# Define common paths in the opensearch docker container.
+security_config_path="./config/opensearch-security"
+security_tools_path="./plugins/opensearch-security/tools"
 
-# get the passwords from the env vars, and hash them
-sec_plugin_path="./plugins/opensearch-security"
+# Get the passwords from the env vars.
 admin_pw="${OPENSEARCH_ADMIN_PASSWORD:?admin password required}"
 kibanaserver_pw="${OPENSEARCH_KIBANASERVER_PASSWORD:?kibana server password required}"
 logginguser_pw="${OPENSEARCH_LOGGINGUSER_PASSWORD:?logging user password required}"
 
-admin_hash=$(${sec_plugin_path}/tools/hash.sh -p "${admin_pw}" | tr " " "\n" | tail -n 1)
-dashboards_hash=$(${sec_plugin_path}/tools/hash.sh -p "${kibanaserver_pw}" | tr " " "\n" | tail -n 1)
-logging_hash=$(${sec_plugin_path}/tools/hash.sh -p "${logginguser_pw}" | tr " " "\n" | tail -n 1)
+# Hash the passwords using the security plugin's hash script.
+admin_hash=$(${security_tools_path}/hash.sh -p "${admin_pw}" | awk '{print $NF}')
+dashboards_hash=$(${security_tools_path}/hash.sh -p "${kibanaserver_pw}" | awk '{print $NF}')
+logging_hash=$(${security_tools_path}/hash.sh -p "${logginguser_pw}" | awk '{print $NF}')
+
+internal_users_file="${security_config_path}/internal_users.yml"
+internal_users_template="${security_config_path}/internal_users.template.yml"
+
+# Create internal_users.yml from the template.
+# This is important to prevent docker mount issues.
+cp "${internal_users_template}" "${internal_users_file}"
+echo "[INFO] Updating password hashes."
+
+# Create a temporary file before modifying.
+cp "${internal_users_file}" "${internal_users_file}.tmp"
 
-# write the passwords to 'internal_users.yml'
-# (the lines to be replaced are marked with comments)
-# this is because the '-rev' flag for 'securityadmin.sh' does not seem to be working for 'internal_users.yml'
-source="./config/internal_users.template.yml"
-target="${sec_plugin_path}/securityconfig/internal_users.yml"
-sed -e "s|^\(\s\+hash:\).*\s\+# ADMIN_PW|\1 ${admin_hash}|" \
-    -e "s|^\(\s\+hash:\).*\s\+# DASHBOARDS_PW|\1 ${dashboards_hash}|" \
-    -e "s|^\(\s\+hash:\).*\s\+# LOGGING_PW|\1 ${logging_hash}|" \
-    "${source}" > "${target}"
+# Add an autogenerated comment and remove the original 1st and 2nd lines.
+{
+    echo "# ------------------------------------------------- #"
+    echo "# Auto-generated from 'internal_users.template.yml' #"
+    echo "# ------------------------------------------------- #"
+    sed '1d;2d' "${internal_users_file}"
+} > "${internal_users_file}.tmp"
 
+# Update the password hashes while preserving all other content.
+sed -E \
+    -e "s|^( *hash: )[^\s]+(  # ADMIN_PW)|\1${admin_hash}\2|" \
+    -e "s|^( *hash: )[^\s]+(  # DASHBOARDS_PW)|\1${dashboards_hash}\2|" \
+    -e "s|^( *hash: )[^\s]+(  # LOGGING_PW)|\1${logging_hash}\2|" \
+    "${internal_users_file}.tmp" > "${internal_users_file}"
 
-# start opensearch (securityadmin.sh needs the security index)
+# Manual cleanup as we persist configuration (see 'docker-compose.base.yml').
+rm "${internal_users_file}.tmp"
+
+# Start OpenSearch (securityadmin.sh needs the security index).
 opensearch -Eplugins.security.allow_default_init_securityindex=true -Ediscovery.type=single-node &
 os_pid=$!
 retries=10
 
-# wait until opensearch is ready to accept connections
+# Wait until OpenSearch is ready to accept connections.
 while [[ "${retries}" -gt 0 ]] ; do
-    echo "waiting for opensearch to be up... $retries"
+    echo "[INFO] Waiting for opensearch to be up. $retries"
     sleep 5s
     if curl -ksu "admin:admin" "https://localhost:9200"; then
       break
@@ -45,23 +67,21 @@ while [[ "${retries}" -gt 0 ]] ; do
 done
 
 if [[ "${retries}" -eq 0 ]]; then
-  echo &>2 "error: exceeded maximum number of retries!"
+  echo &>2 "[ERROR] Exceeded maximum number of retries!"
   exit 1
 fi
 
-# wait for opensearch to be healthy
+# Wait for opensearch to be healthy.
 curl -ku "admin:admin" "https://localhost:9200/_cluster/health?wait_for_status=yellow"
 
-
-# securityadmin needs admin certificates or a keystore to operate
-"${sec_plugin_path}/tools/securityadmin.sh" \
+# The securityadmin script needs admin certificates or a keystore to operate.
+"${security_tools_path}/securityadmin.sh" \
   -rev -icl -nhnv \
   -cacert ./config/root-ca.pem \
   -cert ./config/admin-crt.pem \
   -key ./config/admin-key.pem \
-  -cd "${sec_plugin_path}/securityconfig"
-
+  -cd "${security_config_path}"
 
-# shut down opensearch again
+# Shut down opensearch again.
 sleep 5s
 kill ${os_pid}
diff --git a/docker/opensearch/opensearch.yml b/docker/opensearch/opensearch.yml
index ce4348e..81ede48 100644
--- a/docker/opensearch/opensearch.yml
+++ b/docker/opensearch/opensearch.yml
@@ -26,6 +26,7 @@ plugins.security.allow_default_init_securityindex: false
 plugins.security.ssl.http.enabled: true
 plugins.security.ssl.http.pemkey_filepath: cluster-key.pem
 plugins.security.ssl.http.pemcert_filepath: cluster-crt.pem
+plugins.security.ssl.http.pemtrustedcas_filepath: root-ca.pem
 
 # transport layer TLS, for inter-node communication
 plugins.security.ssl.transport.pemkey_filepath: node-key.pem
diff --git a/docker/opensearch/security/internal_users.template.yml b/docker/opensearch/security/internal_users.template.yml
index ce5731e..1d72ad3 100644
--- a/docker/opensearch/security/internal_users.template.yml
+++ b/docker/opensearch/security/internal_users.template.yml
@@ -1,13 +1,16 @@
----
-# This is the internal user database (for initial setup)
+# This file serves as a template.
+# Contents will be written to the 'internal_users.yml' file.
+#
+# This is the internal user database (for initial setup).
 #
 # The hash value is a bcrypt hash and can be generated with
-# './plugins/opensearch-security/tools/hash.sh' (in an OS node container)
+# './plugins/opensearch-security/tools/hash.sh' (in an OS node container).
 #
-# note: reserved users can't be changed, except for with 'securityadmin.sh'
+# NOTE: Reserved users can't be changed, except for with 'securityadmin.sh'.
 #
-# pre-defined roles can be seen in the documentation:
+# Pre-defined roles can be seen in the documentation:
 # https://opensearch.org/docs/latest/security-plugin/access-control/users-roles/#predefined-roles
+#
 
 _meta:
   type: "internalusers"
@@ -24,7 +27,7 @@ admin:
 kibanaserver:
   hash: "$2y$12$LuNY.MSV7dGzyy85c8v1oOxIRlP0TFEk6qgpGRYHZW0Qk0YVc1qdm"  # DASHBOARDS_PW
   reserved: true
-  description: "Machine account for the OpenSeaerch Dashboards server"
+  description: "Machine account for the OpenSearch Dashboards server"
 
 # the 'logging_user' with the 'logstash' role has permissions to manage the 'logstash-*' indices
 logging_user:
diff --git a/docker/opensearch/security/tenants.yml b/docker/opensearch/security/tenants.yml
index 60f6100..7d923c9 100644
--- a/docker/opensearch/security/tenants.yml
+++ b/docker/opensearch/security/tenants.yml
@@ -1,4 +1,3 @@
----
 _meta:
   type: "tenants"
   config_version: 2
diff --git a/scripts/generate-ssl.sh b/scripts/generate-ssl.sh
index 18edf44..363e8d4 100755
--- a/scripts/generate-ssl.sh
+++ b/scripts/generate-ssl.sh
@@ -7,7 +7,7 @@ set -euo pipefail
 
 # The guard below ensures, in most cases, that the script is executed from the correct directory (project directory).
 # So it should be safe to assume that we can extract the path from the parent of the 'scripts' directory.
-PROJECT_DIR="$(dirname "$(dirname "$(realpath "$0")")")"
+project_dir="$(dirname "$(dirname "$(realpath "$0")")")"
 
 # ---------------- #
 # Execution checks #
@@ -22,18 +22,22 @@ fi
 # --------------------- #
 # Environment variables #
 # --------------------- #
-ENV_FILE="${PROJECT_DIR}/.env"
+env_file="${project_dir}/.env"
 
 # Check if the .env file exists and source it
-if [[ -f "${ENV_FILE}" ]]; then
+if [[ -f "${env_file}" ]]; then
     echo "[INFO] Detected .env file, sourcing it."
-    source "${ENV_FILE}"
+    source "${env_file}"
 fi
 
 # Set hostnames for Dashboards and Opensearch nodes.
 # They are determining whether the setup is in a local or live environment.
-DASHBOARDS_HOSTNAME="${DASHBOARDS_HOSTNAME:-localhost}"
-OPENSEARCH_HOSTNAME="${OPENSEARCH_HOSTNAME:-localhost}"
+dashboards_hostname="${DASHBOARDS_HOSTNAME:-localhost}"
+opensearch_hostname="${OPENSEARCH_HOSTNAME:-localhost}"
+
+certbot_email="${CERTBOT_EMAIL:-}"
+certbot_rsa_key_size="${CERTBOT_RSA_KEY_SIZE:-4096}"
+certbot_staging="${CERTBOT_STAGING:-}"
 
 # --------------------- #
 # Utilities definitions #
@@ -112,9 +116,8 @@ function link_certbot_certificates {
     fi
     echo "[INFO] Permissions adjusted for Let's Encrypt directories."
 
-
     echo "[INFO] Linking Certbot certificates for ${hostname} to ${cert_name}."
-    local ssl_path="${PROJECT_DIR}/docker/certbot/conf/live/${hostname}"
+    local ssl_path="${project_dir}/docker/certbot/conf/live/${hostname}"
 
     if [[ -d "${ssl_path}" ]]; then
         ln -sf "${ssl_path}/privkey.pem" "${cert_name}-key.pem"
@@ -125,11 +128,29 @@ function link_certbot_certificates {
     fi
 }
 
+function append_letsencrypt_ca() {
+    local hostname="$1"
+    local ca_chain="${project_dir}/docker/certbot/conf/live/${hostname}/chain.pem"
+    local root_ca="${project_dir}/ssl/root-ca-crt.pem"
+
+    # Append Let's Encrypt CA to root-ca-crt.pem only if it's not already present.
+    # OpenSearch needs it to recognize the new certificates.
+    chain_cert_content=$(cat "$ca_chain")
+    root_ca_content=$(cat "$root_ca")
+
+    if [[ "$root_ca_content" == *"$chain_cert_content"* ]]; then
+        :
+    else
+        echo -e "\n$chain_cert_content" >> "$root_ca"  # Append correctly formatted certificate.
+        echo "[INFO] Appending Let's Encrypt CA to 'root-ca-crt.pem'."
+    fi
+}
+
 # ------------ #
 # Run sequence #
 # ------------ #
 # Root CA
-subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=ROOT" 
+subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=ROOT"
 openssl genrsa -out root-ca-key.pem 2048
 openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "${subject}" -out root-ca-crt.pem -days 365
 
@@ -143,77 +164,80 @@ generate_certificate "node1" "os-node-1"
 generate_certificate "node2" "os-node-2"
 
 # Determine external SSL certificates generation
-if is_local_setup "${DASHBOARDS_HOSTNAME}" && is_local_setup "${OPENSEARCH_HOSTNAME}"; then
+if is_local_setup "${dashboards_hostname}" || is_local_setup "${opensearch_hostname}"; then
+    echo "[INFO] Detected local setup, generating local certificates for Dashboards and Cluster."
+
     # REST HTTPS cert (for outside communication)
-    generate_if_not_user_defined "cluster" "${OPENSEARCH_HOSTNAME}"
+    generate_if_not_user_defined "cluster" "${opensearch_hostname}"
 
-    echo "[INFO] Detected local setup, generating local certificates for Dashboards and Cluster."
     # Dashboards cert (for outside communication)
-    generate_if_not_user_defined "dashboards" "${DASHBOARDS_HOSTNAME}"
+    generate_if_not_user_defined "dashboards" "${dashboards_hostname}"
 
 else
     echo "[INFO] Detected live setup, generating Let's Encrypt certificates for Dashboards and Cluster."
-    CERTBOT_EMAIL="${CERTBOT_EMAIL:-}"
-    CERTBOT_RSA_KEY_SIZE="${CERTBOT_RSA_KEY_SIZE:-4096}"
-    CERTBOT_STAGING="${CERTBOT_STAGING:-}"
 
     # Select appropriate email argument.
-    if [[ -z "${CERTBOT_EMAIL}" ]]; then
+    if [[ -z "${certbot_email}" ]]; then
         email_arg="--register-unsafely-without-email"
     else
-        email_arg="--email ${CERTBOT_EMAIL}"
+        email_arg="--email ${certbot_email}"
     fi
 
     # Determine certbot staging argument.
-    if [[ -z "${CERTBOT_STAGING}" ]] || [[ ! "${CERTBOT_STAGING}" =~ ^[01]$ ]]; then
+    if [[ -z "${certbot_staging}" ]] || [[ ! "${certbot_staging}" =~ ^[01]$ ]]; then
         echo "[WARN] Invalid or unset value for 'CERTBOT_STAGING'. Setting it to 1 (testing mode)."
-        CERTBOT_STAGING="1"
+        certbot_staging="1"
     fi
-    if [[ "${CERTBOT_STAGING}" != "0" ]]; then staging_arg="--staging"; fi
+    if [[ "${certbot_staging}" != "0" ]]; then staging_arg="--staging"; fi
 
     # Define common certbot command
-    certbot_cmd="certbot certonly ${staging_arg:-} ${email_arg} --rsa-key-size ${CERTBOT_RSA_KEY_SIZE} --standalone --agree-tos --force-renewal"
+    certbot_cmd="certbot certonly ${staging_arg:-} ${email_arg} --rsa-key-size ${certbot_rsa_key_size} --standalone --agree-tos --force-renewal"
+
+    ssl_base_path="${project_dir}/docker/certbot/conf/live"
 
     # Check if Dashboards and Cluster point to the same hostname.
-    if [[ "${DASHBOARDS_HOSTNAME}" == "${OPENSEARCH_HOSTNAME}" ]]; then
+    if [[ "${dashboards_hostname}" == "${opensearch_hostname}" ]]; then
         echo "[INFO] Hostnames for Dashboards and Cluster are the same. Generating a single certificate for both."
-        LIVE_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${DASHBOARDS_HOSTNAME}/fullchain.pem"
+        live_crt_path="${ssl_base_path}/${dashboards_hostname}/fullchain.pem"
 
-        if [[ -L "${LIVE_CRT_PATH}" ]]; then
-            echo "[WARN] Certificates for ${DASHBOARDS_HOSTNAME} already exist. Skipping generation."
+        if [[ -L "${live_crt_path}" ]]; then
+            echo "[WARN] Certificates for ${dashboards_hostname} already exist. Skipping generation."
 
         else
-            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${DASHBOARDS_HOSTNAME}\" certbot"
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${dashboards_hostname}\" certbot"
             # Symlink Certbot certificates
-            link_certbot_certificates "dashboards" "${DASHBOARDS_HOSTNAME}"
-            link_certbot_certificates "cluster" "${OPENSEARCH_HOSTNAME}"
+            link_certbot_certificates "dashboards" "${dashboards_hostname}"
+            link_certbot_certificates "cluster" "${opensearch_hostname}"
+            append_letsencrypt_ca "${dashboards_hostname}"
         fi
 
     else
         echo "[INFO] Hostnames for are different. Generating separate certificates for each."
-        LIVE_DASHBOARDS_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${DASHBOARDS_HOSTNAME}/fullchain.pem"
-        LIVE_OPENSEARCH_CRT_PATH="${PROJECT_DIR}/docker/certbot/conf/live/${OPENSEARCH_HOSTNAME}/fullchain.pem"
+        live_dashboards_crt_path="${ssl_base_path}/${dashboards_hostname}/fullchain.pem"
+        live_opensearch_crt_path="${ssl_base_path}/${opensearch_hostname}/fullchain.pem"
 
-        # Attempt to generate certificate for DASHBOARDS_HOSTNAME.
-        if [[ -L "${LIVE_DASHBOARDS_CRT_PATH}" ]]; then
-            echo "[WARN] Certificates for ${DASHBOARDS_HOSTNAME} already exist. Skipping generation."
+        # Attempt to generate certificate for dashboards_hostname.
+        if [[ -L "${live_dashboards_crt_path}" ]]; then
+            echo "[WARN] Certificates for ${dashboards_hostname} already exist. Skipping generation."
 
         else
-            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${DASHBOARDS_HOSTNAME}\" certbot"
-            link_certbot_certificates "dashboards" "${DASHBOARDS_HOSTNAME}"
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${dashboards_hostname}\" certbot"
+            link_certbot_certificates "dashboards" "${dashboards_hostname}"
+            append_letsencrypt_ca "${dashboards_hostname}"
         fi
 
-        # Attempt to generate certificate for OPENSEARCH_HOSTNAME.
-        if [[ -L "${LIVE_OPENSEARCH_CRT_PATH}" ]]; then
-            echo "[WARN] Certificates for ${OPENSEARCH_HOSTNAME} already exist. Skipping generation."
+        # Attempt to generate certificate for opensearch_hostname.
+        if [[ -L "${live_opensearch_crt_path}" ]]; then
+            echo "[WARN] Certificates for ${opensearch_hostname} already exist. Skipping generation."
 
         else
-            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${OPENSEARCH_HOSTNAME}\" certbot"
-            link_certbot_certificates "cluster" "${OPENSEARCH_HOSTNAME}"
+            eval "docker compose run --rm -p \"80:80\" --entrypoint \"${certbot_cmd} -d ${opensearch_hostname}\" certbot"
+            link_certbot_certificates "cluster" "${opensearch_hostname}"
+            append_letsencrypt_ca "${opensearch_hostname}"
         fi
     fi
 fi
 
 # Ensure read permissions to all files in the 'ssl' directory because 'opensearch' user must be able to read them in the container.
 # This is especially important for the *-key.pem files (private keys).
-chmod -R +r "${PROJECT_DIR}/ssl"
+chmod -R +r "${project_dir}/ssl"
-- 
GitLab