From c521e294488d42f47e49b67e1d79e48e3e67038e Mon Sep 17 00:00:00 2001 From: Sotiris Tsepelakis <sotirios.tsepelakis@tuwien.ac.at> Date: Wed, 27 Jul 2022 16:24:09 +0200 Subject: [PATCH] Implement a small containerized OpenSearch setup * we're going for a small single-machine setup here * we don't include logstash/fluentd in the setup; this is off-loaded to other projects sending their logs to OS deployments * configure the security plugin to use self-generated certificates, and create a bunch of customized internal users * note: apparently, securityadmin's '-rev' flag doesn't do anything for internal_users * instead, we need to go the good old 'sed' route to update the internal users --- .gitignore | 5 ++ README.md | 48 +++++++++++ dashboards/config/opensearch_dashboards.yml | 24 ++++++ docker-compose.base.yml | 41 +++++++++ docker-compose.yml | 51 ++++++++++++ example.env | 18 ++++ opensearch/init-security.sh | 67 +++++++++++++++ opensearch/opensearch.yml | 48 +++++++++++ .../security/internal_users.template.yml | 35 ++++++++ opensearch/security/tenants.yml | 9 ++ scripts/generate-ssl.sh | 83 +++++++++++++++++++ scripts/setup.sh | 19 +++++ ssl/.gitkeep | 0 13 files changed, 448 insertions(+) create mode 100644 .gitignore create mode 100644 dashboards/config/opensearch_dashboards.yml create mode 100644 docker-compose.base.yml create mode 100644 docker-compose.yml create mode 100644 example.env create mode 100755 opensearch/init-security.sh create mode 100644 opensearch/opensearch.yml create mode 100644 opensearch/security/internal_users.template.yml create mode 100644 opensearch/security/tenants.yml create mode 100755 scripts/generate-ssl.sh create mode 100755 scripts/setup.sh create mode 100644 ssl/.gitkeep diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f4fa38d --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +.env + +# ssl keys and certificates +ssl/* +!ssl/.gitkeep diff --git a/README.md b/README.md index a02964c..9e0b2cb 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,50 @@ # CRDM Logging Setup +Containerized setup for the log server of the "Center for Research Data Management" at TU Wien. + + +# Setup + +Setting up the logging server consists of the following steps: + +* Create a `.env` file (c.f. [`example.env`](./example.env)) +* Initialize the TLS/SSL certificates +* Initialize the security index with `docker compose run os-node-1 ./init-security.sh` +* Start the cluster with `docker compose up` + +Note: Populating the `.env` file has to be done manually, but the remaining steps are automated by [`./scripts/setup.sh`](./scripts/setup.sh)! + + +## TLS/SSL certificates + +A root CA and self-signed certificates for inter-container communication (as well as HTTPS certificates for public-facing endpoints) can be generated with [`scripts/generate-ssl.sh`](./scripts/generate-ssl.sh). + +These files include: +* `root-ca-{crt,key}.pem`: Key pair for the root CA +* `root-ca-crt.srl`: Serial number for the root CA +* `node{1,2}-{crt,key}.pem`: Key pairs for the inter-container communication +* `{cluster,dashboards}-{crt,key}.pem`: Key pairs for public-facing endpoints + +Note: For the common name of the public-facing certificates, the script will take the value of `${OPENSEARCH_HOSTNAME}` and `${DASHBOARDS_HOSTNAME}`, respectively. +If either of these variables isn't set, a fallback value of `localhost` will be used. + + +### Custom key pairs for external communication + +Of course, it can be desirable to use custom certificates (that aren't self-signed) on public-facing endpoints. +Such key pairs can be set by placing the corresponding files (`{cluster,dashboards}-{crt,key}.pem`) in the `ssl/` directory. +If the script detects that they exist as regular files (and not as symlinks), it will skip the auto-generation for these files and leave them as is. + + +## Security configuration + +Before being able to use the log server, encryption and authentication/authorization need to be set up. +The script [`./opensearch/init-security.sh`](./opensearch/init-security.sh) (to be executed inside the `node-1` container) takes care of that. + +It creates the users defined in [`./opensearch/security/internal_users.template.yml`](./opensearch/security/internal_users.template.yml), i.e. `admin`, `kibanaserver`, and `logging_user`. +Also, it sets their passwords to the values specified in the following environment variables: +* `OPENSEARCH_ADMIN_PASSWORD` +* `OPENSEARCH_KIBANASERVER_PASSWORD` +* `OPENSEARCH_LOGGINGUSER_PASSWORD` + +Note that this setup script will throw away any internal users defined via the REST API! diff --git a/dashboards/config/opensearch_dashboards.yml b/dashboards/config/opensearch_dashboards.yml new file mode 100644 index 0000000..d8386a1 --- /dev/null +++ b/dashboards/config/opensearch_dashboards.yml @@ -0,0 +1,24 @@ +# Configuration for OpenSearch Dashboards +# +# note: auth config (opensearch.{username,password}) is set via environment variables +# (see docker-compose.yml) + +# general config +server.host: "0" +opensearch.hosts: [ "https://os-node-1:9200" ] +opensearch.requestHeadersWhitelist: [ authorization, securitytenant ] +opensearch.username: "kibanaserver" + +# multi-tenancy +opensearch_security.multitenancy.enabled: true +opensearch_security.multitenancy.tenants.preferred: [ "Private", "Global" ] +opensearch_security.readonly_mode.roles: [ "kibana_read_only" ] + +# https +opensearch_security.cookie.secure: true +server.ssl.enabled: true +server.ssl.certificate: "/usr/share/opensearch-dashboards/config/crt.pem" +server.ssl.key: "/usr/share/opensearch-dashboards/config/key.pem" + +# CA for verification of node certificates +opensearch.ssl.certificateAuthorities: [ "/usr/share/opensearch-dashboards/config/root-ca.pem" ] diff --git a/docker-compose.base.yml b/docker-compose.base.yml new file mode 100644 index 0000000..504f965 --- /dev/null +++ b/docker-compose.base.yml @@ -0,0 +1,41 @@ +version: '3' + +services: + os-node: + image: opensearchproject/opensearch:1 + restart: "unless-stopped" + environment: + - DISABLE_INSTALL_DEMO_CONFIG=true + - node.name=os-node-1 + - discovery.seed_hosts=os-node-1,os-node-2 + - cluster.initial_master_nodes=os-node-1,os-node-2 + - bootstrap.memory_lock=true # along with the memlock settings below, disables swapping + - "OPENSEARCH_JAVA_OPTS=-Xms512m -Xmx512m" # minimum and maximum Java heap size, recommend setting both to 50% of system RAM + - OPENSEARCH_ADMIN_PASSWORD + - OPENSEARCH_KIBANASERVER_PASSWORD + - OPENSEARCH_LOGGINGUSER_PASSWORD + ulimits: + memlock: + soft: -1 + hard: -1 + nofile: + soft: 65536 # maximum number of open files for the OpenSearch user, set to at least 65536 on modern systems + hard: 65536 + volumes: + - os-node-1:/usr/share/opensearch/data + # NOTE: We map every file separately so that we don't overwrite the files created + # in the same directory from opensearch. + - ./opensearch/security/internal_users.template.yml:/usr/share/opensearch/config/internal_users.template.yml:ro + - ./opensearch/security/tenants.yml:/usr/share/opensearch/plugins/opensearch-security/securityconfig/tenants.yml:ro + - ./opensearch/opensearch.yml:/usr/share/opensearch/config/opensearch.yml:ro + - ./opensearch/init-security.sh:/usr/share/opensearch/init-security.sh:ro + # internal ssl files + - ./ssl/node1-crt.pem:/usr/share/opensearch/config/node-crt.pem:ro + - ./ssl/node1-key.pem:/usr/share/opensearch/config/node-key.pem:ro + - ./ssl/admin-crt.pem:/usr/share/opensearch/config/admin-crt.pem:ro + - ./ssl/admin-key.pem:/usr/share/opensearch/config/admin-key.pem:ro + - ./ssl/root-ca-crt.pem:/usr/share/opensearch/config/root-ca.pem:ro + # ssl for external communication + - ./ssl/cluster-crt.pem:/usr/share/opensearch/config/cluster-crt.pem:ro + - ./ssl/cluster-key.pem:/usr/share/opensearch/config/cluster-key.pem:ro + diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..558c345 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,51 @@ +version: '3' + +services: + os-node-1: + extends: + file: docker-compose.base.yml + service: os-node + ports: + - ${OPENSEARCH_PORT:-9200}:9200 + + os-node-2: + extends: + file: docker-compose.base.yml + service: os-node + environment: + - node.name=os-node-2 + volumes: + - os-node-2:/usr/share/opensearch/data + # internal ssl files + - ./ssl/node2-crt.pem:/usr/share/opensearch/config/node-crt.pem:ro + - ./ssl/node2-key.pem:/usr/share/opensearch/config/node-key.pem:ro + - ./ssl/root-ca-crt.pem:/usr/share/opensearch/config/root-ca.pem:ro + + os-dashboards: + image: opensearchproject/opensearch-dashboards:1 + restart: "unless-stopped" + environment: + # note: the dashboards entrypoint performs translations of naming for the env vars + # e.g. 'opensearch.username' is set via 'OPENSEARCH_USERNAME' + - OPENSEARCH_USERNAME=${DASHBOARDS_USERNAME:-kibanaserver} + - OPENSEARCH_PASSWORD=${DASHBOARDS_PASSWORD:-${OPENSEARCH_KIBANASERVER_PASSWORD}} + - OPENSEARCH_SSL_VERIFICATIONMODE=${DASHBOARDS_SSL_VERIFICATIONMODE:-certificate} + ports: + - ${DASHBOARDS_PORT:-443}:5601 + volumes: + # We overwrite the whole directory here since dashboards creates + # files in it that we don't need or aren't used. + - ./dashboards/config/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro + - ./ssl/dashboards-crt.pem:/usr/share/opensearch-dashboards/config/crt.pem:ro + - ./ssl/dashboards-key.pem:/usr/share/opensearch-dashboards/config/key.pem:ro + - ./ssl/root-ca-crt.pem:/usr/share/opensearch-dashboards/config/root-ca.pem:ro + depends_on: + - os-node-1 + - os-node-2 + +volumes: + os-node-1: + os-node-2: + +networks: + default: diff --git a/example.env b/example.env new file mode 100644 index 0000000..048c929 --- /dev/null +++ b/example.env @@ -0,0 +1,18 @@ +# OpenSearch nodes configuration +# ------------------------------ +OPENSEARCH_PORT=8200 +OPENSEARCH_ADMIN_PASSWORD=admin +OPENSEARCH_KIBANASERVER_PASSWORD=kibanaserverpassword +OPENSEARCH_LOGGINGUSER_PASSWORD=password + + +# OpenSearch Dashboards configuration +# ----------------------------------- +DASHBOARDS_PORT=8443 +DASHBOARDS_USERNAME=kibanaserver + +# if the password is not configured, it will fall back to ${OPENSEARCH_KIBANASERVER_PASSWORD} +DASHBOARDS_PASSWORD=kibanaserverpassword + +# verification mode values: 'full', 'certificate', 'none' +DASHBOARDS_SSL_VERIFICATIONMODE=certificate diff --git a/opensearch/init-security.sh b/opensearch/init-security.sh new file mode 100755 index 0000000..371c1ca --- /dev/null +++ b/opensearch/init-security.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# +# update the internal_users via 'securityadmin.sh' to set their passwords +# this should only be called on a new cluster +# +# usage: docker compose run os-node-1 ./init-security.sh + +set -euo pipefail + + +# get the passwords from the env vars, and hash them +sec_plugin_path="./plugins/opensearch-security" +admin_pw="${OPENSEARCH_ADMIN_PASSWORD:?admin password required}" +kibanaserver_pw="${OPENSEARCH_KIBANASERVER_PASSWORD:?kibana server password required}" +logginguser_pw="${OPENSEARCH_LOGGINGUSER_PASSWORD:?logging user password required}" + +admin_hash=$(${sec_plugin_path}/tools/hash.sh -p "${admin_pw}") +dashboards_hash=$(${sec_plugin_path}/tools/hash.sh -p "${kibanaserver_pw}") +logging_hash=$(${sec_plugin_path}/tools/hash.sh -p "${logginguser_pw}") + +# write the passwords to 'internal_users.yml' +# (the lines to be replaced are marked with comments) +# this is because the '-rev' flag for 'securityadmin.sh' does not seem to be working for 'internal_users.yml' +source="./config/internal_users.template.yml" +target="${sec_plugin_path}/securityconfig/internal_users.yml" +sed -e "s|^\(\s\+hash:\).*\s\+# ADMIN_PW|\1 ${admin_hash}|" \ + -e "s|^\(\s\+hash:\).*\s\+# DASHBOARDS_PW|\1 ${dashboards_hash}|" \ + -e "s|^\(\s\+hash:\).*\s\+# LOGGING_PW|\1 ${logging_hash}|" \ + "${source}" > "${target}" + + +# start opensearch (securityadmin.sh needs the security index) +opensearch -Eplugins.security.allow_default_init_securityindex=true -Ediscovery.type=single-node & +os_pid=$! +retries=10 + +# wait until opensearch is ready to accept connections +while [[ "${retries}" -gt 0 ]] ; do + echo "waiting for opensearch to be up... $retries" + sleep 5s + if curl -ksu "admin:admin" "https://localhost:9200"; then + break + fi + retries=$(( ${retries} - 1 )) +done + +if [[ "${retries}" -eq 0 ]]; then + echo &>2 "error: exceeded maximum number of retries!" + exit 1 +fi + +# wait for opensearch to be healthy +curl -ku "admin:admin" "https://localhost:9200/_cluster/health?wait_for_status=yellow" + + +# securityadmin needs admin certificates or a keystore to operate +"${sec_plugin_path}/tools/securityadmin.sh" \ + -rev -icl -nhnv \ + -cacert ./config/root-ca.pem \ + -cert ./config/admin-crt.pem \ + -key ./config/admin-key.pem \ + -cd "${sec_plugin_path}/securityconfig" + + +# shut down opensearch again +sleep 5s +kill ${os_pid} diff --git a/opensearch/opensearch.yml b/opensearch/opensearch.yml new file mode 100644 index 0000000..6ab4a45 --- /dev/null +++ b/opensearch/opensearch.yml @@ -0,0 +1,48 @@ +--- +# OpenSearch configuration +# +# note: we don't use certificate-based authentication + +# general configuration +cluster.name: opensearch-cluster +node.max_local_storage_nodes: 3 + +# bind to all interfaces because we don't know what IP address Docker will assign to us +network.host: 0.0.0.0 + +# don't allow demo certificates +plugins.security.allow_unsafe_democertificates: false + +# initialize security index if it isn't initialized yet: no +# that's part of the 'init-security.sh' script, which also sets the passwords +plugins.security.allow_default_init_securityindex: false + +# REST layer TLS, for communication with the outside world +plugins.security.ssl.http.enabled: true +plugins.security.ssl.http.pemkey_filepath: cluster-key.pem +plugins.security.ssl.http.pemcert_filepath: cluster-crt.pem + +# transport layer TLS, for inter-node communication +plugins.security.ssl.transport.pemkey_filepath: node-key.pem +plugins.security.ssl.transport.pemcert_filepath: node-crt.pem +plugins.security.ssl.transport.pemtrustedcas_filepath: root-ca.pem +plugins.security.ssl.transport.enforce_hostname_verification: false + +# set the known distinguished names for the nodes +# the format is used as reported by the following command: +# openssl x509 -subject -nameopt RFC2253 -noout -in CERT.pem +plugins.security.nodes_dn: + - 'CN=os-node-1,OU=Center for Research Data Management,O=TU Wien,L=Vienna,ST=Vienna,C=AT' + - 'CN=os-node-2,OU=Center for Research Data Management,O=TU Wien,L=Vienna,ST=Vienna,C=AT' + +# admin certificate for 'securityadmin.sh' +plugins.security.authcz.admin_dn: + - 'CN=admin,OU=Center for Research Data Management,O=TU Wien,L=Vienna,ST=Vienna,C=AT' + +# more security configuration +plugins.security.audit.type: internal_opensearch +plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.enable_snapshot_restore_privilege: true +plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] +plugins.security.system_indices.enabled: true +plugins.security.system_indices.indices: [".plugins-ml-model", ".plugins-ml-task", ".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opensearch-notifications-*", ".opensearch-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] diff --git a/opensearch/security/internal_users.template.yml b/opensearch/security/internal_users.template.yml new file mode 100644 index 0000000..ce5731e --- /dev/null +++ b/opensearch/security/internal_users.template.yml @@ -0,0 +1,35 @@ +--- +# This is the internal user database (for initial setup) +# +# The hash value is a bcrypt hash and can be generated with +# './plugins/opensearch-security/tools/hash.sh' (in an OS node container) +# +# note: reserved users can't be changed, except for with 'securityadmin.sh' +# +# pre-defined roles can be seen in the documentation: +# https://opensearch.org/docs/latest/security-plugin/access-control/users-roles/#predefined-roles + +_meta: + type: "internalusers" + config_version: 2 + +admin: + hash: "$2y$12$/YY6etNFBo24PUx2c8S1Duo.RwNZa6H4IhMqGsy5Cv6O5ESRJ9B3W" # ADMIN_PW + reserved: true + backend_roles: + - "admin" + description: "Administrator" + +# built-in user with the permissions for the dashboards +kibanaserver: + hash: "$2y$12$LuNY.MSV7dGzyy85c8v1oOxIRlP0TFEk6qgpGRYHZW0Qk0YVc1qdm" # DASHBOARDS_PW + reserved: true + description: "Machine account for the OpenSeaerch Dashboards server" + +# the 'logging_user' with the 'logstash' role has permissions to manage the 'logstash-*' indices +logging_user: + hash: "$2y$12$LuNY.MSV7dGzyy85c8v1oOxIRlP0TFEk6qgpGRYHZW0Qk0YVc1qdm" # LOGGING_PW + reserved: true + backend_roles: + - "logstash" + description: "Machine account for ingesting logs" diff --git a/opensearch/security/tenants.yml b/opensearch/security/tenants.yml new file mode 100644 index 0000000..60f6100 --- /dev/null +++ b/opensearch/security/tenants.yml @@ -0,0 +1,9 @@ +--- +_meta: + type: "tenants" + config_version: 2 + +# Define your tenants here +CRDM: + reserved: false + description: "Center for Research Data Management" diff --git a/scripts/generate-ssl.sh b/scripts/generate-ssl.sh new file mode 100755 index 0000000..18c3463 --- /dev/null +++ b/scripts/generate-ssl.sh @@ -0,0 +1,83 @@ +#!/bin/sh +# +# script for generating SSL keys and certificates + +set -euo pipefail + +if [[ -d "ssl" ]]; then + cd "ssl" +else + echo >&2 "error: this script needs to be executed from the project directory!" + exit 1 +fi + +# common certificate details +organization="TU Wien" +org_unit="Center for Research Data Management" +country="AT" +state="Vienna" +locality="Vienna" + +function generate_certificate { + # generate keypair + local cert_name="${1}" + local common_name="${2}" + + # generate the ssl files + subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=${common_name}" + openssl genrsa -out "${cert_name}-key-temp.pem" 2048 + openssl pkcs8 -inform PEM -outform PEM -in "${cert_name}-key-temp.pem" -topk8 -nocrypt -v1 PBE-SHA1-3DES -out "${cert_name}-key.pem" + openssl req -new -key "${cert_name}-key.pem" -subj "${subject}" -out "${cert_name}.csr" + openssl x509 -req -in "${cert_name}.csr" -CA root-ca-crt.pem -CAkey root-ca-key.pem -CAcreateserial -sha256 -out "${cert_name}-crt.pem" -days 365 + + # clean up temporary files + rm "${cert_name}-key-temp.pem" "${cert_name}.csr" +} + +function generate_if_not_user_defined { + # generate ssl keypair if it wasn't explicitly set by an operator + local cert_name="${1}" + local common_name="${2}" + local key_file="${cert_name}-key.pem" + local cert_file="${cert_name}-crt.pem" + + if [[ -f "${key_file}" && ! -L "${key_file}" && -f "${cert_file}" && ! -L "${cert_file}" ]]; then + # if both files exist, we assume they were placed here by the operator + echo "info: keeping keypair for ${cert_name}." + + elif [[ ! -e "${key_file}" || -L "${key_file}" || ! -e "${cert_file}" || -L "${cert_file}" ]]; then + # if the files are symlinks, we assume they were auto-generated by the script + # similar if they don't exist + unlink "${key_file}" &>/dev/null || true + unlink "${cert_file}" &>/dev/null || true + generate_certificate "${cert_name}-gen" "${common_name}" + ln -s "${cert_name}-gen-key.pem" "${key_file}" + ln -s "${cert_name}-gen-crt.pem" "${cert_file}" + + else + # maybe the files got turned into directories by the bind mounts? + echo "info: something weird is going on with the files ${cert_name}-{key,crt}.pem" + echo "> please investigate manually." + fi +} + + +# Root CA +subject="/C=${country}/ST=${state}/L=${locality}/O=${organization}/OU=${org_unit}/CN=ROOT" +openssl genrsa -out root-ca-key.pem 2048 +openssl req -new -x509 -sha256 -key root-ca-key.pem -subj "${subject}" -out root-ca-crt.pem -days 365 + +# Admin certificate (for securityadmin.sh) +generate_certificate "admin" "admin" + +# Node 1 cert (for internal communication) +generate_certificate "node1" "os-node-1" + +# Node 2 cert (for internal communication) +generate_certificate "node2" "os-node-2" + +# Dashboards cert (for outside communication) +generate_if_not_user_defined "dashboards" "${OPENSEARCH_HOSTNAME:-localhost}" + +# REST HTTPS cert (for outside communication) +generate_if_not_user_defined "cluster" "${DASHBOARDS_HOSTNAME:-localhost}" diff --git a/scripts/setup.sh b/scripts/setup.sh new file mode 100755 index 0000000..7977d4a --- /dev/null +++ b/scripts/setup.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# +# script for setting up and running the log server + +set -euo pipefail + +if [[ ! -d "ssl" || ! -f "docker-compose.yml" || ! -f "docker-compose.base.yml" ]]; then + echo >&2 "error: this script needs to be executed from the project directory!" + exit 1 +fi + +# generating the ssl key pairs +./scripts/generate-ssl.sh + +# initialize the security configuration +docker compose run os-node-1 ./init-security.sh + +# run the server +docker compose up -d diff --git a/ssl/.gitkeep b/ssl/.gitkeep new file mode 100644 index 0000000..e69de29 -- GitLab