diff --git a/docker-compose.yml b/docker-compose.yml index 95d98f56127c07f4f4c06b46964a036700e944ee..94c8cab30dd752694a45cb75749421acc6c34b13 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -33,8 +33,6 @@ services: ports: - ${DASHBOARDS_PORT:-443}:5601 volumes: - # We overwrite the whole directory here since dashboards creates - # files in it that we don't need or aren't used. - ./dashboards/config/opensearch_dashboards.yml:/usr/share/opensearch-dashboards/config/opensearch_dashboards.yml:ro - ./ssl/dashboards-crt.pem:/usr/share/opensearch-dashboards/config/crt.pem:ro - ./ssl/dashboards-key.pem:/usr/share/opensearch-dashboards/config/key.pem:ro @@ -43,9 +41,31 @@ services: - os-node-1 - os-node-2 + metricbeat: + # Beats versions newer than 7.12.x are not supported by OpenSearch + image: docker.elastic.co/beats/metricbeat-oss:7.12.1 + restart: "unless-stopped" + user: root + environment: + - METRICBEAT_INDEX_INFIX=${METRICBEAT_INDEX_INFIX:-dev} + - METRICBEAT_INTERVAL=${METRICBEAT_INTERVAL:-10s} + - METRICBEAT_OS_HOST=${METRICBEAT_OS_HOST:-https://os-node-1} + - METRICBEAT_OS_USER=${METRICBEAT_OS_USER:-admin} + - METRICBEAT_OS_PASSWORD=${METRICBEAT_OS_PASSWORD:-admin} + volumes: + - metricbeat-data:/usr/share/metricbeat/data + - /var/run/docker.sock:/var/run/docker.sock:ro + - /sys/fs/cgroup:/hostfs/sys/fs/cgroup:ro + - /proc:/hostfs/proc:ro + - /:/hostfs:ro + - ./metricbeat/metricbeat.yml:/usr/share/metricbeat/metricbeat.yml:ro + depends_on: + - os-node-1 + volumes: - os-node-1: - os-node-2: + os-node-1: + os-node-2: + metricbeat-data: networks: default: diff --git a/example.env b/example.env index bea7c134497b324bb593992a22df42a01463ff63..eb4731f2c111458acb978a86af5b16d03b984f7c 100644 --- a/example.env +++ b/example.env @@ -17,3 +17,19 @@ DASHBOARDS_PASSWORD=kibanaserverpassword # verification mode values: 'full', 'certificate', 'none' DASHBOARDS_SSL_VERIFICATIONMODE=certificate + +# Metricbeat configuration +# ----------------------------------- + +# for index naming: to differentiate between deployment environments if stats are sent to the +# same opensearch node (e.g. dev, staging, prod) +METRICBEAT_INDEX_INFIX=dev + +# stats collection interval +METRICBEAT_INTERVAL=5s + +METRICBEAT_OS_HOST=https://os-node-1 + +# credentials must match the ones from opensearch +METRICBEAT_OS_USER=admin +METRICBEAT_OS_PASSWORD=super_secure_password diff --git a/metricbeat/metricbeat.yml b/metricbeat/metricbeat.yml new file mode 100644 index 0000000000000000000000000000000000000000..de832cbef604481074c8eb8db3cb8d18a0670864 --- /dev/null +++ b/metricbeat/metricbeat.yml @@ -0,0 +1,48 @@ +# ########################## Metricbeat Configuration ########################## # + +# ============================ Config Reloading =============================== # +metricbeat.config: + modules: + path: ${path.config}/modules.d/*.yml + # Do not reload module configs as they change + reload.enabled: false + +# ============================== Autodiscover ================================== # +metricbeat.autodiscover: + providers: + - type: docker + hints.enabled: true + +# ========================== Modules configuration ============================ # +metricbeat.modules: + +# -------------------------------- System Module ------------------------------- # +- module: system + metricsets: + - "cpu" # CPU usage + - "filesystem" # File system usage for each mountpoint + - "memory" # Memory usage + period: ${METRICBEAT_INTERVAL} + enabled: true + +# -------------------------------- Docker Module -------------------------------- # +- module: docker + metricsets: + - "container" + hosts: ["unix:///var/run/docker.sock"] + period: ${METRICBEAT_INTERVAL} + enabled: true + +# ================================== Template =================================== # +setup.template.name: "metricbeat-${METRICBEAT_INDEX_INFIX}-%{[agent.version]}" +setup.template.pattern: "metricbeat-${METRICBEAT_INDEX_INFIX}-%{[agent.version]}-*" + +# =================================== Outputs =================================== # + +# ---------------------------- Elasticsearch Output ----------------------------- # +output.elasticsearch: + hosts: "${METRICBEAT_OS_HOST}" + username: "${METRICBEAT_OS_USER}" + password: "${METRICBEAT_OS_PASSWORD}" + ssl.verification_mode: none + index: "metricbeat-${METRICBEAT_INDEX_INFIX}-v%{[agent.version]}-%{+yyyy.MM.dd}" diff --git a/opensearch/opensearch.yml b/opensearch/opensearch.yml index 6ab4a455004836e41fcdb01d518ce52d303e6910..ce4348e8692b685d755fa58ef128000630a8bf95 100644 --- a/opensearch/opensearch.yml +++ b/opensearch/opensearch.yml @@ -5,6 +5,11 @@ # general configuration cluster.name: opensearch-cluster + +# intermediate compatibility solution +# return version 7.10.2 rather than actual OS version to "trick" metricbeat +compatibility.override_main_response_version: true + node.max_local_storage_nodes: 3 # bind to all interfaces because we don't know what IP address Docker will assign to us