From 3823c70170b0109dae56aee99d0c1a0951317ee9 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Mon, 8 Aug 2022 18:28:18 +0530 Subject: [PATCH 1/4] Add couchbase intergration with resource data stream --- packages/couchbase/_dev/build/docs/README.md | 19 +- .../_dev/deploy/docker/docker-compose.yml | 30 ++- .../_dev/deploy/docker/files/Dockerfile | 13 +- .../deploy/docker/files/configure-node.sh | 27 --- .../docker/files/sync-gateway-config.txt | 175 ++++++++++++++++++ packages/couchbase/changelog.yml | 3 + .../_dev/test/pipeline/test-common-config.yml | 2 + .../test/pipeline/test-resource-metrics.json | 39 ++++ .../test-resource-metrics.json-expected.json | 67 +++++++ .../_dev/test/system/test-default-config.yml | 7 + .../resource/agent/stream/stream.yml.hbs | 23 +++ .../elasticsearch/ingest_pipeline/default.yml | 133 +++++++++++++ .../resource/fields/base-fields.yml | 12 ++ .../data_stream/resource/fields/ecs.yml | 26 +++ .../data_stream/resource/fields/fields.yml | 79 ++++++++ .../data_stream/resource/manifest.yml | 31 ++++ .../data_stream/resource/sample_event.json | 110 +++++++++++ packages/couchbase/docs/README.md | 165 ++++++++++++++++- packages/couchbase/manifest.yml | 47 +++++ 19 files changed, 967 insertions(+), 41 deletions(-) delete mode 100644 packages/couchbase/_dev/deploy/docker/files/configure-node.sh create mode 100644 packages/couchbase/_dev/deploy/docker/files/sync-gateway-config.txt create mode 100644 packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json create mode 100644 packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json create mode 100644 packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml create mode 100644 packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs create mode 100644 packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/couchbase/data_stream/resource/fields/base-fields.yml create mode 100644 packages/couchbase/data_stream/resource/fields/ecs.yml create mode 100644 packages/couchbase/data_stream/resource/fields/fields.yml create mode 100644 packages/couchbase/data_stream/resource/manifest.yml create mode 100644 packages/couchbase/data_stream/resource/sample_event.json diff --git a/packages/couchbase/_dev/build/docs/README.md b/packages/couchbase/_dev/build/docs/README.md index 57229519699..24f3c50d162 100644 --- a/packages/couchbase/_dev/build/docs/README.md +++ b/packages/couchbase/_dev/build/docs/README.md @@ -1,8 +1,10 @@ # Couchbase Integration -This Elastic integration collects and parses the [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. +This Elastic integration collects and parses the [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html) and [Resource Utilization](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#resource_utilization) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. -This integration uses `http` metricbeat module to collect `bucket` metrics. +This integration uses: +- `http` metricbeat module to collect `bucket` metrics. +- `prometheus` metricbeat module to collect `resource` metrics. Note: For Couchbase cluster setup, there is an ideal scenario of single host with administrator access for the entire cluster to collect metrics. Providing multiple host from the same cluster might lead to data duplication. In case of multiple clusters, adding a new integration to collect data from different cluster host is a good option. @@ -18,6 +20,11 @@ Host Configuration Format: `http[s]://username:password@host:port` Example Host Configuration: `http://Administrator:password@localhost:8091` +In order to collect data using [Sync Gateway](https://www.couchbase.com/products/sync-gateway), follow the steps given below: +- Download and configure [Sync Gateway](https://docs.couchbase.com/sync-gateway/current/get-started-install.html) +- Download and configure [Sync Gateway Promethus Exporter](https://github.com/couchbaselabs/couchbase-sync-gateway-exporter.git) and provide Sync Gateway Host using --sgw.url flag while running the Exporter App +- Example configuration: `--sgw.url=http://sgw:4985` + ## Metrics ### Bucket @@ -27,3 +34,11 @@ This is the `bucket` data stream. A bucket is a logical container for a related {{event "bucket"}} {{fields "bucket"}} + +### Resource Utilization + +This is the `resource` data stream. The Resource Utilization metrics are related to [MemStats](https://golang.org/pkg/runtime/#MemStats) records statistics about the memory allocator. + +{{event "resource"}} + +{{fields "resource"}} diff --git a/packages/couchbase/_dev/deploy/docker/docker-compose.yml b/packages/couchbase/_dev/deploy/docker/docker-compose.yml index 89c5acee457..97d24af8081 100644 --- a/packages/couchbase/_dev/deploy/docker/docker-compose.yml +++ b/packages/couchbase/_dev/deploy/docker/docker-compose.yml @@ -1,10 +1,32 @@ version: '2.3' services: couchbase: + image: couchbase/server-sandbox:${COUCHBASE_VERSION:-7.1.0} + hostname: cb + healthcheck: + test: ["CMD", "curl", "-f", "http://admin:password@localhost:8091/pools/default"] + interval: 2s + timeout: 2s + retries: 40 + ports: + - 8091 + sgw: + image: couchbase/sync-gateway:2.8.0-enterprise + hostname: sgw + depends_on: + couchbase: + condition: service_healthy + volumes: + - ./files/sync-gateway-config.txt:/etc/sync_gateway/sync_gateway.json + command: -adminInterface :4985 /etc/sync_gateway/sync_gateway.json + ports: + - 4985 + exporter: build: context: ./files/ - args: - COUCHBASE_VERSION: ${COUCHBASE_VERSION:-7.1.0} - hostname: couchbase + dockerfile: Dockerfile ports: - - 8091 + - 9421 + depends_on: + sgw: + condition: service_started diff --git a/packages/couchbase/_dev/deploy/docker/files/Dockerfile b/packages/couchbase/_dev/deploy/docker/files/Dockerfile index a6ecc40469e..5d67cc15371 100644 --- a/packages/couchbase/_dev/deploy/docker/files/Dockerfile +++ b/packages/couchbase/_dev/deploy/docker/files/Dockerfile @@ -1,6 +1,7 @@ -ARG COUCHBASE_VERSION -FROM couchbase:${COUCHBASE_VERSION} -HEALTHCHECK --interval=1s --retries=90 CMD [ "$(curl -s -o /dev/null -w ''%{http_code}'' -u Administrator:password http://localhost:8091/pools/default/buckets/beer-sample)" -eq "200" ] -COPY configure-node.sh /opt/couchbase -RUN chmod -R 777 /opt/couchbase/configure-node.sh -CMD ["/opt/couchbase/configure-node.sh"] +FROM golang:1.18 +WORKDIR /usr/src/ +RUN git clone https://github.com/couchbaselabs/couchbase-sync-gateway-exporter.git +WORKDIR /usr/src/couchbase-sync-gateway-exporter/ +RUN git checkout 9272038aae7fd46575befda06449f008f41b6b01 +RUN go build -v -o /usr/local/bin/app . +CMD ["app","--log.level=debug","--sgw.url=http://sgw:4985"] diff --git a/packages/couchbase/_dev/deploy/docker/files/configure-node.sh b/packages/couchbase/_dev/deploy/docker/files/configure-node.sh deleted file mode 100644 index f38eddfd49d..00000000000 --- a/packages/couchbase/_dev/deploy/docker/files/configure-node.sh +++ /dev/null @@ -1,27 +0,0 @@ -set -e -set -m -/entrypoint.sh couchbase-server & -sleep 1 -waitForCouchbase() { - echo -n "Waiting for Couchbase to start." - for ((i=1; i<=90; i++)) do - if nc -vz 127.0.0.1 8091 2>/dev/null; then - return 0 - fi - ((i++)) - echo -n '.' - sleep 1 - done - echo - echo >&2 "Failed to Start Couchbase" -} -waitForCouchbase -# Setup index and memory quota -curl -v -X POST http://127.0.0.1:8091/pools/default -d memoryQuota=300 -d indexMemoryQuota=300 -# Setup services -curl -v http://127.0.0.1:8091/node/controller/setupServices -d services=kv%2Cn1ql%2Cindex -# Setup credentials -curl -v http://127.0.0.1:8091/settings/web -d port=8091 -d username=Administrator -d password=password -# Load beer-sample bucket -curl -v -u Administrator:password -X POST http://127.0.0.1:8091/sampleBuckets/install -d '["beer-sample"]' -fg 1 diff --git a/packages/couchbase/_dev/deploy/docker/files/sync-gateway-config.txt b/packages/couchbase/_dev/deploy/docker/files/sync-gateway-config.txt new file mode 100644 index 00000000000..4a301acaaab --- /dev/null +++ b/packages/couchbase/_dev/deploy/docker/files/sync-gateway-config.txt @@ -0,0 +1,175 @@ +{ + "interface":":4984", + "log": ["*"], + "logging": { + "log_file_path": "/var/tmp/sglogs", + "console": { + "log_level": "debug", + "log_keys": ["*"] + }, + "error": { + "enabled": true, + "rotation": { + "max_size": 20, + "max_age": 180 + } + }, + "warn": { + "enabled": true, + "rotation": { + "max_size": 20, + "max_age": 90 + } + }, + "info": { + "enabled": false + }, + "debug": { + "enabled": false + } + }, + "databases": { + "travel-sample": { + "import_docs": "continuous", + "bucket":"travel-sample", + "server": "http://cb:8091", + "enable_shared_bucket_access":true, + "delta_sync": { + "enabled":true, + "rev_max_age_seconds": 86400 + }, + "import_filter": ` + function(doc) { + return true; + } + `, + "username": "admin", + "password": "password", + "users":{ + "GUEST": { "disabled": false, "admin_channels": ["*"] }, + "admin": {"password": "password", "admin_channels": ["*"]}, + "demo": {"password": "password"}, + "tester": {"password": "password"} + }, + "num_index_replicas":0, + "cache" :{ + "channel_cache":{ + "max_number":50000 + }, + "rev_cache" :{ + "size":5000 + } + }, + "sync": ` +function sync(doc, oldDoc) { + /* sanity check */ + // check if document was removed from server or via SDK + // In this case, just return + if (isRemoved()) { + return; + } + if (doc.channels ) { + /* Routing */ + // Add doc to the channels specified in the document. + channel(doc.channels); + } + function isCreate() { + // Checking false for the Admin UI to work + return ((oldDoc == false) || (oldDoc == null || oldDoc._deleted) && !isDelete()); + } + function isUpdate() { + return (!isCreate() && !isDelete()); + } + // This is when document is removed via SDK or directly on server + function isRemoved() { + return( isDelete() && oldDoc == null); + } + function isDelete() { + return (doc._deleted == true); + } + function validateNotEmpty(key, value) { + if (!value) { + throw({forbidden: key + " is not provided."}); + } + } + function validateReadOnly(name, value, oldValue) { + if (value != oldValue) { + throw({forbidden: name + " is read-only."}); + } + } +} + ` + }, + "beer-sample": { + "import_docs": "continuous", + "bucket":"travel-sample", + "server": "http://cb:8091", + "enable_shared_bucket_access":true, + "delta_sync": { + "enabled":true, + "rev_max_age_seconds": 86400 + }, + "import_filter": ` + function(doc) { + return true; + } + `, + "username": "admin", + "password": "password", + "users":{ + "GUEST": { "disabled": false, "admin_channels": ["*"] }, + "admin": {"password": "password", "admin_channels": ["*"]}, + "demo": {"password": "password"}, + "tester": {"password": "password"} + }, + "num_index_replicas":0, + "cache" :{ + "channel_cache":{ + "max_number":50000 + }, + "rev_cache" :{ + "size":5000 + } + }, + "sync": ` +function sync(doc, oldDoc) { + /* sanity check */ + // check if document was removed from server or via SDK + // In this case, just return + if (isRemoved()) { + return; + } + if (doc.channels ) { + /* Routing */ + // Add doc to the channels specified in the document. + channel(doc.channels); + } + function isCreate() { + // Checking false for the Admin UI to work + return ((oldDoc == false) || (oldDoc == null || oldDoc._deleted) && !isDelete()); + } + function isUpdate() { + return (!isCreate() && !isDelete()); + } + // This is when document is removed via SDK or directly on server + function isRemoved() { + return( isDelete() && oldDoc == null); + } + function isDelete() { + return (doc._deleted == true); + } + function validateNotEmpty(key, value) { + if (!value) { + throw({forbidden: key + " is not provided."}); + } + } + function validateReadOnly(name, value, oldValue) { + if (value != oldValue) { + throw({forbidden: name + " is read-only."}); + } + } +} + ` + } + } +} \ No newline at end of file diff --git a/packages/couchbase/changelog.yml b/packages/couchbase/changelog.yml index 080741d32ec..269cfcbd639 100644 --- a/packages/couchbase/changelog.yml +++ b/packages/couchbase/changelog.yml @@ -1,6 +1,9 @@ # newer versions go on top - version: "0.1.0" changes: + - description: Couchbase integration package with "resource" data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link - description: Couchbase integration package with "bucket" data stream. type: enhancement link: https://github.com/elastic/integrations/pull/3666 diff --git a/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json new file mode 100644 index 00000000000..bfd513eff82 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json @@ -0,0 +1,39 @@ +{ + "events": [ + { + "@timestamp": "2022-07-15T06:38:37.166Z", + "service": { + "address": "http://localhost:9421/metrics", + "type": "prometheus" + }, + "prometheus": { + "labels": { + "instance": "localhost:9421", + "job": "prometheus" + }, + "metrics": { + "sgw_resource_utilization_goroutines_high_watermark": 297, + "sgw_resource_utilization_warn_count": 9143, + "sgw_resource_utilization_go_memstats_heapinuse": 3.97533184e+08, + "sgw_resource_utilization_process_memory_resident": 5.77753088e+08, + "sgw_resource_utilization_go_memstats_heapreleased": 1.970176e+07, + "sgw_resource_utilization_go_memstats_sys": 5.7015116e+08, + "sgw_resource_utilization_go_memstats_heapalloc": 3.31451256e+08, + "sgw_resource_utilization_system_memory_total": 1.6637505536e+10, + "sgw_resource_utilization_go_memstats_stacksys": 3.76832e+06, + "sgw_resource_utilization_go_memstats_heapidle": 1.35569408e+08, + "sgw_resource_utilization_pub_net_bytes_sent": 1.19484151e+08, + "sgw_resource_utilization_num_goroutines": 240, + "sgw_resource_utilization_process_cpu_percent_utilization": 11.489248142477914, + "sgw_resource_utilization_go_memstats_stackinuse": 3.76832e+06, + "sgw_resource_utilization_pub_net_bytes_recv": 3.72908077e+08, + "sgw_resource_utilization_admin_net_bytes_sent": 1.19484151e+08, + "sgw_resource_utilization_admin_net_bytes_recv": 3.72908077e+08, + "sgw_resource_utilization_go_memstats_pausetotalns": 1.2177207e+08, + "sgw_resource_utilization_error_count": 0, + "go_memstats_last_gc_time_seconds": 1.657867123417899 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json new file mode 100644 index 00000000000..2b77cc32499 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json @@ -0,0 +1,67 @@ +{ + "expected": [ + { + "@timestamp": "2022-07-15T06:38:37.166Z", + "couchbase": { + "resource": { + "admin_net": { + "bytes": { + "received": 3.72908077E8, + "sent": 1.19484151E8 + } + }, + "error": { + "count": 0 + }, + "go_memstats": { + "heap": { + "alloc": 3.31451256E8, + "idle": 1.35569408E8, + "in_use": 3.97533184E8, + "released": 1.970176E7 + }, + "stack": { + "in_use": 3768320.0 + } + }, + "last_gc": 1.657867123417899, + "process": { + "cpu": { + "pct": 11.489248142477914 + }, + "memory": { + "resident": 5.77753088E8 + } + }, + "warn": { + "count": 9143 + } + } + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "database" + ], + "ingested": "2022-08-04T09:09:14.585647824Z", + "kind": "metric", + "module": "couchbase", + "type": [ + "info" + ] + }, + "server": { + "address": "localhost:9421" + }, + "service": { + "address": "http://localhost:9421/metrics", + "type": "prometheus" + }, + "tags": [ + "prometheus" + ] + } + ] +} \ No newline at end of file diff --git a/packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml b/packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..70044322a09 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: exporter +input: prometheus/metrics +vars: + hosts: + - http://Administrator:password@{{Hostname}}:{{Ports}} +data_stream: + vars: ~ diff --git a/packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs b/packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..ad519755fb1 --- /dev/null +++ b/packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs @@ -0,0 +1,23 @@ +metricsets: ["collector"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +metrics_path: /metrics +metrics_filters: + include: ["sgw_resource_utilization*","go_memstats_last_gc_time_seconds"] +{{#if ssl}} +{{ssl}} +{{/if}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml b/packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..55478ef075f --- /dev/null +++ b/packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,133 @@ +--- +description: Pipeline for parsing Couchbase Sync Gateway Resource Utilization metrics. +processors: + - set: + field: ecs.version + value: 8.3.0 + ignore_failure: true + ignore_empty_value: true + - set: + field: event.type + value: [info] + ignore_failure: true + ignore_empty_value: true + - set: + field: event.kind + value: metric + ignore_failure: true + ignore_empty_value: true + - set: + field: event.category + value: [database] + ignore_failure: true + ignore_empty_value: true + - set: + field: event.ingested + value: "{{{_ingest.timestamp}}}" + ignore_failure: true + ignore_empty_value: true + - set: + field: event.module + value: couchbase + override: true + ignore_failure: true + ignore_empty_value: true + - rename: + field: prometheus.labels.instance + target_field: server.address + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_process_cpu_percent_utilization + target_field: couchbase.resource.process.cpu.pct + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_process_memory_resident + target_field: couchbase.resource.process.memory.resident + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_admin_net_bytes_recv + target_field: couchbase.resource.admin_net.bytes.received + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_admin_net_bytes_sent + target_field: couchbase.resource.admin_net.bytes.sent + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapalloc + target_field: couchbase.resource.go_memstats.heap.alloc + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapinuse + target_field: couchbase.resource.go_memstats.heap.in_use + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapidle + target_field: couchbase.resource.go_memstats.heap.idle + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapreleased + target_field: couchbase.resource.go_memstats.heap.released + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_stackinuse + target_field: couchbase.resource.go_memstats.stack.in_use + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_error_count + target_field: couchbase.resource.error.count + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_warn_count + target_field: couchbase.resource.warn.count + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.go_memstats_last_gc_time_seconds + target_field: couchbase.resource.last_gc + ignore_missing: true + ignore_failure: true + - script: + description: Add job to tag list. + lang: painless + ignore_failure: true + source: | + if (ctx.tags == null) { + ctx.tags = new ArrayList(); + } + ctx.tags.add(ctx.prometheus.labels.job) + - remove: + field: prometheus + ignore_failure: true + ignore_missing: true + - script: + description: Drops null/empty values recursively. + lang: painless + source: | + boolean drop(Object o) { + if (o == null || o == "") { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" diff --git a/packages/couchbase/data_stream/resource/fields/base-fields.yml b/packages/couchbase/data_stream/resource/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/couchbase/data_stream/resource/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/couchbase/data_stream/resource/fields/ecs.yml b/packages/couchbase/data_stream/resource/fields/ecs.yml new file mode 100644 index 00000000000..5ce04c4dc40 --- /dev/null +++ b/packages/couchbase/data_stream/resource/fields/ecs.yml @@ -0,0 +1,26 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.dataset +- external: ecs + name: event.duration +- external: ecs + name: event.ingested +- external: ecs + name: event.kind +- external: ecs + name: event.module +- external: ecs + name: event.type +- external: ecs + name: server.address +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: tags diff --git a/packages/couchbase/data_stream/resource/fields/fields.yml b/packages/couchbase/data_stream/resource/fields/fields.yml new file mode 100644 index 00000000000..4206407be22 --- /dev/null +++ b/packages/couchbase/data_stream/resource/fields/fields.yml @@ -0,0 +1,79 @@ +- name: couchbase + type: group + fields: + - name: resource + type: group + fields: + - name: admin_net + type: group + fields: + - name: bytes + type: group + fields: + - name: received + type: scaled_float + description: The total number of bytes received (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. + unit: byte + metric_type: gauge + - name: sent + type: scaled_float + description: The total number of bytes sent (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. + unit: byte + metric_type: gauge + - name: error.count + type: long + description: The total number of errors logged. + metric_type: counter + - name: go_memstats + type: group + fields: + - name: heap + type: group + fields: + - name: alloc + type: scaled_float + description: Bytes of allocated heap objects. + unit: byte + metric_type: gauge + - name: idle + type: scaled_float + description: Bytes in idle (unused) spans. + unit: byte + metric_type: gauge + - name: in_use + type: scaled_float + description: Bytes in in-use spans. + unit: byte + metric_type: gauge + - name: released + type: scaled_float + description: Bytes of physical memory returned to the OS. + unit: byte + metric_type: gauge + - name: stack.in_use + type: scaled_float + description: Bytes in stack spans. + unit: byte + metric_type: gauge + - name: last_gc + type: scaled_float + description: The time the last garbage collection finished, as nanoseconds since 1970 (the UNIX epoch). + unit: nanos + metric_type: gauge + - name: process + type: group + fields: + - name: cpu.pct + type: scaled_float + description: The CPU’s utilization as percentage value. + unit: percent + metric_type: gauge + - name: memory.resident + type: scaled_float + description: The memory utilization (Resident Set Size) for the process, in bytes. + unit: byte + metric_type: gauge + - name: warn.count + type: long + description: The total number of warnings logged. + metric_type: counter diff --git a/packages/couchbase/data_stream/resource/manifest.yml b/packages/couchbase/data_stream/resource/manifest.yml new file mode 100644 index 00000000000..ef7ca4bdf10 --- /dev/null +++ b/packages/couchbase/data_stream/resource/manifest.yml @@ -0,0 +1,31 @@ +title: Couchbase Sync Gateway Resource Utilization metrics. +type: metrics +streams: + - input: prometheus/metrics + template_path: stream.yml.hbs + title: Couchbase Sync Gateway Resource Utilization metrics. + description: This data stream collects Resource Utilization metrics using sync gateway prometheus exporter. + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: tags + type: text + title: Tags + multi: true + required: false + show_user: false + default: + - forwarded + - couchbase-resource + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/couchbase/data_stream/resource/sample_event.json b/packages/couchbase/data_stream/resource/sample_event.json new file mode 100644 index 00000000000..c3578a7b2c2 --- /dev/null +++ b/packages/couchbase/data_stream/resource/sample_event.json @@ -0,0 +1,110 @@ +{ + "@timestamp": "2022-08-01T16:19:43.614Z", + "agent": { + "ephemeral_id": "fd354a8c-e906-40b4-9828-a806f6e5a747", + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.3.2" + }, + "couchbase": { + "resource": { + "admin_net": { + "bytes": { + "received": 0, + "sent": 0 + } + }, + "error": { + "count": 0 + }, + "go_memstats": { + "heap": { + "alloc": 0, + "idle": 0, + "in_use": 0, + "released": 0 + }, + "stack": { + "in_use": 0 + } + }, + "last_gc": 1.66, + "process": { + "cpu": { + "pct": 0 + }, + "memory": { + "resident": 0 + } + }, + "warn": { + "count": 14 + } + } + }, + "data_stream": { + "dataset": "couchbase.resource", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.3.0" + }, + "elastic_agent": { + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "database" + ], + "dataset": "couchbase.resource", + "duration": 24160284, + "ingested": "2022-08-01T16:19:47Z", + "kind": "metric", + "module": "couchbase", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "ip": [ + "172.31.0.6" + ], + "mac": [ + "02:42:ac:1f:00:06" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.4.0-110-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "server": { + "address": "elastic-package-service_exporter_1:9421" + }, + "service": { + "address": "http://elastic-package-service_exporter_1:9421/metrics", + "type": "prometheus" + }, + "tags": [ + "forwarded", + "couchbase-resource", + "prometheus" + ] +} \ No newline at end of file diff --git a/packages/couchbase/docs/README.md b/packages/couchbase/docs/README.md index 40e8092005c..4603607db8e 100644 --- a/packages/couchbase/docs/README.md +++ b/packages/couchbase/docs/README.md @@ -1,8 +1,10 @@ # Couchbase Integration -This Elastic integration collects and parses the [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. +This Elastic integration collects and parses the [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html) and [Resource Utilization](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#resource_utilization) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. -This integration uses `http` metricbeat module to collect `bucket` metrics. +This integration uses: +- `http` metricbeat module to collect `bucket` metrics. +- `prometheus` metricbeat module to collect `resource` metrics. Note: For Couchbase cluster setup, there is an ideal scenario of single host with administrator access for the entire cluster to collect metrics. Providing multiple host from the same cluster might lead to data duplication. In case of multiple clusters, adding a new integration to collect data from different cluster host is a good option. @@ -18,6 +20,11 @@ Host Configuration Format: `http[s]://username:password@host:port` Example Host Configuration: `http://Administrator:password@localhost:8091` +In order to collect data using [Sync Gateway](https://www.couchbase.com/products/sync-gateway), follow the steps given below: +- Download and configure [Sync Gateway](https://docs.couchbase.com/sync-gateway/current/get-started-install.html) +- Download and configure [Sync Gateway Promethus Exporter](https://github.com/couchbaselabs/couchbase-sync-gateway-exporter.git) and provide Sync Gateway Host using --sgw.url flag while running the Exporter App +- Example configuration: `--sgw.url=http://sgw:4985` + ## Metrics ### Bucket @@ -164,3 +171,157 @@ An example event for `bucket` looks as following: | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | | tags | List of keywords used to tag each event. | keyword | | | + +### Resource Utilization + +This is the `resource` data stream. The Resource Utilization metrics are related to [MemStats](https://golang.org/pkg/runtime/#MemStats) records statistics about the memory allocator. + +An example event for `resource` looks as following: + +```json +{ + "@timestamp": "2022-08-01T16:19:43.614Z", + "agent": { + "ephemeral_id": "fd354a8c-e906-40b4-9828-a806f6e5a747", + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.3.2" + }, + "couchbase": { + "resource": { + "admin_net": { + "bytes": { + "received": 0, + "sent": 0 + } + }, + "error": { + "count": 0 + }, + "go_memstats": { + "heap": { + "alloc": 0, + "idle": 0, + "in_use": 0, + "released": 0 + }, + "stack": { + "in_use": 0 + } + }, + "last_gc": 1.66, + "process": { + "cpu": { + "pct": 0 + }, + "memory": { + "resident": 0 + } + }, + "warn": { + "count": 14 + } + } + }, + "data_stream": { + "dataset": "couchbase.resource", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.3.0" + }, + "elastic_agent": { + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "database" + ], + "dataset": "couchbase.resource", + "duration": 24160284, + "ingested": "2022-08-01T16:19:47Z", + "kind": "metric", + "module": "couchbase", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "ip": [ + "172.31.0.6" + ], + "mac": [ + "02:42:ac:1f:00:06" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.4.0-110-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "server": { + "address": "elastic-package-service_exporter_1:9421" + }, + "service": { + "address": "http://elastic-package-service_exporter_1:9421/metrics", + "type": "prometheus" + }, + "tags": [ + "forwarded", + "couchbase-resource", + "prometheus" + ] +} +``` + +**Exported fields** + +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Event timestamp. | date | | | +| couchbase.resource.admin_net.bytes.received | The total number of bytes received (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. | scaled_float | byte | gauge | +| couchbase.resource.admin_net.bytes.sent | The total number of bytes sent (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. | scaled_float | byte | gauge | +| couchbase.resource.error.count | The total number of errors logged. | long | | counter | +| couchbase.resource.go_memstats.heap.alloc | Bytes of allocated heap objects. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.heap.idle | Bytes in idle (unused) spans. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.heap.in_use | Bytes in in-use spans. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.heap.released | Bytes of physical memory returned to the OS. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.stack.in_use | Bytes in stack spans. | scaled_float | byte | gauge | +| couchbase.resource.last_gc | The time the last garbage collection finished, as nanoseconds since 1970 (the UNIX epoch). | scaled_float | nanos | gauge | +| couchbase.resource.process.cpu.pct | The CPU’s utilization as percentage value. | scaled_float | percent | gauge | +| couchbase.resource.process.memory.resident | The memory utilization (Resident Set Size) for the process, in bytes. | scaled_float | byte | gauge | +| couchbase.resource.warn.count | The total number of warnings logged. | long | | counter | +| data_stream.dataset | Data stream dataset. | constant_keyword | | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | | +| data_stream.type | Data stream type. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | | +| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | +| tags | List of keywords used to tag each event. | keyword | | | + diff --git a/packages/couchbase/manifest.yml b/packages/couchbase/manifest.yml index 390dc65b771..161908bfce4 100644 --- a/packages/couchbase/manifest.yml +++ b/packages/couchbase/manifest.yml @@ -23,5 +23,52 @@ policy_templates: - type: http/metrics title: Collect Couchbase metrics using http module description: Collect Couchbase metrics using http module. + - type: prometheus/metrics + title: Collect Couchbase Sync Gateway metrics using prometheus module. + description: Collect Couchbase Sync Gateway metrics using prometheus module. + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + description: "Host for Sync Gateway promethus exporter (example: http://username:password@localhost:9421)." + default: + - http://username:password@localhost:9421 + - name: ssl + type: yaml + title: SSL Configuration + default: | + # ssl.certificate_authorities: + # - | + # -----BEGIN CERTIFICATE----- + # MIID+jCCAuKgAwIBAgIGAJJMzlxLMA0GCSqGSIb3DQEBCwUAMHoxCzAJBgNVBAYT + # AlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHROb2RlMDExFjAUBgNV + # BAsTDURlZmF1bHRDZWxsMDExGTAXBgNVBAsTEFJvb3QgQ2VydGlmaWNhdGUxEjAQ + # BgNVBAMTCWxvY2FsaG9zdDAeFw0yMTEyMTQyMjA3MTZaFw0yMjEyMTQyMjA3MTZa + # MF8xCzAJBgNVBAYTAlVTMQwwCgYDVQQKEwNJQk0xFjAUBgNVBAsTDURlZmF1bHRO + # b2RlMDExFjAUBgNVBAsTDURlZmF1bHRDZWxsMDExEjAQBgNVBAMTCWxvY2FsaG9z + # dDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMv5HCsJZIpI5zCy+jXV + # z6lmzNc9UcVSEEHn86h6zT6pxuY90TYeAhlZ9hZ+SCKn4OQ4GoDRZhLPTkYDt+wW + # CV3NTIy9uCGUSJ6xjCKoxClJmgSQdg5m4HzwfY4ofoEZ5iZQ0Zmt62jGRWc0zuxj + # hegnM+eO2reBJYu6Ypa9RPJdYJsmn1RNnC74IDY8Y95qn+WZj//UALCpYfX41hko + # i7TWD9GKQO8SBmAxhjCDifOxVBokoxYrNdzESl0LXvnzEadeZTd9BfUtTaBHhx6t + # njqqCPrbTY+3jAbZFd4RiERPnhLVKMytw5ot506BhPrUtpr2lusbN5svNXjuLeea + # MMUCAwEAAaOBoDCBnTATBgNVHSMEDDAKgAhOatpLwvJFqjAdBgNVHSUEFjAUBggr + # BgEFBQcDAQYIKwYBBQUHAwIwVAYDVR0RBE0wS4E+UHJvZmlsZVVVSUQ6QXBwU3J2 + # MDEtQkFTRS05MDkzMzJjMC1iNmFiLTQ2OTMtYWI5NC01Mjc1ZDI1MmFmNDiCCWxv + # Y2FsaG9zdDARBgNVHQ4ECgQITzqhA5sO8O4wDQYJKoZIhvcNAQELBQADggEBAKR0 + # gY/BM69S6BDyWp5dxcpmZ9FS783FBbdUXjVtTkQno+oYURDrhCdsfTLYtqUlP4J4 + # CHoskP+MwJjRIoKhPVQMv14Q4VC2J9coYXnePhFjE+6MaZbTjq9WaekGrpKkMaQA + # iQt5b67jo7y63CZKIo9yBvs7sxODQzDn3wZwyux2vPegXSaTHR/rop/s/mPk3YTS + # hQprs/IVtPoWU4/TsDN3gIlrAYGbcs29CAt5q9MfzkMmKsuDkTZD0ry42VjxjAmk + # xw23l/k8RoD1wRWaDVbgpjwSzt+kl+vJE/ip2w3h69eEZ9wbo6scRO5lCO2JM4Pr + # 7RhLQyWn2u00L7/9Omw= + # -----END CERTIFICATE----- + description: i.e. certificate_authorities, supported_protocols, verification_mode etc. + multi: false + required: false + show_user: false owner: github: elastic/obs-service-integrations From 987d16fa4a7228c1ad4e51bd1494b36cc348ecb9 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Mon, 8 Aug 2022 18:39:37 +0530 Subject: [PATCH 2/4] Add changelog entry --- packages/couchbase/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/couchbase/changelog.yml b/packages/couchbase/changelog.yml index 269cfcbd639..9fa335b42cc 100644 --- a/packages/couchbase/changelog.yml +++ b/packages/couchbase/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Couchbase integration package with "resource" data stream. type: enhancement - link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link + link: https://github.com/elastic/integrations/pull/3971 - description: Couchbase integration package with "bucket" data stream. type: enhancement link: https://github.com/elastic/integrations/pull/3666 From b695e3ce5b7df29c8a133187f5506610959ad904 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Tue, 6 Sep 2022 15:44:10 +0530 Subject: [PATCH 3/4] Nits --- packages/couchbase/manifest.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/packages/couchbase/manifest.yml b/packages/couchbase/manifest.yml index 8a1a85ef46b..755f024c72d 100644 --- a/packages/couchbase/manifest.yml +++ b/packages/couchbase/manifest.yml @@ -21,8 +21,8 @@ policy_templates: description: Collect Couchbase metrics. inputs: - type: http/metrics - title: Collect Couchbase metrics using http module - description: Collect Couchbase metrics using http module. + title: Collect Couchbase metrics + description: Collect Couchbase metrics via Couchbase instances. vars: - name: hosts type: text @@ -30,12 +30,12 @@ policy_templates: multi: true required: true show_user: true - description: "Hosts for Couchbase metrics (example: http://username:password@localhost:8091)." + description: "Hosts of Couchbase instances (example: http://username:password@localhost:8091)." default: - http://username:password@localhost:8091 - type: prometheus/metrics - title: Collect Couchbase Sync Gateway metrics using prometheus module. - description: Collect Couchbase Sync Gateway metrics using prometheus module. + title: Collect additional Couchbase metrics using Sync Gateway + description: Collect additional Couchbase metrics using Sync Gateway prometheus exporter. vars: - name: hosts type: text @@ -43,7 +43,7 @@ policy_templates: multi: true required: true show_user: true - description: "Host for Sync Gateway promethus exporter (example: http://username:password@localhost:9421)." + description: "Hosts of Sync Gateway prometheus exporter (example: http://username:password@localhost:9421)." default: - http://username:password@localhost:9421 - name: ssl From eb4448e97d3cdf5d4eff1085ce0d7311d55cf6e4 Mon Sep 17 00:00:00 2001 From: kush-elastic Date: Mon, 12 Sep 2022 23:16:08 +0530 Subject: [PATCH 4/4] Nits --- packages/couchbase/manifest.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/couchbase/manifest.yml b/packages/couchbase/manifest.yml index 3fa45abaf01..36fdf95a059 100644 --- a/packages/couchbase/manifest.yml +++ b/packages/couchbase/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: couchbase title: Couchbase -version: 0.3.0 +version: 0.4.0 license: basic description: Collect metrics from Couchbase databases with Elastic Agent. type: integration