diff --git a/packages/couchbase/_dev/build/docs/README.md b/packages/couchbase/_dev/build/docs/README.md index 0158c187950..883f086c730 100644 --- a/packages/couchbase/_dev/build/docs/README.md +++ b/packages/couchbase/_dev/build/docs/README.md @@ -12,11 +12,11 @@ For example, you could use the data from this integration to know when there are The Couchbase integration collects metrics data. -Metrics give you insight into the state of the Couchbase. Metrics data streams collected by the Couchbase integration include [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html), [Cluster](https://docs.couchbase.com/server/current/rest-api/rest-cluster-details.html), [Couchbase Lite Replication](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#cbl_replication_pull), [Delta Sync](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#delta_sync), [GSI views](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#gsi_views), [Import](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#shared_bucket_import), and [Security](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#security) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. +Metrics give you insight into the state of the Couchbase. Metrics data streams collected by the Couchbase integration include [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html), [Cluster](https://docs.couchbase.com/server/current/rest-api/rest-cluster-details.html), [Couchbase Lite Replication](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#cbl_replication_pull), [Delta Sync](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#delta_sync), [GSI views](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#gsi_views), [Import](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#shared_bucket_import), [Resource Utilization](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#resource_utilization), and [Security](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#security) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. This integration uses: - `http` metricbeat module to collect `bucket` and `cluster` metrics. -- `prometheus` metricbeat module to collect `cbl_replication`, `delta_sync`, `gsi_views`, `import`, and `security` metrics. +- `prometheus` metricbeat module to collect `cbl_replication`, `delta_sync`, `gsi_views`, `import`, `resource`, and `security` metrics. Note: For Couchbase cluster setup, there is an ideal scenario of a single host with administrator access for the entire cluster to collect metrics. Providing multiple hosts from the same cluster might lead to data duplication. In the case of multiple clusters, adding a new integration to collect data from different cluster hosts is a good option. @@ -86,3 +86,11 @@ Global Secondary Indexes (GSI) support queries made by the Query Service. {{event "miscellaneous"}} {{fields "miscellaneous"}} + +### Resource Utilization + +This is the `resource` data stream. The Resource Utilization metrics are related to [MemStats](https://golang.org/pkg/runtime/#MemStats) records statistics about the memory allocator. + +{{event "resource"}} + +{{fields "resource"}} diff --git a/packages/couchbase/changelog.yml b/packages/couchbase/changelog.yml index 9f94797eb3e..10e1140a591 100644 --- a/packages/couchbase/changelog.yml +++ b/packages/couchbase/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "0.5.0" + changes: + - description: Couchbase integration package with "resource" data stream. + type: enhancement + link: https://github.com/elastic/integrations/pull/3971 - version: "0.4.0" changes: - description: Couchbase integration package with "miscellaneous" data stream. diff --git a/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..c39dc386179 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,2 @@ +dynamic_fields: + event.ingested: ".*" diff --git a/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json new file mode 100644 index 00000000000..bfd513eff82 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json @@ -0,0 +1,39 @@ +{ + "events": [ + { + "@timestamp": "2022-07-15T06:38:37.166Z", + "service": { + "address": "http://localhost:9421/metrics", + "type": "prometheus" + }, + "prometheus": { + "labels": { + "instance": "localhost:9421", + "job": "prometheus" + }, + "metrics": { + "sgw_resource_utilization_goroutines_high_watermark": 297, + "sgw_resource_utilization_warn_count": 9143, + "sgw_resource_utilization_go_memstats_heapinuse": 3.97533184e+08, + "sgw_resource_utilization_process_memory_resident": 5.77753088e+08, + "sgw_resource_utilization_go_memstats_heapreleased": 1.970176e+07, + "sgw_resource_utilization_go_memstats_sys": 5.7015116e+08, + "sgw_resource_utilization_go_memstats_heapalloc": 3.31451256e+08, + "sgw_resource_utilization_system_memory_total": 1.6637505536e+10, + "sgw_resource_utilization_go_memstats_stacksys": 3.76832e+06, + "sgw_resource_utilization_go_memstats_heapidle": 1.35569408e+08, + "sgw_resource_utilization_pub_net_bytes_sent": 1.19484151e+08, + "sgw_resource_utilization_num_goroutines": 240, + "sgw_resource_utilization_process_cpu_percent_utilization": 11.489248142477914, + "sgw_resource_utilization_go_memstats_stackinuse": 3.76832e+06, + "sgw_resource_utilization_pub_net_bytes_recv": 3.72908077e+08, + "sgw_resource_utilization_admin_net_bytes_sent": 1.19484151e+08, + "sgw_resource_utilization_admin_net_bytes_recv": 3.72908077e+08, + "sgw_resource_utilization_go_memstats_pausetotalns": 1.2177207e+08, + "sgw_resource_utilization_error_count": 0, + "go_memstats_last_gc_time_seconds": 1.657867123417899 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json new file mode 100644 index 00000000000..2b77cc32499 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/pipeline/test-resource-metrics.json-expected.json @@ -0,0 +1,67 @@ +{ + "expected": [ + { + "@timestamp": "2022-07-15T06:38:37.166Z", + "couchbase": { + "resource": { + "admin_net": { + "bytes": { + "received": 3.72908077E8, + "sent": 1.19484151E8 + } + }, + "error": { + "count": 0 + }, + "go_memstats": { + "heap": { + "alloc": 3.31451256E8, + "idle": 1.35569408E8, + "in_use": 3.97533184E8, + "released": 1.970176E7 + }, + "stack": { + "in_use": 3768320.0 + } + }, + "last_gc": 1.657867123417899, + "process": { + "cpu": { + "pct": 11.489248142477914 + }, + "memory": { + "resident": 5.77753088E8 + } + }, + "warn": { + "count": 9143 + } + } + }, + "ecs": { + "version": "8.3.0" + }, + "event": { + "category": [ + "database" + ], + "ingested": "2022-08-04T09:09:14.585647824Z", + "kind": "metric", + "module": "couchbase", + "type": [ + "info" + ] + }, + "server": { + "address": "localhost:9421" + }, + "service": { + "address": "http://localhost:9421/metrics", + "type": "prometheus" + }, + "tags": [ + "prometheus" + ] + } + ] +} \ No newline at end of file diff --git a/packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml b/packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml new file mode 100644 index 00000000000..70044322a09 --- /dev/null +++ b/packages/couchbase/data_stream/resource/_dev/test/system/test-default-config.yml @@ -0,0 +1,7 @@ +service: exporter +input: prometheus/metrics +vars: + hosts: + - http://Administrator:password@{{Hostname}}:{{Ports}} +data_stream: + vars: ~ diff --git a/packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs b/packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs new file mode 100644 index 00000000000..ad519755fb1 --- /dev/null +++ b/packages/couchbase/data_stream/resource/agent/stream/stream.yml.hbs @@ -0,0 +1,23 @@ +metricsets: ["collector"] +hosts: +{{#each hosts}} + - {{this}} +{{/each}} +period: {{period}} +metrics_path: /metrics +metrics_filters: + include: ["sgw_resource_utilization*","go_memstats_last_gc_time_seconds"] +{{#if ssl}} +{{ssl}} +{{/if}} +tags: +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml b/packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..55478ef075f --- /dev/null +++ b/packages/couchbase/data_stream/resource/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,133 @@ +--- +description: Pipeline for parsing Couchbase Sync Gateway Resource Utilization metrics. +processors: + - set: + field: ecs.version + value: 8.3.0 + ignore_failure: true + ignore_empty_value: true + - set: + field: event.type + value: [info] + ignore_failure: true + ignore_empty_value: true + - set: + field: event.kind + value: metric + ignore_failure: true + ignore_empty_value: true + - set: + field: event.category + value: [database] + ignore_failure: true + ignore_empty_value: true + - set: + field: event.ingested + value: "{{{_ingest.timestamp}}}" + ignore_failure: true + ignore_empty_value: true + - set: + field: event.module + value: couchbase + override: true + ignore_failure: true + ignore_empty_value: true + - rename: + field: prometheus.labels.instance + target_field: server.address + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_process_cpu_percent_utilization + target_field: couchbase.resource.process.cpu.pct + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_process_memory_resident + target_field: couchbase.resource.process.memory.resident + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_admin_net_bytes_recv + target_field: couchbase.resource.admin_net.bytes.received + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_admin_net_bytes_sent + target_field: couchbase.resource.admin_net.bytes.sent + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapalloc + target_field: couchbase.resource.go_memstats.heap.alloc + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapinuse + target_field: couchbase.resource.go_memstats.heap.in_use + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapidle + target_field: couchbase.resource.go_memstats.heap.idle + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_heapreleased + target_field: couchbase.resource.go_memstats.heap.released + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_go_memstats_stackinuse + target_field: couchbase.resource.go_memstats.stack.in_use + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_error_count + target_field: couchbase.resource.error.count + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.sgw_resource_utilization_warn_count + target_field: couchbase.resource.warn.count + ignore_missing: true + ignore_failure: true + - rename: + field: prometheus.metrics.go_memstats_last_gc_time_seconds + target_field: couchbase.resource.last_gc + ignore_missing: true + ignore_failure: true + - script: + description: Add job to tag list. + lang: painless + ignore_failure: true + source: | + if (ctx.tags == null) { + ctx.tags = new ArrayList(); + } + ctx.tags.add(ctx.prometheus.labels.job) + - remove: + field: prometheus + ignore_failure: true + ignore_missing: true + - script: + description: Drops null/empty values recursively. + lang: painless + source: | + boolean drop(Object o) { + if (o == null || o == "") { + return true; + } else if (o instanceof Map) { + ((Map) o).values().removeIf(v -> drop(v)); + return (((Map) o).size() == 0); + } else if (o instanceof List) { + ((List) o).removeIf(v -> drop(v)); + return (((List) o).length == 0); + } + return false; + } + drop(ctx); +on_failure: + - set: + field: error.message + value: "{{{_ingest.on_failure_message}}}" diff --git a/packages/couchbase/data_stream/resource/fields/base-fields.yml b/packages/couchbase/data_stream/resource/fields/base-fields.yml new file mode 100644 index 00000000000..7c798f4534c --- /dev/null +++ b/packages/couchbase/data_stream/resource/fields/base-fields.yml @@ -0,0 +1,12 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. diff --git a/packages/couchbase/data_stream/resource/fields/ecs.yml b/packages/couchbase/data_stream/resource/fields/ecs.yml new file mode 100644 index 00000000000..5ce04c4dc40 --- /dev/null +++ b/packages/couchbase/data_stream/resource/fields/ecs.yml @@ -0,0 +1,26 @@ +- external: ecs + name: ecs.version +- external: ecs + name: error.message +- external: ecs + name: event.category +- external: ecs + name: event.dataset +- external: ecs + name: event.duration +- external: ecs + name: event.ingested +- external: ecs + name: event.kind +- external: ecs + name: event.module +- external: ecs + name: event.type +- external: ecs + name: server.address +- external: ecs + name: service.address +- external: ecs + name: service.type +- external: ecs + name: tags diff --git a/packages/couchbase/data_stream/resource/fields/fields.yml b/packages/couchbase/data_stream/resource/fields/fields.yml new file mode 100644 index 00000000000..4206407be22 --- /dev/null +++ b/packages/couchbase/data_stream/resource/fields/fields.yml @@ -0,0 +1,79 @@ +- name: couchbase + type: group + fields: + - name: resource + type: group + fields: + - name: admin_net + type: group + fields: + - name: bytes + type: group + fields: + - name: received + type: scaled_float + description: The total number of bytes received (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. + unit: byte + metric_type: gauge + - name: sent + type: scaled_float + description: The total number of bytes sent (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. + unit: byte + metric_type: gauge + - name: error.count + type: long + description: The total number of errors logged. + metric_type: counter + - name: go_memstats + type: group + fields: + - name: heap + type: group + fields: + - name: alloc + type: scaled_float + description: Bytes of allocated heap objects. + unit: byte + metric_type: gauge + - name: idle + type: scaled_float + description: Bytes in idle (unused) spans. + unit: byte + metric_type: gauge + - name: in_use + type: scaled_float + description: Bytes in in-use spans. + unit: byte + metric_type: gauge + - name: released + type: scaled_float + description: Bytes of physical memory returned to the OS. + unit: byte + metric_type: gauge + - name: stack.in_use + type: scaled_float + description: Bytes in stack spans. + unit: byte + metric_type: gauge + - name: last_gc + type: scaled_float + description: The time the last garbage collection finished, as nanoseconds since 1970 (the UNIX epoch). + unit: nanos + metric_type: gauge + - name: process + type: group + fields: + - name: cpu.pct + type: scaled_float + description: The CPU’s utilization as percentage value. + unit: percent + metric_type: gauge + - name: memory.resident + type: scaled_float + description: The memory utilization (Resident Set Size) for the process, in bytes. + unit: byte + metric_type: gauge + - name: warn.count + type: long + description: The total number of warnings logged. + metric_type: counter diff --git a/packages/couchbase/data_stream/resource/manifest.yml b/packages/couchbase/data_stream/resource/manifest.yml new file mode 100644 index 00000000000..ef7ca4bdf10 --- /dev/null +++ b/packages/couchbase/data_stream/resource/manifest.yml @@ -0,0 +1,31 @@ +title: Couchbase Sync Gateway Resource Utilization metrics. +type: metrics +streams: + - input: prometheus/metrics + template_path: stream.yml.hbs + title: Couchbase Sync Gateway Resource Utilization metrics. + description: This data stream collects Resource Utilization metrics using sync gateway prometheus exporter. + vars: + - name: period + type: text + title: Period + multi: false + required: true + show_user: true + default: 10s + - name: tags + type: text + title: Tags + multi: true + required: false + show_user: false + default: + - forwarded + - couchbase-resource + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. diff --git a/packages/couchbase/data_stream/resource/sample_event.json b/packages/couchbase/data_stream/resource/sample_event.json new file mode 100644 index 00000000000..c3578a7b2c2 --- /dev/null +++ b/packages/couchbase/data_stream/resource/sample_event.json @@ -0,0 +1,110 @@ +{ + "@timestamp": "2022-08-01T16:19:43.614Z", + "agent": { + "ephemeral_id": "fd354a8c-e906-40b4-9828-a806f6e5a747", + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.3.2" + }, + "couchbase": { + "resource": { + "admin_net": { + "bytes": { + "received": 0, + "sent": 0 + } + }, + "error": { + "count": 0 + }, + "go_memstats": { + "heap": { + "alloc": 0, + "idle": 0, + "in_use": 0, + "released": 0 + }, + "stack": { + "in_use": 0 + } + }, + "last_gc": 1.66, + "process": { + "cpu": { + "pct": 0 + }, + "memory": { + "resident": 0 + } + }, + "warn": { + "count": 14 + } + } + }, + "data_stream": { + "dataset": "couchbase.resource", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.3.0" + }, + "elastic_agent": { + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "database" + ], + "dataset": "couchbase.resource", + "duration": 24160284, + "ingested": "2022-08-01T16:19:47Z", + "kind": "metric", + "module": "couchbase", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "ip": [ + "172.31.0.6" + ], + "mac": [ + "02:42:ac:1f:00:06" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.4.0-110-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "server": { + "address": "elastic-package-service_exporter_1:9421" + }, + "service": { + "address": "http://elastic-package-service_exporter_1:9421/metrics", + "type": "prometheus" + }, + "tags": [ + "forwarded", + "couchbase-resource", + "prometheus" + ] +} \ No newline at end of file diff --git a/packages/couchbase/docs/README.md b/packages/couchbase/docs/README.md index 34f04be33f2..614b0b1b231 100644 --- a/packages/couchbase/docs/README.md +++ b/packages/couchbase/docs/README.md @@ -12,11 +12,11 @@ For example, you could use the data from this integration to know when there are The Couchbase integration collects metrics data. -Metrics give you insight into the state of the Couchbase. Metrics data streams collected by the Couchbase integration include [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html), [Cluster](https://docs.couchbase.com/server/current/rest-api/rest-cluster-details.html), [Couchbase Lite Replication](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#cbl_replication_pull), [Delta Sync](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#delta_sync), [GSI views](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#gsi_views), [Import](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#shared_bucket_import), and [Security](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#security) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. +Metrics give you insight into the state of the Couchbase. Metrics data streams collected by the Couchbase integration include [Bucket](https://docs.couchbase.com/server/current/rest-api/rest-buckets-summary.html), [Cluster](https://docs.couchbase.com/server/current/rest-api/rest-cluster-details.html), [Couchbase Lite Replication](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#cbl_replication_pull), [Delta Sync](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#delta_sync), [GSI views](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#gsi_views), [Import](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#shared_bucket_import), [Resource Utilization](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#resource_utilization), and [Security](https://docs.couchbase.com/sync-gateway/current/stats-monitoring.html#security) metrics from [Couchbase](https://www.couchbase.com/) so that the user could monitor and troubleshoot the performance of the Couchbase instances. This integration uses: - `http` metricbeat module to collect `bucket` and `cluster` metrics. -- `prometheus` metricbeat module to collect `cbl_replication`, `delta_sync`, `gsi_views`, `import`, and `security` metrics. +- `prometheus` metricbeat module to collect `cbl_replication`, `delta_sync`, `gsi_views`, `import`, `resource`, and `security` metrics. Note: For Couchbase cluster setup, there is an ideal scenario of a single host with administrator access for the entire cluster to collect metrics. Providing multiple hosts from the same cluster might lead to data duplication. In the case of multiple clusters, adding a new integration to collect data from different cluster hosts is a good option. @@ -756,3 +756,157 @@ An example event for `miscellaneous` looks as following: | service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | tags | List of keywords used to tag each event. | keyword | | + +### Resource Utilization + +This is the `resource` data stream. The Resource Utilization metrics are related to [MemStats](https://golang.org/pkg/runtime/#MemStats) records statistics about the memory allocator. + +An example event for `resource` looks as following: + +```json +{ + "@timestamp": "2022-08-01T16:19:43.614Z", + "agent": { + "ephemeral_id": "fd354a8c-e906-40b4-9828-a806f6e5a747", + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "name": "docker-fleet-agent", + "type": "metricbeat", + "version": "8.3.2" + }, + "couchbase": { + "resource": { + "admin_net": { + "bytes": { + "received": 0, + "sent": 0 + } + }, + "error": { + "count": 0 + }, + "go_memstats": { + "heap": { + "alloc": 0, + "idle": 0, + "in_use": 0, + "released": 0 + }, + "stack": { + "in_use": 0 + } + }, + "last_gc": 1.66, + "process": { + "cpu": { + "pct": 0 + }, + "memory": { + "resident": 0 + } + }, + "warn": { + "count": 14 + } + } + }, + "data_stream": { + "dataset": "couchbase.resource", + "namespace": "ep", + "type": "metrics" + }, + "ecs": { + "version": "8.3.0" + }, + "elastic_agent": { + "id": "8c9fc977-b861-4bc8-81a2-68143b5b85ba", + "snapshot": false, + "version": "8.3.2" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "database" + ], + "dataset": "couchbase.resource", + "duration": 24160284, + "ingested": "2022-08-01T16:19:47Z", + "kind": "metric", + "module": "couchbase", + "type": [ + "info" + ] + }, + "host": { + "architecture": "x86_64", + "containerized": true, + "hostname": "docker-fleet-agent", + "ip": [ + "172.31.0.6" + ], + "mac": [ + "02:42:ac:1f:00:06" + ], + "name": "docker-fleet-agent", + "os": { + "codename": "focal", + "family": "debian", + "kernel": "5.4.0-110-generic", + "name": "Ubuntu", + "platform": "ubuntu", + "type": "linux", + "version": "20.04.4 LTS (Focal Fossa)" + } + }, + "metricset": { + "name": "collector", + "period": 10000 + }, + "server": { + "address": "elastic-package-service_exporter_1:9421" + }, + "service": { + "address": "http://elastic-package-service_exporter_1:9421/metrics", + "type": "prometheus" + }, + "tags": [ + "forwarded", + "couchbase-resource", + "prometheus" + ] +} +``` + +**Exported fields** + +| Field | Description | Type | Unit | Metric Type | +|---|---|---|---|---| +| @timestamp | Event timestamp. | date | | | +| couchbase.resource.admin_net.bytes.received | The total number of bytes received (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. | scaled_float | byte | gauge | +| couchbase.resource.admin_net.bytes.sent | The total number of bytes sent (since node start-up) on the network interface to which the Sync Gateway api.admin_interface is bound. | scaled_float | byte | gauge | +| couchbase.resource.error.count | The total number of errors logged. | long | | counter | +| couchbase.resource.go_memstats.heap.alloc | Bytes of allocated heap objects. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.heap.idle | Bytes in idle (unused) spans. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.heap.in_use | Bytes in in-use spans. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.heap.released | Bytes of physical memory returned to the OS. | scaled_float | byte | gauge | +| couchbase.resource.go_memstats.stack.in_use | Bytes in stack spans. | scaled_float | byte | gauge | +| couchbase.resource.last_gc | The time the last garbage collection finished, as nanoseconds since 1970 (the UNIX epoch). | scaled_float | nanos | gauge | +| couchbase.resource.process.cpu.pct | The CPU’s utilization as percentage value. | scaled_float | percent | gauge | +| couchbase.resource.process.memory.resident | The memory utilization (Resident Set Size) for the process, in bytes. | scaled_float | byte | gauge | +| couchbase.resource.warn.count | The total number of warnings logged. | long | | counter | +| data_stream.dataset | Data stream dataset. | constant_keyword | | | +| data_stream.namespace | Data stream namespace. | constant_keyword | | | +| data_stream.type | Data stream type. | constant_keyword | | | +| ecs.version | ECS version this event conforms to. `ecs.version` is a required field and must exist in all events. When querying across multiple indices -- which may conform to slightly different ECS versions -- this field lets integrations adjust to the schema version of the events. | keyword | | | +| error.message | Error message. | match_only_text | | | +| event.category | This is one of four ECS Categorization Fields, and indicates the second level in the ECS category hierarchy. `event.category` represents the "big buckets" of ECS categories. For example, filtering on `event.category:process` yields all events relating to process activity. This field is closely related to `event.type`, which is used as a subcategory. This field is an array. This will allow proper categorization of some events that fall in multiple categories. | keyword | | | +| event.dataset | Name of the dataset. If an event source publishes more than one type of log or events (e.g. access log, error log), the dataset is used to specify which one the event comes from. It's recommended but not required to start the dataset name with the module name, followed by a dot, then the dataset name. | keyword | | | +| event.duration | Duration of the event in nanoseconds. If event.start and event.end are known this value should be the difference between the end and start time. | long | | | +| event.ingested | Timestamp when an event arrived in the central data store. This is different from `@timestamp`, which is when the event originally occurred. It's also different from `event.created`, which is meant to capture the first time an agent saw the event. In normal conditions, assuming no tampering, the timestamps should chronologically look like this: `@timestamp` \< `event.created` \< `event.ingested`. | date | | | +| event.kind | This is one of four ECS Categorization Fields, and indicates the highest level in the ECS category hierarchy. `event.kind` gives high-level information about what type of information the event contains, without being specific to the contents of the event. For example, values of this field distinguish alert events from metric events. The value of this field can be used to inform how these kinds of events should be handled. They may warrant different retention, different access control, it may also help understand whether the data coming in at a regular interval or not. | keyword | | | +| event.module | Name of the module this data is coming from. If your monitoring agent supports the concept of modules or plugins to process events of a given source (e.g. Apache logs), `event.module` should contain the name of this module. | keyword | | | +| event.type | This is one of four ECS Categorization Fields, and indicates the third level in the ECS category hierarchy. `event.type` represents a categorization "sub-bucket" that, when used along with the `event.category` field values, enables filtering events down to a level appropriate for single visualization. This field is an array. This will allow proper categorization of some events that fall in multiple event types. | keyword | | | +| server.address | Some event server addresses are defined ambiguously. The event will sometimes list an IP, a domain or a unix socket. You should always store the raw address in the `.address` field. Then it should be duplicated to `.ip` or `.domain`, depending on which one it is. | keyword | | | +| service.address | Address where data about this service was collected from. This should be a URI, network address (ipv4:port or [ipv6]:port) or a resource path (sockets). | keyword | | | +| service.type | The type of the service data is collected from. The type can be used to group and correlate logs and metrics from one service type. Example: If logs or metrics are collected from Elasticsearch, `service.type` would be `elasticsearch`. | keyword | | | +| tags | List of keywords used to tag each event. | keyword | | | + diff --git a/packages/couchbase/manifest.yml b/packages/couchbase/manifest.yml index 27214fd2966..7a04d47d552 100644 --- a/packages/couchbase/manifest.yml +++ b/packages/couchbase/manifest.yml @@ -1,7 +1,7 @@ format_version: 1.0.0 name: couchbase title: Couchbase -version: 0.4.0 +version: 0.5.0 license: basic description: Collect metrics from Couchbase databases with Elastic Agent. type: integration @@ -33,6 +33,19 @@ policy_templates: description: "Hosts of Couchbase instances (example: http://username:password@localhost:8091)." default: - http://username:password@localhost:8091 + - type: prometheus/metrics + title: Collect additional Couchbase metrics using Sync Gateway + description: Collect additional Couchbase metrics using Sync Gateway prometheus exporter. + vars: + - name: hosts + type: text + title: Hosts + multi: true + required: true + show_user: true + description: "Hosts of Sync Gateway prometheus exporter (example: http://username:password@localhost:9421)." + default: + - http://username:password@localhost:9421 - name: ssl type: yaml title: SSL Configuration