From 160ba03fae84bdac8576eeb6e29fb24d40f3e68c Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Tue, 7 Jul 2026 12:32:04 +0200 Subject: [PATCH 01/12] Bump the version --- packages/azure_frontdoor/changelog.yml | 5 +++++ packages/azure_frontdoor/manifest.yml | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml index 2b6841c3bbf..0f103e6f691 100644 --- a/packages/azure_frontdoor/changelog.yml +++ b/packages/azure_frontdoor/changelog.yml @@ -1,4 +1,9 @@ # newer versions go on top +- version: "2.4.0" + changes: + - description: Add Health Probe dataset + type: enhancement + link: https://github.com/elastic/integrations/pull/17245 - version: "2.3.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. diff --git a/packages/azure_frontdoor/manifest.yml b/packages/azure_frontdoor/manifest.yml index 5e36842e958..e914e9cbffd 100644 --- a/packages/azure_frontdoor/manifest.yml +++ b/packages/azure_frontdoor/manifest.yml @@ -1,7 +1,7 @@ format_version: "3.0.2" name: azure_frontdoor title: "Azure Frontdoor" -version: "2.3.0" +version: "2.4.0" description: "This Elastic integration collects logs from Azure Frontdoor." type: integration categories: From 43c1e780bdd4ae6f791c9e997bcede33da7a3661 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Tue, 7 Jul 2026 12:35:15 +0200 Subject: [PATCH 02/12] Add health probe dataset --- .../_dev/test/pipeline/test-common-config.yml | 6 + .../_dev/test/pipeline/test-health-probe.log | 2 + .../test-health-probe.log-expected.json | 157 ++++++++++++++ .../agent/stream/azure-eventhub.yml.hbs | 59 ++++++ .../elasticsearch/ingest_pipeline/default.yml | 197 ++++++++++++++++++ .../data_stream/health_probe/fields/agent.yml | 12 ++ .../health_probe/fields/base-fields.yml | 18 ++ .../health_probe/fields/fields.yml | 36 ++++ .../data_stream/health_probe/manifest.yml | 40 ++++ .../health_probe/sample_event.json | 84 ++++++++ 10 files changed, 611 insertions(+) create mode 100644 packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml create mode 100644 packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log create mode 100644 packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json create mode 100644 packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs create mode 100644 packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml create mode 100644 packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml create mode 100644 packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml create mode 100644 packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml create mode 100644 packages/azure_frontdoor/data_stream/health_probe/manifest.yml create mode 100644 packages/azure_frontdoor/data_stream/health_probe/sample_event.json diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml new file mode 100644 index 00000000000..74960388383 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml @@ -0,0 +1,6 @@ +dynamic_fields: + "event.ingested": ".*" +fields: + tags: + - preserve_original_event + - azure-frontdoor-health-probe diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log new file mode 100644 index 00000000000..c5061d3f41c --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log @@ -0,0 +1,2 @@ +{"time":"2026-07-07T09:31:41Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"259ffc3865154b8d8ca2c82692b60cf1","pop":"BL","httpVerb":"HEAD","result":"OriginError","httpStatusCode":"404","probeUrl":"http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/","originName":"test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io","originIP":"216.160.83.56","totalLatencyMilliseconds":"142","connectionLatencyMilliseconds":"71","DNSLatencyMicroseconds":"2000"}} +{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"216.160.83.56","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"500"}} diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json new file mode 100644 index 00000000000..cfe99b77ca6 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json @@ -0,0 +1,157 @@ +{ + "expected": [ + { + "@timestamp": "2026-07-07T09:31:41.000Z", + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 71, + "dns_latency_microseconds": 2000, + "health_probe_id": "259ffc3865154b8d8ca2c82692b60cf1", + "origin_name": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "pop": "BL", + "result": "OriginError", + "total_latency_milliseconds": 142 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "destination": { + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.56" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "original": "{\"time\":\"2026-07-07T09:31:41Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"259ffc3865154b8d8ca2c82692b60cf1\",\"pop\":\"BL\",\"httpVerb\":\"HEAD\",\"result\":\"OriginError\",\"httpStatusCode\":\"404\",\"probeUrl\":\"http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/\",\"originName\":\"test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io\",\"originIP\":\"216.160.83.56\",\"totalLatencyMilliseconds\":\"142\",\"connectionLatencyMilliseconds\":\"71\",\"DNSLatencyMicroseconds\":\"2000\"}}", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "HEAD" + }, + "response": { + "status_code": 404 + } + }, + "related": { + "ip": [ + "216.160.83.56" + ] + }, + "tags": [ + "preserve_original_event", + "azure-frontdoor-health-probe" + ], + "url": { + "domain": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "full": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "original": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "path": "/", + "port": 80, + "scheme": "http" + } + }, + { + "@timestamp": "2026-07-07T10:00:00.000Z", + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 10, + "dns_latency_microseconds": 500, + "health_probe_id": "abc123def456", + "origin_name": "backend.example.com", + "pop": "AMS", + "result": "Success", + "total_latency_milliseconds": 25 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "destination": { + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.56" + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"216.160.83.56\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"500\"}}", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "status_code": 200 + } + }, + "related": { + "ip": [ + "216.160.83.56" + ] + }, + "tags": [ + "preserve_original_event", + "azure-frontdoor-health-probe" + ], + "url": { + "domain": "backend.example.com", + "full": "https://backend.example.com/health", + "original": "https://backend.example.com/health", + "path": "/health", + "scheme": "https" + } + } + ] +} diff --git a/packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs b/packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs new file mode 100644 index 00000000000..fb00f02724c --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/agent/stream/azure-eventhub.yml.hbs @@ -0,0 +1,59 @@ +{{#if auth_type}} +auth_type: {{auth_type}} +{{/if}} +{{#if connection_string}} +connection_string: {{connection_string}} +{{/if}} +{{#if eventhub_namespace}} +eventhub_namespace: {{eventhub_namespace}} +{{/if}} +{{#if tenant_id}} +tenant_id: {{tenant_id}} +{{/if}} +{{#if client_id}} +client_id: {{client_id}} +{{/if}} +{{#if client_secret}} +client_secret: {{client_secret}} +{{/if}} +{{#contains "client_secret" auth_type}} +{{#if authority_host}} +authority_host: {{authority_host}} +{{/if}} +{{/contains}} +{{#if storage_account_container }} +storage_account_container: {{storage_account_container}} +{{else}} +{{#if eventhub}} +storage_account_container: frontdoor-health-probe-{{eventhub}} +{{/if}} +{{/if}} +{{#if eventhub}} +eventhub: {{eventhub}} +{{/if}} +{{#if consumer_group}} +consumer_group: {{consumer_group}} +{{/if}} +{{#if storage_account}} +storage_account: {{storage_account}} +{{/if}} +{{#if storage_account_key}} +storage_account_key: {{storage_account_key}} +{{/if}} +{{#if resource_manager_endpoint}} +resource_manager_endpoint: {{resource_manager_endpoint}} +{{/if}} +tags: +{{#if preserve_original_event}} + - preserve_original_event +{{/if}} +{{#each tags as |tag|}} + - {{tag}} +{{/each}} +{{#contains "forwarded" tags}} +publisher_pipeline.disable_host: true +{{/contains}} +{{#if processors}} +processors: +{{processors}} +{{/if}} diff --git a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml new file mode 100644 index 00000000000..f2a0737465d --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml @@ -0,0 +1,197 @@ +--- +description: Pipeline for processing azure frontdoor health probe logs +processors: + - set: + field: ecs.version + value: "8.11.0" + - set: + field: event.category + value: [network] + - set: + field: event.type + value: [connection] + - rename: + field: message + target_field: event.original + ignore_missing: true + if: 'ctx.event?.original == null' + - set: + field: cloud.provider + value: azure + - drop: + description: Drop if invalid json + if: 'ctx.event?.original != null && ctx.event.original.contains(''"records"'')' + - json: + field: event.original + target_field: azure.frontdoor.health_probe + - script: + description: Drops fields with empty or N/A values recursively + tag: script_to_drop_invalid_values + lang: painless + source: | + void handleMap(Map map) { + map.values().removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || v == 'N/A' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + void handleList(List list) { + list.removeIf(v -> { + if (v instanceof Map) { + handleMap(v); + } else if (v instanceof List) { + handleList(v); + } + return v == null || v == '' || v == 'N/A' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + }); + } + handleMap(ctx); + - rename: + field: azure.frontdoor.health_probe.resourceId + target_field: azure.frontdoor.resource_id + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.operationName + target_field: azure.frontdoor.operation_name + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.category + target_field: azure.frontdoor.category + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.healthProbeId + target_field: azure.frontdoor.health_probe.health_probe_id + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.pop + target_field: azure.frontdoor.health_probe.pop + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.httpVerb + target_field: http.request.method + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.result + target_field: azure.frontdoor.health_probe.result + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.httpStatusCode + target_field: http.response.status_code + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.probeUrl + target_field: url.original + ignore_missing: true + - uri_parts: + field: url.original + ignore_missing: true + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + - set: + field: url.full + copy_from: url.original + ignore_empty_value: true + - rename: + field: azure.frontdoor.health_probe.properties.originName + target_field: azure.frontdoor.health_probe.origin_name + ignore_missing: true + - rename: + field: azure.frontdoor.health_probe.properties.originIP + target_field: destination.ip + ignore_missing: true + - convert: + field: azure.frontdoor.health_probe.properties.totalLatencyMilliseconds + target_field: azure.frontdoor.health_probe.total_latency_milliseconds + type: long + tag: convert_total_latency + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.health_probe.properties.totalLatencyMilliseconds + ignore_missing: true + - convert: + field: azure.frontdoor.health_probe.properties.connectionLatencyMilliseconds + target_field: azure.frontdoor.health_probe.connection_latency_milliseconds + type: long + tag: convert_connection_latency + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.health_probe.properties.connectionLatencyMilliseconds + ignore_missing: true + - convert: + field: azure.frontdoor.health_probe.properties.DNSLatencyMicroseconds + target_field: azure.frontdoor.health_probe.dns_latency_microseconds + type: long + tag: convert_dns_latency + on_failure: + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + ignore_missing: true + - remove: + field: azure.frontdoor.health_probe.properties.DNSLatencyMicroseconds + ignore_missing: true + - convert: + field: http.response.status_code + type: long + ignore_missing: true + - date: + field: azure.frontdoor.health_probe.time + target_field: "@timestamp" + formats: + - ISO8601 + - remove: + field: + - azure.frontdoor.health_probe.time + - azure.frontdoor.health_probe.properties + ignore_missing: true + - geoip: + field: destination.ip + target_field: destination.geo + ignore_missing: true + - geoip: + database_file: GeoLite2-ASN.mmdb + field: destination.ip + target_field: destination.as + properties: + - asn + - organization_name + ignore_missing: true + - rename: + field: destination.as.asn + target_field: destination.as.number + ignore_missing: true + - rename: + field: destination.as.organization_name + target_field: destination.as.organization.name + ignore_missing: true + - append: + field: related.ip + value: '{{{destination.ip}}}' + allow_duplicates: false + if: ctx.destination?.ip != null + +on_failure: + - set: + field: event.kind + value: pipeline_error + - append: + field: tags + value: preserve_original_event + allow_duplicates: false + - append: + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml new file mode 100644 index 00000000000..4b15225a4d4 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/agent.yml @@ -0,0 +1,12 @@ +- name: cloud.image.id + type: keyword + description: Image ID for the cloud instance. +- name: host.containerized + type: boolean + description: If the host is a container. +- name: host.os.build + type: keyword + description: OS build information. +- name: host.os.codename + type: keyword + description: OS codename, if any. diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml new file mode 100644 index 00000000000..208aa91eeb8 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/base-fields.yml @@ -0,0 +1,18 @@ +- name: data_stream.type + type: constant_keyword + description: Data stream type. +- name: data_stream.dataset + type: constant_keyword + description: Data stream dataset. +- name: data_stream.namespace + type: constant_keyword + description: Data stream namespace. +- name: '@timestamp' + type: date + description: Event timestamp. +- name: input.type + type: keyword + description: Input type. +- name: log.offset + type: long + description: Log offset. diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml new file mode 100644 index 00000000000..6274162db45 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml @@ -0,0 +1,36 @@ +- name: azure.frontdoor + type: group + fields: + - name: resource_id + type: keyword + description: Azure Resource ID. + - name: category + type: keyword + description: Azure Frontdoor category name. + - name: operation_name + type: keyword + description: Azure operation name. + - name: health_probe + type: group + fields: + - name: health_probe_id + type: keyword + description: Unique identifier for the health probe request. + - name: pop + type: keyword + description: The edge PoP (Point of Presence) that issued the health probe. + - name: result + type: keyword + description: Result of the health probe (e.g., OriginError, Success). + - name: origin_name + type: keyword + description: The hostname of the origin that was probed. + - name: total_latency_milliseconds + type: long + description: Total latency of the health probe in milliseconds. + - name: connection_latency_milliseconds + type: long + description: TCP connection latency of the health probe in milliseconds. + - name: dns_latency_microseconds + type: long + description: DNS resolution latency of the health probe in microseconds. diff --git a/packages/azure_frontdoor/data_stream/health_probe/manifest.yml b/packages/azure_frontdoor/data_stream/health_probe/manifest.yml new file mode 100644 index 00000000000..28d452ceacc --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/manifest.yml @@ -0,0 +1,40 @@ +title: "FrontDoor Health Probe" +type: logs +streams: + - input: "azure-eventhub" + template_path: "azure-eventhub.yml.hbs" + title: "Azure Frontdoor Health Probe logs" + description: "Collect Azure Frontdoor health probe logs using azure-eventhub input" + vars: + - name: tags + type: text + title: Tags + multi: true + required: true + show_user: false + default: + - azure-frontdoor-health-probe + - forwarded + - name: preserve_original_event + required: true + show_user: true + title: Preserve original event + description: Preserves a raw copy of the original event, added to the field `event.original` + type: bool + multi: false + default: false + - name: storage_account_container + type: text + title: Storage Account Container + multi: false + required: false + show_user: false + description: > + The storage account container where the integration stores the checkpoint data for the consumer group. It is an advanced option to use with extreme care. You MUST use a dedicated storage account container for each Azure log type (activity, sign-in, audit logs, and others). DO NOT REUSE the same container name for more than one Azure log type. See [Container Names](https://docs.microsoft.com/en-us/rest/api/storageservices/naming-and-referencing-containers--blobs--and-metadata#container-names) for details on naming rules from Microsoft. The integration generates a default container name if not specified. + - name: processors + type: yaml + title: Processors + multi: false + required: false + show_user: false + description: "Processors are used to reduce the number of fields in the exported event or to enhance the event with metadata. This executes in the agent before the logs are parsed. See [Processors](https://www.elastic.co/guide/en/beats/filebeat/current/filtering-and-enhancing-data.html) for details. " diff --git a/packages/azure_frontdoor/data_stream/health_probe/sample_event.json b/packages/azure_frontdoor/data_stream/health_probe/sample_event.json new file mode 100644 index 00000000000..d8e884b1f76 --- /dev/null +++ b/packages/azure_frontdoor/data_stream/health_probe/sample_event.json @@ -0,0 +1,84 @@ +{ + "@timestamp": "2026-07-07T09:31:41.000Z", + "agent": { + "ephemeral_id": "a1b2c3d4-e5f6-7890-abcd-ef1234567890", + "id": "d7426e8a-1535-4d9a-8f1e-1d5eab23567b", + "name": "docker-fleet-agent", + "type": "filebeat", + "version": "8.6.0" + }, + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 71, + "dns_latency_microseconds": 2000, + "health_probe_id": "259ffc3865154b8d8ca2c82692b60cf1", + "origin_name": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "pop": "BL", + "result": "OriginError", + "total_latency_milliseconds": 142 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "data_stream": { + "dataset": "azure_frontdoor.health_probe", + "namespace": "ep", + "type": "logs" + }, + "destination": { + "ip": "4.246.53.113" + }, + "ecs": { + "version": "8.11.0" + }, + "elastic_agent": { + "id": "d7426e8a-1535-4d9a-8f1e-1d5eab23567b", + "snapshot": false, + "version": "8.6.0" + }, + "event": { + "agent_id_status": "verified", + "category": [ + "network" + ], + "dataset": "azure_frontdoor.health_probe", + "ingested": "2026-07-07T09:32:00Z", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "HEAD" + }, + "response": { + "status_code": 404 + } + }, + "input": { + "type": "azure-eventhub" + }, + "related": { + "ip": [ + "4.246.53.113" + ] + }, + "tags": [ + "azure-frontdoor-health-probe", + "forwarded" + ], + "url": { + "domain": "test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io", + "full": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "original": "http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/", + "path": "/", + "port": 80, + "scheme": "http" + } +} From 8d58b099e2e6b46c17e1114ac2c86f9dcc1977d1 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Tue, 7 Jul 2026 13:36:21 +0200 Subject: [PATCH 03/12] Make sure to deal with lowercase n/a --- .../health_probe/_dev/test/pipeline/test-health-probe.log | 2 +- .../_dev/test/pipeline/test-health-probe.log-expected.json | 3 +-- .../health_probe/elasticsearch/ingest_pipeline/default.yml | 4 ++-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log index c5061d3f41c..a0532cda21b 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log @@ -1,2 +1,2 @@ {"time":"2026-07-07T09:31:41Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"259ffc3865154b8d8ca2c82692b60cf1","pop":"BL","httpVerb":"HEAD","result":"OriginError","httpStatusCode":"404","probeUrl":"http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/","originName":"test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io","originIP":"216.160.83.56","totalLatencyMilliseconds":"142","connectionLatencyMilliseconds":"71","DNSLatencyMicroseconds":"2000"}} -{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"216.160.83.56","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"500"}} +{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"216.160.83.56","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"n/a"}} diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json index cfe99b77ca6..153880f969b 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json @@ -84,7 +84,6 @@ "category": "FrontDoorHealthProbeLog", "health_probe": { "connection_latency_milliseconds": 10, - "dns_latency_microseconds": 500, "health_probe_id": "abc123def456", "origin_name": "backend.example.com", "pop": "AMS", @@ -123,7 +122,7 @@ "category": [ "network" ], - "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"216.160.83.56\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"500\"}}", + "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"216.160.83.56\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"n/a\"}}", "type": [ "connection" ] diff --git a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml index f2a0737465d..5d901448b44 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml @@ -36,7 +36,7 @@ processors: } else if (v instanceof List) { handleList(v); } - return v == null || v == '' || v == 'N/A' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + return v == null || v == '' || v == 'N/A' || v == 'n/a' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) }); } void handleList(List list) { @@ -46,7 +46,7 @@ processors: } else if (v instanceof List) { handleList(v); } - return v == null || v == '' || v == 'N/A' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) + return v == null || v == '' || v == 'N/A' || v == 'n/a' || (v instanceof Map && v.size() == 0) || (v instanceof List && v.size() == 0) }); } handleMap(ctx); From b8898a1054b92171afacea4ee010a916ac298a76 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Tue, 7 Jul 2026 13:42:24 +0200 Subject: [PATCH 04/12] Update PR number --- packages/azure_frontdoor/changelog.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure_frontdoor/changelog.yml b/packages/azure_frontdoor/changelog.yml index 0f103e6f691..226ddfd3320 100644 --- a/packages/azure_frontdoor/changelog.yml +++ b/packages/azure_frontdoor/changelog.yml @@ -3,7 +3,7 @@ changes: - description: Add Health Probe dataset type: enhancement - link: https://github.com/elastic/integrations/pull/17245 + link: https://github.com/elastic/integrations/pull/20016 - version: "2.3.0" changes: - description: Add client secret authentication support for Azure Event Hub with RBAC. From ca610775f3b88efb03d5587de3146a8f6eaa8226 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Tue, 7 Jul 2026 14:01:17 +0200 Subject: [PATCH 05/12] Replace latinism with plain english --- .../azure_frontdoor/data_stream/health_probe/fields/fields.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml b/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml index 6274162db45..7bbbb215dcc 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml +++ b/packages/azure_frontdoor/data_stream/health_probe/fields/fields.yml @@ -21,7 +21,7 @@ description: The edge PoP (Point of Presence) that issued the health probe. - name: result type: keyword - description: Result of the health probe (e.g., OriginError, Success). + description: "Result of the health probe (examples: OriginError, Success)." - name: origin_name type: keyword description: The hostname of the origin that was probed. From 9669a03ee600be84cc3a848a7d97c5b2d52ddd6c Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 08:01:57 +0200 Subject: [PATCH 06/12] Apply suggestions from code review Co-authored-by: Dan Kortschak --- .../elasticsearch/ingest_pipeline/default.yml | 20 ++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml index 5d901448b44..264362db010 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml @@ -14,13 +14,13 @@ processors: field: message target_field: event.original ignore_missing: true - if: 'ctx.event?.original == null' + if: ctx.event?.original == null - set: field: cloud.provider value: azure - drop: description: Drop if invalid json - if: 'ctx.event?.original != null && ctx.event.original.contains(''"records"'')' + if: ctx.event?.original != null && ctx.event.original.contains('"records"') - json: field: event.original target_field: azure.frontdoor.health_probe @@ -184,6 +184,17 @@ processors: allow_duplicates: false if: ctx.destination?.ip != null + - set: + field: event.kind + tag: set_pipeline_error_into_event_kind + value: pipeline_error + if: ctx.error?.message != null + - append: + field: tags + tag: append_tags + value: preserve_original_event + allow_duplicates: false + if: ctx.error?.message != null on_failure: - set: field: event.kind @@ -194,4 +205,7 @@ on_failure: allow_duplicates: false - append: field: error.message - value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' + value: >- + Processor '{{{ _ingest.on_failure_processor_type }}}' + {{{#_ingest.on_failure_processor_tag}}}with tag '{{{ _ingest.on_failure_processor_tag }}}' + {{{/_ingest.on_failure_processor_tag}}}failed with message '{{{ _ingest.on_failure_message }}}' From 31479298667db7352b0d0ffe30aeee90efd8256c Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 08:04:07 +0200 Subject: [PATCH 07/12] Remove event.ingested from test-common-config.yml --- .../health_probe/_dev/test/pipeline/test-common-config.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml index 74960388383..95f3801cbb1 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-common-config.yml @@ -1,5 +1,3 @@ -dynamic_fields: - "event.ingested": ".*" fields: tags: - preserve_original_event From 1fd94fcf132e578cf9f8e80355248c6b547d93c2 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 08:10:30 +0200 Subject: [PATCH 08/12] Add if condition to geoip processors as per review --- .../health_probe/elasticsearch/ingest_pipeline/default.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml index 264362db010..5ac09a000ec 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml @@ -162,6 +162,7 @@ processors: field: destination.ip target_field: destination.geo ignore_missing: true + if: ctx.destination?.ip != null - geoip: database_file: GeoLite2-ASN.mmdb field: destination.ip @@ -170,6 +171,7 @@ processors: - asn - organization_name ignore_missing: true + if: ctx.destination?.ip != null - rename: field: destination.as.asn target_field: destination.as.number From c0439c5cac236ca1608ba8fa7cb1817c64be071c Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 08:13:35 +0200 Subject: [PATCH 09/12] Update README --- packages/azure_frontdoor/docs/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md index 756bdfde5a8..963b91d03d4 100644 --- a/packages/azure_frontdoor/docs/README.md +++ b/packages/azure_frontdoor/docs/README.md @@ -11,8 +11,9 @@ The Azure Frontdoor logs integration retrieves the following types of log data f - **Access Logs**: Logs categorized as `FrontDoorAccessLog`. - **Web Application Firewall (WAF) Logs**: Logs categorized as `FrontDoorWebApplicationFirewallLog`. +- **Health Probe Logs**: Logs categorized as `FrontDoorHealthProbeLog`. -Currently, the integration does not support **Activity Logs** or **Health Probe logs**. +Currently, the integration does not support **Activity Logs**. ## Data streams @@ -20,6 +21,7 @@ This integration collects two types of data streams: - access log - waf logs +- health probe ## Requirements From fdcc472723cb5228c9f8fdef2cff120839ad6c02 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 08:18:14 +0200 Subject: [PATCH 10/12] Update README.md --- .../azure_frontdoor/_dev/build/docs/README.md | 8 ++++- packages/azure_frontdoor/docs/README.md | 29 ++++++++++++++++++- 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/packages/azure_frontdoor/_dev/build/docs/README.md b/packages/azure_frontdoor/_dev/build/docs/README.md index 8fb57e8ac9f..7684489b86f 100644 --- a/packages/azure_frontdoor/_dev/build/docs/README.md +++ b/packages/azure_frontdoor/_dev/build/docs/README.md @@ -11,8 +11,9 @@ The Azure Frontdoor logs integration retrieves the following types of log data f - **Access Logs**: Logs categorized as `FrontDoorAccessLog`. - **Web Application Firewall (WAF) Logs**: Logs categorized as `FrontDoorWebApplicationFirewallLog`. +- **Health Probe Logs**: Logs categorized as `FrontDoorHealthProbeLog`. -Currently, the integration does not support **Activity Logs** or **Health Probe logs**. +Currently, the integration does not support **Activity Logs**. ## Data streams @@ -20,6 +21,7 @@ This integration collects two types of data streams: - access log - waf logs +- health probe logs ## Requirements @@ -141,3 +143,7 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th ## WAF Logs {{fields "waf"}} + +## Health Probe Logs + +{{fields "health_probe"}} \ No newline at end of file diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md index 963b91d03d4..4b566311c00 100644 --- a/packages/azure_frontdoor/docs/README.md +++ b/packages/azure_frontdoor/docs/README.md @@ -21,7 +21,7 @@ This integration collects two types of data streams: - access log - waf logs -- health probe +- health probe logs ## Requirements @@ -222,3 +222,30 @@ Users can also use this in case of a Hybrid Cloud model, where one may define th | input.type | Input type. | keyword | | log.offset | Log offset. | long | + +## Health Probe Logs + +**Exported fields** + +| Field | Description | Type | +|---|---|---| +| @timestamp | Event timestamp. | date | +| azure.frontdoor.category | Azure Frontdoor category name. | keyword | +| azure.frontdoor.health_probe.connection_latency_milliseconds | TCP connection latency of the health probe in milliseconds. | long | +| azure.frontdoor.health_probe.dns_latency_microseconds | DNS resolution latency of the health probe in microseconds. | long | +| azure.frontdoor.health_probe.health_probe_id | Unique identifier for the health probe request. | keyword | +| azure.frontdoor.health_probe.origin_name | The hostname of the origin that was probed. | keyword | +| azure.frontdoor.health_probe.pop | The edge PoP (Point of Presence) that issued the health probe. | keyword | +| azure.frontdoor.health_probe.result | Result of the health probe (examples: OriginError, Success). | keyword | +| azure.frontdoor.health_probe.total_latency_milliseconds | Total latency of the health probe in milliseconds. | long | +| azure.frontdoor.operation_name | Azure operation name. | keyword | +| azure.frontdoor.resource_id | Azure Resource ID. | keyword | +| cloud.image.id | Image ID for the cloud instance. | keyword | +| data_stream.dataset | Data stream dataset. | constant_keyword | +| data_stream.namespace | Data stream namespace. | constant_keyword | +| data_stream.type | Data stream type. | constant_keyword | +| host.containerized | If the host is a container. | boolean | +| host.os.build | OS build information. | keyword | +| host.os.codename | OS codename, if any. | keyword | +| input.type | Input type. | keyword | +| log.offset | Log offset. | long | From a15ee510adb79362c3519788252d88b35c897741 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 14:46:39 +0200 Subject: [PATCH 11/12] Add support for IP:PORT format in originIP. Add test-case --- .../_dev/test/pipeline/test-health-probe.log | 3 +- .../test-health-probe.log-expected.json | 87 ++++++++++++++++++- .../elasticsearch/ingest_pipeline/default.yml | 30 ++++++- 3 files changed, 113 insertions(+), 7 deletions(-) diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log index a0532cda21b..10af0358072 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log @@ -1,2 +1,3 @@ {"time":"2026-07-07T09:31:41Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"259ffc3865154b8d8ca2c82692b60cf1","pop":"BL","httpVerb":"HEAD","result":"OriginError","httpStatusCode":"404","probeUrl":"http://test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io:80/","originName":"test.internal.orangemushroom-b8de5ef7.westus2.azurecontainerapps.io","originIP":"216.160.83.56","totalLatencyMilliseconds":"142","connectionLatencyMilliseconds":"71","DNSLatencyMicroseconds":"2000"}} -{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"216.160.83.56","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"n/a"}} +{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"216.160.83.56:443","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"n/a"}} +{"time":"2026-07-07T10:00:00Z","resourceId":"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD","category":"FrontDoorHealthProbeLog","operationName":"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write","properties":{"healthProbeId":"abc123def456","pop":"AMS","httpVerb":"GET","result":"Success","httpStatusCode":"200","probeUrl":"https://backend.example.com/health","originName":"backend.example.com","originIP":"[2001:db8::1]:8080","totalLatencyMilliseconds":"25","connectionLatencyMilliseconds":"10","DNSLatencyMicroseconds":"n/a"}} diff --git a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json index 153880f969b..ba6f5b8ecb5 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json +++ b/packages/azure_frontdoor/data_stream/health_probe/_dev/test/pipeline/test-health-probe.log-expected.json @@ -22,6 +22,7 @@ "provider": "azure" }, "destination": { + "address": "216.160.83.56", "as": { "number": 209 }, @@ -98,6 +99,7 @@ "provider": "azure" }, "destination": { + "address": "216.160.83.56:443", "as": { "number": 209 }, @@ -113,7 +115,8 @@ "region_iso_code": "US-WA", "region_name": "Washington" }, - "ip": "216.160.83.56" + "ip": "216.160.83.56", + "port": 443 }, "ecs": { "version": "8.11.0" @@ -122,7 +125,7 @@ "category": [ "network" ], - "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"216.160.83.56\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"n/a\"}}", + "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"216.160.83.56:443\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"n/a\"}}", "type": [ "connection" ] @@ -151,6 +154,86 @@ "path": "/health", "scheme": "https" } + }, + { + "@timestamp": "2026-07-07T10:00:00.000Z", + "azure": { + "frontdoor": { + "category": "FrontDoorHealthProbeLog", + "health_probe": { + "connection_latency_milliseconds": 10, + "health_probe_id": "abc123def456", + "origin_name": "backend.example.com", + "pop": "AMS", + "result": "Success", + "total_latency_milliseconds": 25 + }, + "operation_name": "Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write", + "resource_id": "/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD" + } + }, + "cloud": { + "provider": "azure" + }, + "destination": { + "address": "[2001:db8::1]:8080", + "as": { + "number": 65551, + "organization": { + "name": "Documentation ASN" + } + }, + "geo": { + "city_name": "Greenwich", + "continent_name": "Europe", + "country_iso_code": "GB", + "country_name": "United Kingdom", + "location": { + "lat": 51.47687, + "lon": -4.1E-4 + }, + "region_iso_code": "GB-ENG", + "region_name": "England" + }, + "ip": "2001:db8::1", + "port": 8080 + }, + "ecs": { + "version": "8.11.0" + }, + "event": { + "category": [ + "network" + ], + "original": "{\"time\":\"2026-07-07T10:00:00Z\",\"resourceId\":\"/SUBSCRIPTIONS/49D1B571-1CBE-402D-B523-AFEE3C19B64E/RESOURCEGROUPS/WAF-RG/PROVIDERS/MICROSOFT.CDN/PROFILES/TESTCLOUD\",\"category\":\"FrontDoorHealthProbeLog\",\"operationName\":\"Microsoft.Cdn/Profiles/FrontDoorHealthProbeLog/Write\",\"properties\":{\"healthProbeId\":\"abc123def456\",\"pop\":\"AMS\",\"httpVerb\":\"GET\",\"result\":\"Success\",\"httpStatusCode\":\"200\",\"probeUrl\":\"https://backend.example.com/health\",\"originName\":\"backend.example.com\",\"originIP\":\"[2001:db8::1]:8080\",\"totalLatencyMilliseconds\":\"25\",\"connectionLatencyMilliseconds\":\"10\",\"DNSLatencyMicroseconds\":\"n/a\"}}", + "type": [ + "connection" + ] + }, + "http": { + "request": { + "method": "GET" + }, + "response": { + "status_code": 200 + } + }, + "related": { + "ip": [ + "2001:db8::1" + ] + }, + "tags": [ + "preserve_original_event", + "azure-frontdoor-health-probe" + ], + "url": { + "domain": "backend.example.com", + "full": "https://backend.example.com/health", + "original": "https://backend.example.com/health", + "path": "/health", + "scheme": "https" + } } ] } diff --git a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml index 5ac09a000ec..a160e58dcf2 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml +++ b/packages/azure_frontdoor/data_stream/health_probe/elasticsearch/ingest_pipeline/default.yml @@ -101,10 +101,6 @@ processors: field: azure.frontdoor.health_probe.properties.originName target_field: azure.frontdoor.health_probe.origin_name ignore_missing: true - - rename: - field: azure.frontdoor.health_probe.properties.originIP - target_field: destination.ip - ignore_missing: true - convert: field: azure.frontdoor.health_probe.properties.totalLatencyMilliseconds target_field: azure.frontdoor.health_probe.total_latency_milliseconds @@ -153,11 +149,37 @@ processors: target_field: "@timestamp" formats: - ISO8601 + - rename: + field: azure.frontdoor.health_probe.properties.originIP + target_field: destination.address + ignore_missing: true - remove: field: - azure.frontdoor.health_probe.time - azure.frontdoor.health_probe.properties ignore_missing: true + - convert: + description: Convert destination.address to ip type, if it contains a port, extract the IP part + tag: convert_destination_address + field: destination.address + target_field: destination.ip + type: ip + if: ctx.destination?.address != null + on_failure: + - grok: + description: Extract destination.port from destination.address if it contains a port + tag: grok_destination_port + field: destination.address + patterns: + - '%{IPV4:destination.ip}:%{NUMBER:destination.port:long}' + - '\[%{IPV6:destination.ip}\]:%{NUMBER:destination.port:long}' + ignore_missing: true + if: ctx.destination?.address != null && ctx.destination.address.contains(':') + on_failure: + - append: + tag: append_error_message_1e8f0b9e + field: error.message + value: 'Processor {{{_ingest.on_failure_processor_type}}} with tag {{{_ingest.on_failure_processor_tag}}} in pipeline {{{_ingest.on_failure_pipeline}}} failed with message: {{{_ingest.on_failure_message}}}' - geoip: field: destination.ip target_field: destination.geo From 409da345deec33c36834f1ef9e6e3daa2574e509 Mon Sep 17 00:00:00 2001 From: Mirko Bez Date: Wed, 8 Jul 2026 18:19:59 +0200 Subject: [PATCH 12/12] Fix automatic check complaints --- .../azure_frontdoor/_dev/build/docs/README.md | 2 +- .../health_probe/sample_event.json | 20 +++++++++++++++++-- packages/azure_frontdoor/docs/README.md | 2 +- 3 files changed, 20 insertions(+), 4 deletions(-) diff --git a/packages/azure_frontdoor/_dev/build/docs/README.md b/packages/azure_frontdoor/_dev/build/docs/README.md index 7684489b86f..3498714e9d1 100644 --- a/packages/azure_frontdoor/_dev/build/docs/README.md +++ b/packages/azure_frontdoor/_dev/build/docs/README.md @@ -17,7 +17,7 @@ Currently, the integration does not support **Activity Logs**. ## Data streams -This integration collects two types of data streams: +This integration collects three types of data streams: - access log - waf logs diff --git a/packages/azure_frontdoor/data_stream/health_probe/sample_event.json b/packages/azure_frontdoor/data_stream/health_probe/sample_event.json index d8e884b1f76..da7ffe2967b 100644 --- a/packages/azure_frontdoor/data_stream/health_probe/sample_event.json +++ b/packages/azure_frontdoor/data_stream/health_probe/sample_event.json @@ -32,7 +32,23 @@ "type": "logs" }, "destination": { - "ip": "4.246.53.113" + "address": "216.160.83.56", + "as": { + "number": 209 + }, + "geo": { + "city_name": "Milton", + "continent_name": "North America", + "country_iso_code": "US", + "country_name": "United States", + "location": { + "lat": 47.2513, + "lon": -122.3149 + }, + "region_iso_code": "US-WA", + "region_name": "Washington" + }, + "ip": "216.160.83.56" }, "ecs": { "version": "8.11.0" @@ -81,4 +97,4 @@ "port": 80, "scheme": "http" } -} +} \ No newline at end of file diff --git a/packages/azure_frontdoor/docs/README.md b/packages/azure_frontdoor/docs/README.md index 4b566311c00..430836efd58 100644 --- a/packages/azure_frontdoor/docs/README.md +++ b/packages/azure_frontdoor/docs/README.md @@ -17,7 +17,7 @@ Currently, the integration does not support **Activity Logs**. ## Data streams -This integration collects two types of data streams: +This integration collects three types of data streams: - access log - waf logs