From bab1019eb3bbceaed1c2d108b7f123801960a6b8 Mon Sep 17 00:00:00 2001 From: Jordan <51442161+jordansh@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:11:44 +0300 Subject: [PATCH 1/3] siem: p1 integrations esql fixes and ingest script improvements - Fix dataset names in azure_openai and azure_ai_foundry evaluations (were matching azure.open_ai / azure.ai_foundry; actual datasets are azure_openai.logs / azure_ai_foundry.logs) - Add m365_defender.alert.title as event.action for alert stream, overriding ingest-time detectionStatus with the human-readable alert name - Fix m365_defender host.target conditions to require user.name to avoid spurious host-as-target matches on non-user events - Extend m365_defender user.name fallback to event stream via process.user.name - Ingest script: handle nanosecond Unix timestamps (fixes sysdig alerts/event) - Ingest script: expand dotted config keys to nested dicts (fixes salesforce pipeline routing) - Ingest script: inject data_stream.dataset when absent from fixtures (fixes Azure, ForgeRock) - Ingest script: rewrite YAML parser to handle nested mappings (fixes microsoft_dhcp) - Ingest script: inject log.file.path from fixture config (fixes cisco_umbrella routing) Co-Authored-By: Claude Sonnet 4.6 --- .../esql-evaluations/azure_ai_foundry.ts | 26 +-- dev/domain/esql-evaluations/azure_openai.ts | 22 +-- .../esql-evaluations/enrichmentQuery.esql | 10 +- dev/domain/esql-evaluations/m365_defender.ts | 1 + .../install_and_ingest.py | 185 ++++++++++++++++-- dev/domain/p1/m365_defender.md | 38 ++-- 6 files changed, 216 insertions(+), 66 deletions(-) diff --git a/dev/domain/esql-evaluations/azure_ai_foundry.ts b/dev/domain/esql-evaluations/azure_ai_foundry.ts index 099bb8c954a..2c52f430516 100644 --- a/dev/domain/esql-evaluations/azure_ai_foundry.ts +++ b/dev/domain/esql-evaluations/azure_ai_foundry.ts @@ -22,13 +22,13 @@ export const azure_ai_foundryEvaluations = { esql: `| EVAL user.id = CASE( user.id IS NOT NULL, user.id, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.ai_foundry.properties.object_id, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.ai_foundry.properties.object_id, null ), host.ip = CASE( host.ip IS NOT NULL, host.ip, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", source.ip, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.caller_ip_address, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", source.ip, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.caller_ip_address, null )`, }, @@ -38,9 +38,9 @@ export const azure_ai_foundryEvaluations = { esql: `| EVAL event.action = CASE( event.action IS NOT NULL, event.action, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.ai_foundry.operation_name, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.operation_name, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.operation_id, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.ai_foundry.operation_name, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.operation_name, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.operation_id, null )`, }, @@ -50,21 +50,21 @@ export const azure_ai_foundryEvaluations = { esql: `| EVAL service.target.id = CASE( service.target.id IS NOT NULL, service.target.id, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.resource.id, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.properties.model_deployment_name, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.backend_request_body.model, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.resource.id, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.properties.model_deployment_name, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.backend_request_body.model, null ), service.target.name = CASE( service.target.name IS NOT NULL, service.target.name, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.resource.name, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.properties.model_deployment_name, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.backend_request_body.model, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "Audit", azure.resource.name, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "RequestResponse", azure.ai_foundry.properties.model_deployment_name, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.backend_request_body.model, null ), entity.target.id = CASE( entity.target.id IS NOT NULL, entity.target.id, - data_stream.dataset == "azure.ai_foundry" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.backend_response_body.id, + data_stream.dataset == "azure_ai_foundry.logs" AND data_stream.type == "logs" AND azure.ai_foundry.category == "GatewayLogs", azure.ai_foundry.properties.backend_response_body.id, null )`, }, diff --git a/dev/domain/esql-evaluations/azure_openai.ts b/dev/domain/esql-evaluations/azure_openai.ts index 107e69c5819..36d7a994e7e 100644 --- a/dev/domain/esql-evaluations/azure_openai.ts +++ b/dev/domain/esql-evaluations/azure_openai.ts @@ -22,13 +22,13 @@ export const azure_openaiEvaluations = { esql: `| EVAL user.id = CASE( user.id IS NOT NULL, user.id, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "Audit", azure.open_ai.properties.object_id, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "Audit", azure.open_ai.properties.object_id, null ), host.ip = CASE( host.ip IS NOT NULL, host.ip, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "GatewayLogs", source.ip, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "RequestResponse", azure.open_ai.caller_ip_address, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "GatewayLogs", source.ip, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "RequestResponse", azure.open_ai.caller_ip_address, null )`, }, @@ -38,8 +38,8 @@ export const azure_openaiEvaluations = { esql: `| EVAL event.action = CASE( event.action IS NOT NULL, event.action, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category IN ("Audit", "RequestResponse"), azure.open_ai.operation_name, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "GatewayLogs", azure.open_ai.properties.operation_id, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category IN ("Audit", "RequestResponse"), azure.open_ai.operation_name, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "GatewayLogs", azure.open_ai.properties.operation_id, null )`, }, @@ -49,16 +49,16 @@ export const azure_openaiEvaluations = { esql: `| EVAL service.target.id = CASE( service.target.id IS NOT NULL, service.target.id, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "Audit", azure.resource.id, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "RequestResponse", azure.open_ai.properties.model_deployment_name, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "GatewayLogs", azure.open_ai.properties.backend_request_body.model, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "Audit", azure.resource.id, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "RequestResponse", azure.open_ai.properties.model_deployment_name, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "GatewayLogs", azure.open_ai.properties.backend_request_body.model, null ), service.target.name = CASE( service.target.name IS NOT NULL, service.target.name, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "Audit", azure.resource.name, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "RequestResponse", azure.open_ai.properties.model_name, - data_stream.dataset == "azure.open_ai" AND azure.open_ai.category == "GatewayLogs", azure.open_ai.properties.backend_response_body.model, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "Audit", azure.resource.name, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "RequestResponse", azure.open_ai.properties.model_name, + data_stream.dataset == "azure_openai.logs" AND azure.open_ai.category == "GatewayLogs", azure.open_ai.properties.backend_response_body.model, null )`, }, diff --git a/dev/domain/esql-evaluations/enrichmentQuery.esql b/dev/domain/esql-evaluations/enrichmentQuery.esql index d7cfc0f37ed..78057f275a0 100644 --- a/dev/domain/esql-evaluations/enrichmentQuery.esql +++ b/dev/domain/esql-evaluations/enrichmentQuery.esql @@ -247,7 +247,7 @@ jamf_pro.events.event.authorized_username, data_stream.dataset == "m365_defender.event" AND m365_defender.event.initiating_process.account_name IS NOT NULL, m365_defender.event.initiating_process.account_name, - data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND process.user.name IS NOT NULL, + data_stream.dataset IN ("m365_defender.event", "m365_defender.alert", "m365_defender.incident") AND process.user.name IS NOT NULL, process.user.name, data_stream.dataset == "microsoft_intune.audit" AND user.email IS NOT NULL, user.email, @@ -359,6 +359,10 @@ gitlab.pages.msg, data_stream.dataset == "greenhouse.audit" AND greenhouse.audit.event.type IS NOT NULL, greenhouse.audit.event.type, + data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL, + m365_defender.alert.title, + event.action IS NOT NULL, + event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, data_stream.dataset == "openai.images" AND openai.images.source IS NOT NULL, @@ -852,7 +856,7 @@ m365_defender.event.additional_fields.DestinationComputerObjectGuid, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.id IS NOT NULL, host.id, - data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL, + data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL, host.id, data_stream.dataset == "qualys_vmdr.asset_host_detection" AND host.id IS NOT NULL, host.id, @@ -895,7 +899,7 @@ m365_defender.event.destination.device_name, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.name IS NOT NULL, host.name, - data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL, + data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL, host.name, data_stream.dataset == "microsoft_dhcp.log" AND event.action IN ("dhcp-new", "dhcp-dns-update", "ipv6-dns-update-request", "ipv6-dns-update-successful", "ipv6-dns-update-failed", "ipv6-dns-update-request-failed") AND source.address IS NOT NULL, source.address, diff --git a/dev/domain/esql-evaluations/m365_defender.ts b/dev/domain/esql-evaluations/m365_defender.ts index 7e64507fb56..b7803e7260c 100644 --- a/dev/domain/esql-evaluations/m365_defender.ts +++ b/dev/domain/esql-evaluations/m365_defender.ts @@ -59,6 +59,7 @@ export const m365_defenderEvaluations = { section: "Combined ES|QL \u2014 event action", esql: `| EVAL event.action = CASE( + data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL, m365_defender.alert.title, event.action IS NOT NULL, event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, null diff --git a/dev/domain/install-p1-integrations/install_and_ingest.py b/dev/domain/install-p1-integrations/install_and_ingest.py index 2a8f4abf59f..f613ff7088d 100755 --- a/dev/domain/install-p1-integrations/install_and_ingest.py +++ b/dev/domain/install-p1-integrations/install_and_ingest.py @@ -295,6 +295,112 @@ def stream_target(package: str, stream: str) -> StreamTarget: return StreamTarget(package=package, stream=stream, data_type=data_type, dataset=dataset) +def _parse_simple_yaml(text: str) -> dict[str, Any]: + """Parse simple nested-mapping YAML with scalar values and flat list sequences.""" + lines = [ + ln.rstrip() + for ln in text.splitlines() + if ln.strip() and not ln.lstrip().startswith("#") + ] + + def _unquote(s: str) -> str: + if len(s) >= 2 and s[0] == s[-1] and s[0] in ('"', "'"): + return s[1:-1] + return s + + pos = [0] # mutable cursor shared across recursive calls + + def parse_mapping(min_indent: int) -> dict[str, Any]: + result: dict[str, Any] = {} + while pos[0] < len(lines): + line = lines[pos[0]] + indent = len(line) - len(line.lstrip()) + if indent < min_indent: + break + content = line.lstrip() + if ":" not in content or content.startswith("- "): + pos[0] += 1 + continue + key, _, value = content.partition(":") + key = _unquote(key.strip()) + value = value.strip() + pos[0] += 1 + if value: + result[key] = _unquote(value) + else: + # Look ahead to decide: list or nested mapping + if pos[0] < len(lines): + next_content = lines[pos[0]].lstrip() + if next_content.startswith("- "): + lst: list[Any] = [] + next_indent = len(lines[pos[0]]) - len(next_content) + while pos[0] < len(lines): + nl = lines[pos[0]] + ni = len(nl) - len(nl.lstrip()) + nc = nl.lstrip() + if ni < next_indent or not nc.startswith("- "): + break + lst.append(_unquote(nc[2:].strip())) + pos[0] += 1 + result[key] = lst + else: + child_indent = len(lines[pos[0]]) - len(next_content) + result[key] = parse_mapping(child_indent) + return result + + return parse_mapping(0) + + +def _expand_dotted_keys(d: dict[str, Any]) -> dict[str, Any]: + """Expand dotted keys like {'event.provider': 'X'} into nested dicts {'event': {'provider': 'X'}}.""" + result: dict[str, Any] = {} + for key, value in d.items(): + if isinstance(value, dict): + value = _expand_dotted_keys(value) + if "." in key: + parts = key.split(".") + node = result + for part in parts[:-1]: + node = node.setdefault(part, {}) + node[parts[-1]] = value + else: + result[key] = value + return result + + +def _deep_merge(base: dict[str, Any], overlay: dict[str, Any]) -> None: + """Merge overlay into base in-place; nested dicts are merged recursively.""" + for key, value in overlay.items(): + if key in base and isinstance(base[key], dict) and isinstance(value, dict): + _deep_merge(base[key], value) + else: + base[key] = value + + +def _fixture_config_fields(fixture: Path) -> dict[str, Any]: + """Return merged 'fields' from test-common-config.yml and per-fixture *-config.yml.""" + merged: dict[str, Any] = {} + common = fixture.parent / "test-common-config.yml" + if common.exists(): + try: + parsed = _parse_simple_yaml(common.read_text()) + fields = parsed.get("fields", {}) + if isinstance(fields, dict): + _deep_merge(merged, fields) + except Exception: + pass + per_file = fixture.parent / (fixture.name + "-config.yml") + if per_file.exists(): + try: + parsed = _parse_simple_yaml(per_file.read_text()) + fields = parsed.get("fields", {}) + if isinstance(fields, dict): + _deep_merge(merged, fields) + except Exception: + pass + return _expand_dotted_keys(merged) + + def is_fixture_file(path: Path) -> bool: name = path.name if name in SKIP_FIXTURE_NAMES: @@ -309,9 +415,15 @@ def discover_fixtures(package: str) -> list[Path]: if not pkg_dir.is_dir(): return [] fixtures: list[Path] = [] + streams_with_fixtures: set[str] = set() for path in sorted(pkg_dir.glob("data_stream/*/_dev/test/pipeline/*")): if path.is_file() and is_fixture_file(path): fixtures.append(path) + streams_with_fixtures.add(path.relative_to(pkg_dir).parts[1]) + # Fallback: for any stream without pipeline fixtures, index sample_event.json directly + for path in sorted(pkg_dir.glob("data_stream/*/sample_event.json")): + if path.is_file() and path.relative_to(pkg_dir).parts[1] not in streams_with_fixtures: + fixtures.append(path) return fixtures @@ -331,8 +443,13 @@ def _to_iso_timestamp(value: Any) -> str | None: text = value.strip() return text if text else None if isinstance(value, (int, float)): - # Heuristic: epoch ms vs sec - seconds = value / 1000 if value > 1e11 else value + # Heuristic: epoch ns vs ms vs sec + if value > 1e15: + seconds = value / 1e9 # nanoseconds (e.g. sysdig.timestamp) + elif value > 1e11: + seconds = value / 1000 # milliseconds + else: + seconds = value # seconds return datetime.fromtimestamp(seconds, tz=timezone.utc).strftime("%Y-%m-%dT%H:%M:%S.%f")[:-3] + "Z" return None @@ -397,6 +514,7 @@ def ensure_timestamp(doc: dict[str, Any]) -> None: def load_documents(fixture: Path, package: str, run_id: str) -> list[dict[str, Any]]: text = fixture.read_text() docs: list[dict[str, Any]] = [] + config_fields = _fixture_config_fields(fixture) if fixture.suffix == ".log": for line in text.splitlines(): @@ -420,6 +538,8 @@ def load_documents(fixture: Path, package: str, run_id: str) -> list[dict[str, A tagged: list[dict[str, Any]] = [] for doc in docs: merged = dict(doc) + if config_fields: + _deep_merge(merged, config_fields) existing_tags = merged.get("tags") if existing_tags is None: tags: list[str] = [] @@ -439,8 +559,17 @@ def load_documents(fixture: Path, package: str, run_id: str) -> list[dict[str, A return tagged +_VERSION_ONLY_RE = re.compile(r"^-\d+\.\d+\.\d+$") + + def resolve_ingest_pipeline(es_url: str, auth: Auth | None, base_name: str) -> str | None: - """Resolve Fleet-installed pipeline name (may include package version suffix).""" + """Resolve Fleet-installed pipeline name (may include package version suffix). + + Fleet installs both a main versioned pipeline (e.g. logs-azure_openai.logs-1.13.0) + and named sub-pipelines (e.g. logs-azure_openai.logs-1.13.0-azure-shared-pipeline). + A plain reverse sort picks the sub-pipeline first because the extra suffix sorts later. + We filter to only bare versioned names before falling back to all matches. + """ status, body = http_json("GET", f"{es_url}/_ingest/pipeline/{base_name}", auth=auth) if status == 200: return base_name @@ -448,8 +577,10 @@ def resolve_ingest_pipeline(es_url: str, auth: Auth | None, base_name: str) -> s # Fleet registers versioned pipelines, e.g. logs-slack.audit-1.28.0 status, body = http_json("GET", f"{es_url}/_ingest/pipeline/{base_name}*", auth=auth) if status == 200 and isinstance(body, dict) and body: - matches = sorted(body.keys(), reverse=True) - return matches[0] + # Prefer pipelines whose only suffix is a bare version (-X.Y.Z), excluding + # sub-pipelines like logs-azure_openai.logs-1.13.0-azure-shared-pipeline. + main = [k for k in body if _VERSION_ONLY_RE.match(k[len(base_name):])] + return sorted(main, reverse=True)[0] if main else sorted(body, reverse=True)[0] return None @@ -590,7 +721,7 @@ def bulk_ingest( es_url: str, auth: Auth | None, target: StreamTarget, - pipeline: str, + pipeline: str | None, documents: list[dict[str, Any]], *, chunk_size: int = 200, @@ -602,12 +733,10 @@ def bulk_ingest( chunk = documents[i : i + chunk_size] lines: list[str] = [] for doc in chunk: - action = { - "create": { - "_index": target.data_stream, - "pipeline": pipeline, - } - } + create: dict[str, Any] = {"_index": target.data_stream} + if pipeline: + create["pipeline"] = pipeline + action = {"create": create} lines.append(json.dumps(action, separators=(",", ":"))) lines.append(json.dumps(doc, separators=(",", ":"))) payload = ("\n".join(lines) + "\n").encode() @@ -766,6 +895,16 @@ def ingest_package_fixtures( ) continue + # Ensure data_stream.dataset is set so constant_keyword fields get a value. + # Many fixtures omit it; without it the field is invisible in Kibana Discover. + for doc in documents: + if not isinstance(doc.get("data_stream"), dict): + doc["data_stream"] = { + "dataset": target.dataset, + "type": target.data_type, + "namespace": "default", + } + if dry_run: report.fixtures.append( FixtureResult(str(fixture.relative_to(REPO)), stream, len(documents), 0) @@ -773,17 +912,21 @@ def ingest_package_fixtures( report.events_submitted += len(documents) continue - pipeline = resolve_ingest_pipeline(es_url, auth, target.pipeline) - if not pipeline: - msg = f"ingest pipeline not found: {target.pipeline}" - report.fixtures.append( - FixtureResult(str(fixture.relative_to(REPO)), stream, 0, 0, msg) - ) - report.notes.append(msg) - continue + is_sample = fixture.name == "sample_event.json" + if is_sample: + pipeline_name: str | None = "_none" # bypass default pipeline; doc is already processed + else: + pipeline_name = resolve_ingest_pipeline(es_url, auth, target.pipeline) + if not pipeline_name: + msg = f"ingest pipeline not found: {target.pipeline}" + report.fixtures.append( + FixtureResult(str(fixture.relative_to(REPO)), stream, 0, 0, msg) + ) + report.notes.append(msg) + continue try: - submitted, bulk_errors = bulk_ingest(es_url, auth, target, pipeline, documents) + submitted, bulk_errors = bulk_ingest(es_url, auth, target, pipeline_name, documents) except Exception as exc: # noqa: BLE001 report.fixtures.append( FixtureResult(str(fixture.relative_to(REPO)), stream, 0, 0, str(exc)) diff --git a/dev/domain/p1/m365_defender.md b/dev/domain/p1/m365_defender.md index 182bb5a8ecf..012df5860a4 100644 --- a/dev/domain/p1/m365_defender.md +++ b/dev/domain/p1/m365_defender.md @@ -306,7 +306,7 @@ flowchart LR **Stream:** `m365_defender.alert` · **Fixture:** `packages/m365_defender/data_stream/alert/_dev/test/pipeline/test-alert.log-expected.json` ``` -Actor (user, CDPUserIS-38411) → detected → Target (host, clw555test) +Actor (user, CDPUserIS-38411) → Suspicious PowerShell command line → Target (host, clw555test) ``` #### Actor @@ -325,11 +325,11 @@ Actor (user, CDPUserIS-38411) → detected → Target (host, clw555test) | Field | Value | | --- | --- | -| action | detected | -| source_field | `event.action` | -| source_value | detected | +| action | Suspicious PowerShell command line | +| source_field | `m365_defender.alert.title` | +| source_value | Suspicious PowerShell command line | -Note: `event.action` reflects evidence `detectionStatus` (detection outcome), not the alert title (`Suspicious PowerShell command line`). +Note: ES\|QL overrides the ingest-time `event.action` (`detectionStatus: detected`) with `m365_defender.alert.title` — the human-readable detection name. #### Target @@ -350,7 +350,7 @@ Note: `event.action` reflects evidence `detectionStatus` (detection outcome), no ```mermaid flowchart LR - A["Actor: CDPUserIS-38411"] --> E["detected"] + A["Actor: CDPUserIS-38411"] --> E["Suspicious PowerShell command line"] E --> T["Target: clw555test"] ``` @@ -380,7 +380,7 @@ flowchart LR | `user.id` | `m365_defender.event.initiating_process.account_sid` | `data_stream.dataset == "m365_defender.event" AND user.id IS NULL` | high | **vendor fallback** (device telemetry) | | `user.name` | `user.name` | `user.name IS NOT NULL` | high | **preserve existing** — column-level | | `user.name` | `m365_defender.event.initiating_process.account_name` | `data_stream.dataset == "m365_defender.event" AND user.name IS NULL` | high | **vendor fallback** (`test-device.log-expected.json`) | -| `user.name` | `process.user.name` | `data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND user.name IS NULL` | high | **vendor fallback** — evidence process owner (`test-alert.log-expected.json`) | +| `user.name` | `process.user.name` | `data_stream.dataset IN ("m365_defender.event", "m365_defender.alert", "m365_defender.incident") AND user.name IS NULL` | high | **vendor fallback** — endpoint/evidence process owner (`test-alert.log-expected.json`) | | `user.email` | — | — | high | **ingest-only — no ES\|QL** — pipelines set `user.email` / `AccountUpn`; no alternate query-time path | | `user.domain` | — | — | high | **ingest-only — no ES\|QL** — pipelines set `user.domain`; no alternate query-time path | | `host.ip` | `host.ip` | `host.ip IS NOT NULL` | high | **preserve existing** — column-level | @@ -393,11 +393,11 @@ flowchart LR | `host.target.id` | `host.target.id` | `host.target.id IS NOT NULL` | high | **preserve existing** (none today) | | `host.target.id` | `m365_defender.event.additional_fields.DestinationComputerObjectGuid` | `data_stream.dataset == "m365_defender.event" AND event.action IN ("samr-query", "dns-query")` | high | **vendor fallback** — SAMR fixture GUID | | `host.target.id` | `host.id` | `data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.id IS NOT NULL` | high | **de-facto** evidence device (`test-alert.log-expected.json`) | -| `host.target.id` | `host.id` | `data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL` | high | **de-facto** onboarded sensor host (e.g. `dpapiaccessed`) | +| `host.target.id` | `host.id` | `data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL` | high | **de-facto** onboarded sensor host (e.g. `dpapiaccessed`); guard prevents self-loop when host is actor | | `host.target.name` | `host.target.name` | `host.target.name IS NOT NULL` | high | **preserve existing** | | `host.target.name` | `m365_defender.event.destination.device_name` | `data_stream.dataset == "m365_defender.event" AND m365_defender.event.destination.device_name IS NOT NULL` | high | **de-facto** identity query target | | `host.target.name` | `host.name` | `data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.name IS NOT NULL` | high | **de-facto** evidence hostname | -| `host.target.name` | `host.name` | `data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL` | high | **de-facto** device telemetry target host | +| `host.target.name` | `host.name` | `data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL` | high | **de-facto** device telemetry target host; guard prevents self-loop when host is actor | | `host.target.ip` | `host.target.ip` | `host.target.ip IS NOT NULL` | high | **preserve existing** | | `host.target.ip` | `destination.ip` | `data_stream.dataset == "m365_defender.event" AND destination.ip IS NOT NULL` | high | **de-facto** remote peer (`test-app-and-identity.log-expected.json`) | | `user.target.name` | `user.target.name` | `user.target.name IS NOT NULL` | high | **preserve existing** | @@ -416,7 +416,8 @@ flowchart LR | Output column | Source field(s) | Condition (dataset + optional) | Confidence | Notes | | --- | --- | --- | --- | --- | -| `event.action` | `event.action` | `event.action IS NOT NULL` | high | **preserve existing** (arrays on alert/incident) | +| `event.action` | `m365_defender.alert.title` | `data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL` | high | **override** — alert title as human-readable action; replaces ingest-time evidence `detectionStatus` (`detected`) | +| `event.action` | `event.action` | `event.action IS NOT NULL` | high | **preserve existing** (arrays on non-alert datasets) | | `event.action` | `m365_defender.event.action.type` | `data_stream.dataset == "m365_defender.event" AND event.action IS NULL AND m365_defender.event.action.type IS NOT NULL` | medium | **vendor fallback** — BehaviorInfo/AlertInfo hunting gap; raw vendor verb | ### Detection flags (mandatory — run first) @@ -467,7 +468,7 @@ Set **`entity.target.type`** only in the fallback branch (correct ECS name — n user.name = CASE( user.name IS NOT NULL, user.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.initiating_process.account_name IS NOT NULL, m365_defender.event.initiating_process.account_name, - data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND process.user.name IS NOT NULL, process.user.name, + data_stream.dataset IN ("m365_defender.event", "m365_defender.alert", "m365_defender.incident") AND process.user.name IS NOT NULL, process.user.name, null ), host.ip = CASE( @@ -484,6 +485,7 @@ Set **`entity.target.type`** only in the fallback branch (correct ECS name — n ```esql | EVAL event.action = CASE( + data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL, m365_defender.alert.title, event.action IS NOT NULL, event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, null @@ -498,14 +500,14 @@ Set **`entity.target.type`** only in the fallback branch (correct ECS name — n host.target.id IS NOT NULL, host.target.id, data_stream.dataset == "m365_defender.event" AND event.action IN ("samr-query", "dns-query") AND m365_defender.event.additional_fields.DestinationComputerObjectGuid IS NOT NULL, m365_defender.event.additional_fields.DestinationComputerObjectGuid, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.id IS NOT NULL, host.id, - data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL, host.id, + data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL, host.id, null ), host.target.name = CASE( host.target.name IS NOT NULL, host.target.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.destination.device_name IS NOT NULL, m365_defender.event.destination.device_name, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.name IS NOT NULL, host.name, - data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL, host.name, + data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL, host.name, null ), host.target.ip = CASE( @@ -554,13 +556,13 @@ FROM logs-* action_exists = event.action IS NOT NULL | EVAL user.id = CASE(user.id IS NOT NULL, user.id, data_stream.dataset == "m365_defender.event" AND m365_defender.event.additional_fields.SourceAccountSid IS NOT NULL, m365_defender.event.additional_fields.SourceAccountSid, data_stream.dataset == "m365_defender.event" AND m365_defender.event.initiating_process.account_sid IS NOT NULL, m365_defender.event.initiating_process.account_sid, null), - user.name = CASE(user.name IS NOT NULL, user.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.initiating_process.account_name IS NOT NULL, m365_defender.event.initiating_process.account_name, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND process.user.name IS NOT NULL, process.user.name, null), + user.name = CASE(user.name IS NOT NULL, user.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.initiating_process.account_name IS NOT NULL, m365_defender.event.initiating_process.account_name, data_stream.dataset IN ("m365_defender.event", "m365_defender.alert", "m365_defender.incident") AND process.user.name IS NOT NULL, process.user.name, null), host.ip = CASE(host.ip IS NOT NULL, host.ip, data_stream.dataset == "m365_defender.event" AND user.name IS NULL AND source.ip IS NOT NULL, source.ip, null) | EVAL - event.action = CASE(event.action IS NOT NULL, event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, null) + event.action = CASE(data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL, m365_defender.alert.title, event.action IS NOT NULL, event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, null) | EVAL - host.target.id = CASE(host.target.id IS NOT NULL, host.target.id, data_stream.dataset == "m365_defender.event" AND event.action IN ("samr-query", "dns-query") AND m365_defender.event.additional_fields.DestinationComputerObjectGuid IS NOT NULL, m365_defender.event.additional_fields.DestinationComputerObjectGuid, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.id IS NOT NULL, host.id, data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL, host.id, null), - host.target.name = CASE(host.target.name IS NOT NULL, host.target.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.destination.device_name IS NOT NULL, m365_defender.event.destination.device_name, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.name IS NOT NULL, host.name, data_stream.dataset == "m365_defender.event" AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL, host.name, null), + host.target.id = CASE(host.target.id IS NOT NULL, host.target.id, data_stream.dataset == "m365_defender.event" AND event.action IN ("samr-query", "dns-query") AND m365_defender.event.additional_fields.DestinationComputerObjectGuid IS NOT NULL, m365_defender.event.additional_fields.DestinationComputerObjectGuid, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.id IS NOT NULL, host.id, data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.id IS NOT NULL, host.id, null), + host.target.name = CASE(host.target.name IS NOT NULL, host.target.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.destination.device_name IS NOT NULL, m365_defender.event.destination.device_name, data_stream.dataset IN ("m365_defender.alert", "m365_defender.incident") AND host.name IS NOT NULL, host.name, data_stream.dataset == "m365_defender.event" AND user.name IS NOT NULL AND event.action NOT IN ("samr-query", "dns-query", "logonsuccess", "logonfailed") AND host.name IS NOT NULL, host.name, null), host.target.ip = CASE(host.target.ip IS NOT NULL, host.target.ip, data_stream.dataset == "m365_defender.event" AND destination.ip IS NOT NULL, destination.ip, null), user.target.name = CASE(user.target.name IS NOT NULL, user.target.name, data_stream.dataset == "m365_defender.event" AND m365_defender.event.query.target IS NOT NULL, m365_defender.event.query.target, null), user.target.email = CASE(user.target.email IS NOT NULL, user.target.email, data_stream.dataset == "m365_defender.event" AND email.to.address IS NOT NULL, email.to.address, null), @@ -588,4 +590,4 @@ FROM logs-* - **Multi-valued `host.*` on alerts/incidents** — arrays from evidence flattening; consumers may need `MV_FIRST()` when scalar host targets are required. - **Identity logon (`logonsuccess`/`logonfailed`)** — `host.*` is source sensor; prefer `service.target.name` ← `application.name`, not `host.target.*` from `host.id`. - **Cloud audit rows** — may lack user principal; only `source.ip` available as weak actor context. -- **`event.action` on alert/incident** — array of `detectionStatus` values (`detected`, `prevented`); semantically detection outcome, not hunting `ActionType` or alert title. +- **`event.action` on alert** — ingest populates an array of evidence `detectionStatus` values (`detected`, `prevented`); ES|QL overrides this with `m365_defender.alert.title` (the human-readable detection name) via the CASE block above. The `incident` stream has no equivalent title field and retains `detectionStatus`. From 8e1c0dd56fde3b7a73b792ca712f991a767d470c Mon Sep 17 00:00:00 2001 From: Jordan <51442161+jordansh@users.noreply.github.com> Date: Thu, 2 Jul 2026 15:46:42 +0300 Subject: [PATCH 2/3] m365_defender: remove alert.title override for event.action Reverts the use of m365_defender.alert.title as event.action for the alert stream. The ingest-time event.action (detectionStatus) is preserved. Co-Authored-By: Claude Sonnet 4.6 --- dev/domain/esql-evaluations/enrichmentQuery.esql | 4 ---- dev/domain/esql-evaluations/m365_defender.ts | 1 - 2 files changed, 5 deletions(-) diff --git a/dev/domain/esql-evaluations/enrichmentQuery.esql b/dev/domain/esql-evaluations/enrichmentQuery.esql index 78057f275a0..217b80f9d69 100644 --- a/dev/domain/esql-evaluations/enrichmentQuery.esql +++ b/dev/domain/esql-evaluations/enrichmentQuery.esql @@ -359,10 +359,6 @@ gitlab.pages.msg, data_stream.dataset == "greenhouse.audit" AND greenhouse.audit.event.type IS NOT NULL, greenhouse.audit.event.type, - data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL, - m365_defender.alert.title, - event.action IS NOT NULL, - event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, data_stream.dataset == "openai.images" AND openai.images.source IS NOT NULL, diff --git a/dev/domain/esql-evaluations/m365_defender.ts b/dev/domain/esql-evaluations/m365_defender.ts index b7803e7260c..7e64507fb56 100644 --- a/dev/domain/esql-evaluations/m365_defender.ts +++ b/dev/domain/esql-evaluations/m365_defender.ts @@ -59,7 +59,6 @@ export const m365_defenderEvaluations = { section: "Combined ES|QL \u2014 event action", esql: `| EVAL event.action = CASE( - data_stream.dataset == "m365_defender.alert" AND m365_defender.alert.title IS NOT NULL, m365_defender.alert.title, event.action IS NOT NULL, event.action, data_stream.dataset == "m365_defender.event" AND m365_defender.event.action.type IS NOT NULL, m365_defender.event.action.type, null From 4a1087c9761d2ee24a6bf9d20e9aae545eaf84fa Mon Sep 17 00:00:00 2001 From: Jordan <51442161+jordansh@users.noreply.github.com> Date: Tue, 7 Jul 2026 23:01:18 +0300 Subject: [PATCH 3/3] siem: add graph node check script for p1 integrations Playwright script that navigates to the Kibana graph investigation page, runs a query per integration, and captures entity/action nodes. Outputs an HTML report and results.json to ~/Documents/graph-node-runs//. Co-Authored-By: Claude Sonnet 4.6 --- dev/domain/graph-node-check/.gitignore | 1 + dev/domain/graph-node-check/graph-nodes.js | 530 +++++++++++++++++++++ dev/domain/graph-node-check/package.json | 13 + 3 files changed, 544 insertions(+) create mode 100644 dev/domain/graph-node-check/.gitignore create mode 100644 dev/domain/graph-node-check/graph-nodes.js create mode 100644 dev/domain/graph-node-check/package.json diff --git a/dev/domain/graph-node-check/.gitignore b/dev/domain/graph-node-check/.gitignore new file mode 100644 index 00000000000..c2658d7d1b3 --- /dev/null +++ b/dev/domain/graph-node-check/.gitignore @@ -0,0 +1 @@ +node_modules/ diff --git a/dev/domain/graph-node-check/graph-nodes.js b/dev/domain/graph-node-check/graph-nodes.js new file mode 100644 index 00000000000..76c1ceb0643 --- /dev/null +++ b/dev/domain/graph-node-check/graph-nodes.js @@ -0,0 +1,530 @@ +'use strict'; + +const { chromium } = require('playwright'); +const fs = require('fs'); +const path = require('path'); + +// ─── Timestamp ──────────────────────────────────────────────────────────────── +const _now = new Date(); +const _pad = (n) => String(n).padStart(2, '0'); +const RUN_TIMESTAMP = + `${_now.getFullYear()}-${_pad(_now.getMonth() + 1)}-${_pad(_now.getDate())}` + + `_${_pad(_now.getHours())}h${_pad(_now.getMinutes())}`; + +// ─── Config ─────────────────────────────────────────────────────────────────── +const KIBANA_URL = process.env.KIBANA_URL || 'http://localhost:5601'; + +const CONFIG = { + kibanaUrl: KIBANA_URL, + username: process.env.KIBANA_USER || 'elastic', + password: process.env.KIBANA_PASS || 'changeme', + headless: process.env.HEADLESS === 'true', + outputDir: path.join(process.env.HOME || '~', 'Documents', 'graph-node-runs', RUN_TIMESTAMP), + eventsUrl: `${KIBANA_URL}/app/security/users/events?flyout=(preview:!())&timelineFlyout=(preview:!())&timeline=(activeTab:query,isOpen:!f,query:(expression:%27%27,kind:kuery))&timerange=(global:(linkTo:!(timeline),timerange:(from:%272006-06-29T21:00:00.000Z%27,fromStr:now-20y%2Fd,kind:relative,to:%272026-06-30T10:28:54.288Z%27,toStr:now)),timeline:(linkTo:!(global),timerange:(from:%272006-06-29T21:00:00.000Z%27,fromStr:now-20y%2Fd,kind:relative,to:%272026-06-30T10:28:54.288Z%27,toStr:now)),valueReport:(linkTo:!(),timerange:(from:%272026-05-30T10:28:34.608Z%27,fromStr:now-1M,kind:relative,to:%272026-06-30T10:28:34.608Z%27,toStr:now)))`, + graphSettleMs: 6_000, // time to wait after query submission for graph to load + graphStableMs: 1_000, // polling interval while checking for stable node count + graphTimeoutMs: 20_000, // max time to wait for nodes to stabilize +}; + +// ─── Integrations list ──────────────────────────────────────────────────────── +const INTEGRATIONS = [ + 'qualys_vmdr', 'ti_misp', 'gitlab', 'slack', 'linux', 'citrix_waf', + 'zscaler_zia', 'suricata', 'fortinet_fortigate', 'cisco_secure_email_gateway', + 'cisco_umbrella', 'infoblox_bloxone_ddi', 'cisco_meraki', 'darktrace', + 'microsoft_dhcp', 'openai', 'snort', 'ping_one', 'checkpoint_email', + 'm365_defender', 'corelight', 'salesforce', 'sysdig', 'wiz', 'snyk', + 'prisma_cloud', 'gcp_vertexai', 'jamf_pro', 'azure_ai_foundry', + 'azure_app_service', 'azure_openai', 'microsoft_intune', 'aws_securityhub', + 'aws_cloudtrail_otel', 'aws_vpcflow_otel', 'aws_bedrock', 'aws_bedrock_agentcore', + 'ping_federate', 'cyera', 'entityanalytics_ad', 'entityanalytics_okta', + 'extrahop', 'forgerock', 'greenhouse', 'osquery', 'servicenow', 'tanium', +]; + +const results = []; +function log(msg) { process.stdout.write(msg + '\n'); } +function setupOutput() { fs.mkdirSync(CONFIG.outputDir, { recursive: true }); } + +// ─── Login ──────────────────────────────────────────────────────────────────── +async function login(page) { + log('Logging in…'); + await page.goto(`${CONFIG.kibanaUrl}/login`); + await page.waitForSelector('[data-test-subj="loginUsername"]', { timeout: 30_000 }); + await page.fill('[data-test-subj="loginUsername"]', CONFIG.username); + await page.fill('[data-test-subj="loginPassword"]', CONFIG.password); + await page.click('[data-test-subj="loginSubmit"]'); + await page.waitForURL(/\/app\//, { timeout: 30_000 }); + log('Logged in.'); +} + +// ─── Expand the Visualizations section and wait for graph to load ───────────── +async function expandVisualizationsAndWait(page) { + // Make sure we're on the Overview tab + const overviewTab = page.locator('[data-test-subj="securitySolutionFlyoutOverviewTab"]').first(); + if (await overviewTab.isVisible({ timeout: 3_000 }).catch(() => false)) { + await overviewTab.click(); + await page.waitForTimeout(800); + } + + // Scroll the flyout body down to reveal the Visualizations section + const flyoutBody = page.locator('.euiFlyoutBody__overflow').first(); + if (await flyoutBody.isVisible({ timeout: 2_000 }).catch(() => false)) { + await flyoutBody.evaluate((el) => { el.scrollTop += 400; }); + await page.waitForTimeout(600); + } + + // Expand the Visualizations accordion if it's collapsed + const vizHeader = page.locator('[data-test-subj="securitySolutionFlyoutVisualizationsHeader"]').first(); + if (await vizHeader.isVisible({ timeout: 5_000 }).catch(() => false)) { + const vizContent = page.locator('[data-test-subj="securitySolutionFlyoutVisualizationsContent"]').first(); + const contentVisible = await vizContent.isVisible({ timeout: 1_000 }).catch(() => false); + if (!contentVisible) { + await vizHeader.click(); + await page.waitForTimeout(1_000); + } + } + + // Wait for graph loading skeleton to disappear (up to 20s) + const loading = page.locator('[data-test-subj="securitySolutionFlyoutGraphPreviewLoading"]').first(); + const deadline = Date.now() + 20_000; + while (Date.now() < deadline) { + if (!(await loading.isVisible({ timeout: 500 }).catch(() => false))) break; + await page.waitForTimeout(1_000); + } +} + +// ─── Navigate to graph investigation page via first available flyout ────────── +// Returns the page object to use going forward (may be a new tab). +async function openGraphPage(page, context) { + log('\nNavigating to users/events page…'); + await page.goto(CONFIG.eventsUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 }); + // Wait for the events table to actually render (Kibana can be slow) + await page.waitForSelector('[data-test-subj="expand-event"]', { timeout: 20_000 }).catch(() => null); + await page.waitForTimeout(1_000); + + // Try across multiple pagination pages to find a row with a rendered graph + let pageNum = 1; + while (pageNum <= 5) { + const expandBtns = page.locator('[data-test-subj="expand-event"]'); + const rowCount = await expandBtns.count(); + log(` Page ${pageNum}: ${rowCount} rows`); + + for (let i = 0; i < rowCount; i++) { + const btn = page.locator('[data-test-subj="expand-event"]').nth(i); + if (!(await btn.isVisible({ timeout: 2_000 }).catch(() => false))) continue; + + await btn.click(); + await page.waitForTimeout(2_000); + + // Expand visualizations and wait for graph + await expandVisualizationsAndWait(page); + + const graphLink = page.locator('[data-test-subj="securitySolutionFlyoutGraphPreviewTitleLink"]').first(); + const visible = await graphLink.isVisible({ timeout: 3_000 }).catch(() => false); + + if (visible) { + log(` Found rendered graph on page ${pageNum}, row ${i + 1} — opening graph investigation page…`); + + // The link may open in a new tab — listen before clicking + const newTabPromise = context.waitForEvent('page', { timeout: 8_000 }).catch(() => null); + await graphLink.click(); + + const newTab = await newTabPromise; + if (newTab) { + log(' Opened in new tab — switching to it.'); + await newTab.waitForLoadState('domcontentloaded', { timeout: 20_000 }); + await newTab.waitForTimeout(3_000); + newTab.setDefaultTimeout(30_000); + log(' Arrived at graph investigation page.'); + return newTab; + } + + // Same-page navigation + await page.waitForTimeout(4_000); + log(' Arrived at graph investigation page.'); + return page; + } + + // Close flyout and try next row + await page.keyboard.press('Escape'); + await page.waitForTimeout(600); + } + + // Move to next pagination page + const nextBtn = page.locator('[data-test-subj="pagination-button-next"]').first(); + const nextExists = await nextBtn.isVisible({ timeout: 3_000 }).catch(() => false); + const nextDisabled = await nextBtn.isDisabled({ timeout: 1_000 }).catch(() => true); + if (!nextExists || nextDisabled) break; + + await nextBtn.click(); + await page.waitForTimeout(3_000); + pageNum++; + } + + throw new Error('Could not find a flyout with a rendered graph in the first 5 pages.'); +} + +// ─── Set time filter to Last 20 years ──────────────────────────────────────── +async function setTimeFilter(page) { + log('\nSetting time filter to Last 20 years…'); + + // Click the badge to open the time input, then type -20y and submit + const badge = page.locator('[data-test-subj="dateRangePickerDurationBadge"]').first(); + await badge.waitFor({ state: 'visible', timeout: 15_000 }); + await badge.click(); + await page.waitForTimeout(800); + + // Type -20y — works whether the badge becomes an input or opens a text field + await page.keyboard.press('Meta+a'); + await page.keyboard.type('-20y'); + await page.keyboard.press('Enter'); + await page.waitForTimeout(2_000); + + log(' Time filter set (-20y).'); +} + +// ─── Open the graph search bar via its toggle button ───────────────────────── +async function ensureSearchBarOpen(page) { + // Always click the toggle unconditionally — do NOT check for unifiedQueryInput + // first, because the global Kibana search bar also has that test ID and would + // give a false-positive "already open" early return. + const toggleBtn = page.locator('[data-test-subj="cloudSecurityGraphGraphInvestigationToggleSearch"]').first(); + await toggleBtn.waitFor({ state: 'visible', timeout: 10_000 }); + await toggleBtn.click(); + await page.waitForTimeout(600); + log(' Search bar toggled.'); +} + +// ─── Submit a query in the graph search bar ─────────────────────────────────── +async function setQuery(page, integration) { + // Some integrations require a non-default KQL query to find their documents. + // Keys are integration names; values are the full KQL query string to use. + const QUERY_OVERRIDES = { + openai: 'data_stream.dataset : "openai*"', // event.dataset not in inverted index (logsdb) + salesforce: 'event.module : "salesforce"', // event.dataset/data_stream.dataset not searchable; wildcard also doesn't work + gcp_vertexai: 'data_stream.dataset : "gcp_vertexai*"', // event.dataset wildcard fails (logsdb); data_stream.dataset wildcard works + azure_openai: 'data_stream.dataset : "azure_openai*"', + azure_app_service:'data_stream.dataset : "azure_app_service*"', + azure_ai_foundry: 'data_stream.dataset : "azure_ai_foundry*"', + forgerock: 'data_stream.dataset : "forgerock*"', + }; + const query = QUERY_OVERRIDES[integration] ?? `event.dataset : "${integration}*"`; + + // Use .last() — the graph panel's input is added to the DOM AFTER the toggle + // click, so it comes after the global Kibana search bar (which uses the same + // test ID) and is always the last match. + const input = page.locator('[data-test-subj="unifiedQueryInput"]').last(); + await input.waitFor({ state: 'visible', timeout: 8_000 }); + + // force:true bypasses EUI portal overlays intercepting pointer events + await input.click({ clickCount: 3, force: true }); + await page.waitForTimeout(200); + await page.keyboard.type(query); + + // Dismiss any autocomplete, then submit + await page.keyboard.press('Escape'); + await page.waitForTimeout(200); + await page.keyboard.press('Enter'); +} + +// ─── Wait for graph nodes to stabilise after a query ───────────────────────── +async function waitForGraph(page) { + // The graph renders an EuiProgress bar (role="progressbar") inside the + // investigation container while isFetching is true. Waiting for it to appear + // then disappear confirms the graph reloaded for the new query, preventing + // stale nodes from a previous query being read as the current result. + const container = page.locator('[data-test-subj="cloudSecurityGraphGraphInvestigation"]'); + const progressBar = container.locator('[role="progressbar"]').first(); + + const appeared = await progressBar + .waitFor({ state: 'visible', timeout: 6_000 }) + .then(() => true) + .catch(() => false); + + if (appeared) { + await progressBar.waitFor({ state: 'hidden', timeout: CONFIG.graphTimeoutMs }); + await page.waitForTimeout(500); + } else { + // No progress bar detected — graph loaded instantly or query had no effect + await page.waitForTimeout(2_000); + } + + // Stability check: poll until entity-node count is stable for two consecutive reads + let prev = -1; + let stable = 0; + const deadline = Date.now() + 5_000; + + while (Date.now() < deadline) { + const current = await page + .locator('[data-test-subj="cloudSecurityGraphGraphInvestigationEntityNodeButton"]') + .count(); + if (current === prev) { + stable++; + if (stable >= 2) break; + } else { + stable = 0; + prev = current; + } + await page.waitForTimeout(CONFIG.graphStableMs); + } +} + +// ─── Extract all visible graph nodes, sorted by horizontal position ─────────── +async function extractNodes(page) { + const nodes = await page.evaluate(() => { + const out = []; + + // Entity nodes: the button is the clickable target, but the VALUE text lives + // in a sibling cloudSecurityGraphGraphInvestigationEntityNodeDetails div — + // NOT inside the button itself. Query the details divs directly. + for (const div of document.querySelectorAll( + '[data-test-subj="cloudSecurityGraphGraphInvestigationEntityNodeDetails"]' + )) { + const fullTextEl = div.querySelector('[data-test-subj="fullText"]'); + const tagEl = div.querySelector( + '[data-test-subj="cloudSecurityGraphGraphInvestigationTagText"]' + ); + const rect = div.getBoundingClientRect(); + // Skip elements that are not visible on screen + if (rect.width === 0 && rect.height === 0) continue; + out.push({ + kind: 'entity', + tag: tagEl?.textContent?.trim() || 'Entity', + value: fullTextEl?.textContent?.trim() || '(unknown)', + x: rect.left, + y: rect.top, + }); + } + + // Action / label nodes + for (const div of document.querySelectorAll('[data-test-subj="label-node-text"]')) { + const fullTextEl = div.querySelector('[data-test-subj="fullText"]'); + const rect = div.getBoundingClientRect(); + if (rect.width === 0 && rect.height === 0) continue; + out.push({ + kind: 'action', + tag: 'Action', + value: fullTextEl?.textContent?.trim() || div.textContent?.trim() || '(unknown)', + x: rect.left, + y: rect.top, + }); + } + + // Sort left-to-right (visual graph order), y as tiebreak + out.sort((a, b) => a.x - b.x || a.y - b.y); + return out; + }); + + return nodes; +} + +// ─── Build a text graph visualization from sorted nodes ─────────────────────── +function buildGraphText(nodes) { + if (nodes.length === 0) return '(no nodes found)'; + + // Build a chain string from the sorted node list + const parts = nodes.map((n) => + n.kind === 'entity' ? `[${n.tag}: ${n.value}]` : `→ ${n.value} →` + ); + + // Wrap long chains at action boundaries for readability + const lines = []; + let current = ''; + for (const p of parts) { + if (current && p.startsWith('→') && current.length > 80) { + lines.push(current.trimEnd()); + current = ' '; + } + current += (current.trim() ? ' ' : '') + p; + } + if (current.trim()) lines.push(current.trimEnd()); + + return lines.join('\n'); +} + +// ─── HTML escaping ──────────────────────────────────────────────────────────── +function esc(str) { + return String(str ?? '') + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"'); +} + +// ─── Generate HTML report ───────────────────────────────────────────────────── +function generateReport() { + const withNodes = results.filter((r) => r.nodeCount > 0).length; + const noNodes = results.filter((r) => r.nodeCount === 0 && !r.error).length; + const errors = results.filter((r) => !!r.error).length; + + const rows = results + .map((r) => { + const hasNodes = r.nodeCount > 0; + const rowBg = r.error ? '#fff8f8' : hasNodes ? '#f0fff4' : '#fff'; + const statusText = r.error + ? `⚠️ Error` + : hasNodes + ? `✅ ${r.nodeCount} nodes` + : `❌ No nodes`; + const statusColor = r.error ? '#888' : hasNodes ? '#007a3e' : '#b00020'; + + return ` + ${esc(r.integration)} + ${statusText} + ${r.entityCount} + ${r.labelCount} + 
${esc(r.graphText)}
+ ${r.error ? esc(r.error) : ''} + `; + }) + .join(''); + + const html = ` + + + +Graph Node Test — ${RUN_TIMESTAMP} + + + +

Graph Node Test Report

+

Run: ${RUN_TIMESTAMP}  |  Integrations tested: ${results.length}  |  Time filter: Last 20 years

+ +
+
${withNodes}
With nodes
+
${noNodes}
No nodes
+
${errors}
Errors
+
${results.length}
Total
+
+ + + + + + + + + + + + + ${rows} +
IntegrationStatusEntity nodesAction labelsGraph visualizationError
+ +`; + + const reportPath = path.join(CONFIG.outputDir, 'report.html'); + fs.writeFileSync(reportPath, html); + fs.writeFileSync( + path.join(CONFIG.outputDir, 'results.json'), + JSON.stringify(results, null, 2) + ); + log(`\nReport: file://${reportPath}`); +} + +// ─── Main ───────────────────────────────────────────────────────────────────── +async function main() { + setupOutput(); + + const browser = await chromium.launch({ headless: CONFIG.headless, slowMo: 50 }); + const context = await browser.newContext({ viewport: { width: 1600, height: 900 } }); + const page = await context.newPage(); + page.setDefaultTimeout(30_000); + + let graphPageUrl = null; + + // Re-open the graph investigation page from the saved URL and restore state. + const reopenGraphPage = async () => { + log(' [Reopening graph page…]'); + const gp = await context.newPage(); + gp.setDefaultTimeout(30_000); + await gp.goto(graphPageUrl, { waitUntil: 'domcontentloaded', timeout: 30_000 }); + await gp.waitForTimeout(2_000); + await ensureSearchBarOpen(gp); + await setTimeFilter(gp); + return gp; + }; + + try { + await login(page); + let graphPage = await openGraphPage(page, context); + graphPageUrl = graphPage.url(); + log(` Graph page URL: ${graphPageUrl}`); + await ensureSearchBarOpen(graphPage); + await setTimeFilter(graphPage); + + log(`\nTesting ${INTEGRATIONS.length} integrations…`); + + for (let idx = 0; idx < INTEGRATIONS.length; idx++) { + const integration = INTEGRATIONS[idx]; + log(`\n[${idx + 1}/${INTEGRATIONS.length}] ${integration}`); + + // Recover if the tab was closed between iterations + if (graphPage.isClosed()) { + graphPage = await reopenGraphPage(); + } + + const runIntegration = async (gp) => { + await setQuery(gp, integration); + await waitForGraph(gp); + return extractNodes(gp); + }; + + let nodes; + try { + nodes = await runIntegration(graphPage); + } catch (err) { + if (err.message.includes('closed') && graphPageUrl) { + log(' [Tab crashed — recovering…]'); + try { + graphPage = await reopenGraphPage(); + nodes = await runIntegration(graphPage); + } catch (retryErr) { + log(` Error (after recovery): ${retryErr.message}`); + results.push({ integration, nodeCount: 0, entityCount: 0, labelCount: 0, nodes: [], graphText: '', error: retryErr.message }); + continue; + } + } else { + log(` Error: ${err.message}`); + results.push({ integration, nodeCount: 0, entityCount: 0, labelCount: 0, nodes: [], graphText: '', error: err.message }); + continue; + } + } + + const entities = nodes.filter((n) => n.kind === 'entity'); + const actions = nodes.filter((n) => n.kind === 'action'); + const graphText = buildGraphText(nodes); + + log(` Nodes: ${nodes.length} (${entities.length} entity, ${actions.length} action)`); + if (nodes.length > 0) log(` ${graphText.split('\n')[0]}`); + + results.push({ integration, nodeCount: nodes.length, entityCount: entities.length, labelCount: actions.length, nodes, graphText, error: null }); + } + } catch (err) { + log(`\nFatal error: ${err.message}`); + } finally { + generateReport(); + await browser.close(); + } + + log('\nDone.'); +} + +main().catch((err) => { + log(`Fatal: ${err.stack}`); + process.exit(1); +}); diff --git a/dev/domain/graph-node-check/package.json b/dev/domain/graph-node-check/package.json new file mode 100644 index 00000000000..70a44654fb5 --- /dev/null +++ b/dev/domain/graph-node-check/package.json @@ -0,0 +1,13 @@ +{ + "name": "graph-node-check", + "version": "1.0.0", + "description": "Playwright script that verifies graph nodes render for each P1 integration", + "main": "graph-nodes.js", + "scripts": { + "start": "node graph-nodes.js", + "install-browsers": "npx playwright install chromium" + }, + "dependencies": { + "playwright": "^1.40.0" + } +}