diff --git a/Directory.Packages.props b/Directory.Packages.props
index 2a9fd3f388..633a64f397 100644
--- a/Directory.Packages.props
+++ b/Directory.Packages.props
@@ -55,6 +55,7 @@
+
@@ -86,7 +87,6 @@
-
diff --git a/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj b/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj
index 0f96c00f34..501904008c 100644
--- a/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj
+++ b/src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj
@@ -18,7 +18,7 @@
-
+
diff --git a/src/Elastic.Markdown/Elastic.Markdown.csproj b/src/Elastic.Markdown/Elastic.Markdown.csproj
index ed5f7a3a37..70425216c2 100644
--- a/src/Elastic.Markdown/Elastic.Markdown.csproj
+++ b/src/Elastic.Markdown/Elastic.Markdown.csproj
@@ -46,6 +46,10 @@
+
+
+
+
diff --git a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs
index e5feac3bf4..fd78d70141 100644
--- a/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs
+++ b/src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs
@@ -9,8 +9,9 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Cysharp.IO;
-using Tomlet;
-using Tomlet.Models;
+using Tomlyn;
+using Tomlyn.Model;
+using Tomlyn.Serialization;
namespace Elastic.Markdown.Extensions.DetectionRules;
@@ -35,6 +36,9 @@ public record VersionLockEntry
[JsonSerializable(typeof(Dictionary))]
internal sealed partial class VersionLockJsonContext : JsonSerializerContext;
+[TomlSerializable(typeof(TomlTable))]
+internal sealed partial class DetectionRuleTomlContext : TomlSerializerContext;
+
public record DetectionRuleThreat
{
public required string Framework { get; init; }
@@ -63,8 +67,6 @@ public record DetectionRuleTechnique : DetectionRuleSubTechnique
public record DetectionRule
{
- // TomlParser is stateful across calls; create a new instance per parse to avoid accumulation bugs in serve/reload mode
-
// Cached version lock data, loaded once per build
private static FrozenDictionary? VersionLock;
@@ -141,27 +143,46 @@ public static void InitializeVersionLock(IFileSystem fileSystem, IDirectoryInfo?
[SuppressMessage("Reliability", "CA2012:Use ValueTasks correctly")]
public static DetectionRule From(IFileInfo source)
{
- TomlDocument model;
+ TomlTable model;
try
{
- // Use optimized Utf8StreamReader for better I/O performance
using var reader = new Utf8StreamReader(source.FullName, fileOpenMode: FileOpenMode.Throughput);
var sourceText = Encoding.UTF8.GetString(reader.ReadToEndAsync().GetAwaiter().GetResult());
- model = new TomlParser().Parse(sourceText);
+ model = TomlSerializer.Deserialize(sourceText, DetectionRuleTomlContext.Default.TomlTable)!;
}
catch (Exception e)
{
throw new Exception($"Could not parse toml in: {source.FullName}", e);
}
- if (!model.TryGetValue("metadata", out var node) || node is not TomlTable metadata)
+ if (!model.TryGetValue("metadata", out var metadataObj) || metadataObj is not TomlTable metadata)
throw new Exception($"Could not find metadata section in {source.FullName}");
- if (!model.TryGetValue("rule", out node) || node is not TomlTable rule)
+ if (!model.TryGetValue("rule", out var ruleObj) || ruleObj is not TomlTable rule)
throw new Exception($"Could not find rule section in {source.FullName}");
+ try
+ {
+ return BuildRule(metadata, rule);
+ }
+ catch (Exception e)
+ {
+ throw new Exception($"Could not read fields from: {source.FullName}", e);
+ }
+ }
+
+ internal static DetectionRule FromToml(string toml)
+ {
+ var model = TomlSerializer.Deserialize(toml, DetectionRuleTomlContext.Default.TomlTable)!;
+ var metadata = (TomlTable)model["metadata"]!;
+ var rule = (TomlTable)model["rule"]!;
+ return BuildRule(metadata, rule);
+ }
+
+ private static DetectionRule BuildRule(TomlTable metadata, TomlTable rule)
+ {
var threats = GetThreats(rule);
- var ruleId = rule.GetString("rule_id");
+ var ruleId = GetString(rule, "rule_id");
// Get max_signals from TOML, default to 100 if not specified
var maxSignals = TryGetInt(rule, "max_signals") ?? 100;
@@ -174,13 +195,13 @@ public static DetectionRule From(IFileInfo source)
return new DetectionRule
{
Authors = TryGetStringArray(rule, "author"),
- Description = rule.GetString("description"),
- Type = rule.GetString("type"),
+ Description = GetString(rule, "description"),
+ Type = GetString(rule, "type"),
Language = TryGetString(rule, "language"),
- License = rule.GetString("license"),
+ License = GetString(rule, "license"),
RiskScore = TryGetInt(rule, "risk_score") ?? 0,
RuleId = ruleId,
- Severity = rule.GetString("severity"),
+ Severity = GetString(rule, "severity"),
Tags = TryGetStringArray(rule, "tags"),
Indices = TryGetStringArray(rule, "index"),
References = TryGetStringArray(rule, "references"),
@@ -188,7 +209,7 @@ public static DetectionRule From(IFileInfo source)
Setup = TryGetString(rule, "setup"),
Query = TryGetString(rule, "query"),
Note = TryGetString(rule, "note"),
- Name = rule.GetString("name"),
+ Name = GetString(rule, "name"),
RunsEvery = TryGetString(rule, "interval"),
MaximumAlertsPerExecution = maxSignals,
Version = version,
@@ -200,97 +221,83 @@ public static DetectionRule From(IFileInfo source)
private static DetectionRuleThreat[] GetThreats(TomlTable model)
{
- if (!model.TryGetValue("threat", out var node) || node is not TomlArray threats)
+ if (!model.TryGetValue("threat", out var node) || node is not TomlTableArray threats)
return [];
- var threatsList = new List(threats.ArrayValues.Count);
- foreach (var value in threats)
+ var threatsList = new List(threats.Count);
+ foreach (var threatTable in threats.OfType())
{
- if (value is not TomlTable threatTable)
- continue;
-
- var framework = threatTable.GetString("framework");
+ var framework = GetString(threatTable, "framework");
var techniques = ReadTechniques(threatTable);
-
var tactic = ReadTactic(threatTable);
- var threat = new DetectionRuleThreat
+ threatsList.Add(new DetectionRuleThreat
{
Framework = framework,
- Techniques = techniques.ToArray(),
+ Techniques = techniques,
Tactic = tactic
- };
- threatsList.Add(threat);
+ });
}
- return threatsList.ToArray();
+ return [.. threatsList];
}
- private static IReadOnlyCollection ReadTechniques(TomlTable threatTable)
+ private static DetectionRuleTechnique[] ReadTechniques(TomlTable threatTable)
{
- var techniquesArray = threatTable.TryGetValue("technique", out var node) && node is TomlArray ta ? ta : null;
- if (techniquesArray is null)
+ if (!threatTable.TryGetValue("technique", out var node) || node is not TomlTableArray techniquesArray)
return [];
+
var techniques = new List(techniquesArray.Count);
- foreach (var t in techniquesArray)
+ foreach (var techniqueTable in techniquesArray.OfType())
{
- if (t is not TomlTable techniqueTable)
- continue;
- var id = techniqueTable.GetString("id");
- var name = techniqueTable.GetString("name");
- var reference = techniqueTable.GetString("reference");
techniques.Add(new DetectionRuleTechnique
{
- Id = id,
- Name = name,
- Reference = reference,
- SubTechniques = ReadSubTechniques(techniqueTable).ToArray()
+ Id = GetString(techniqueTable, "id"),
+ Name = GetString(techniqueTable, "name"),
+ Reference = GetString(techniqueTable, "reference"),
+ SubTechniques = ReadSubTechniques(techniqueTable)
});
}
- return techniques;
+ return [.. techniques];
}
- private static IReadOnlyCollection ReadSubTechniques(TomlTable techniqueTable)
+
+ private static DetectionRuleSubTechnique[] ReadSubTechniques(TomlTable techniqueTable)
{
- var subArray = techniqueTable.TryGetValue("subtechnique", out var node) && node is TomlArray ta ? ta : null;
- if (subArray is null)
+ if (!techniqueTable.TryGetValue("subtechnique", out var node) || node is not TomlTableArray subArray)
return [];
+
var subTechniques = new List(subArray.Count);
- foreach (var t in subArray)
+ foreach (var subTable in subArray.OfType())
{
- if (t is not TomlTable subTechniqueTable)
- continue;
- var id = subTechniqueTable.GetString("id");
- var name = subTechniqueTable.GetString("name");
- var reference = subTechniqueTable.GetString("reference");
subTechniques.Add(new DetectionRuleSubTechnique
{
- Id = id,
- Name = name,
- Reference = reference
+ Id = GetString(subTable, "id"),
+ Name = GetString(subTable, "name"),
+ Reference = GetString(subTable, "reference")
});
}
- return subTechniques;
+ return [.. subTechniques];
}
private static DetectionRuleTactic ReadTactic(TomlTable threatTable)
{
- var tacticTable = threatTable.GetSubTable("tactic");
- var id = tacticTable.GetString("id");
- var name = tacticTable.GetString("name");
- var reference = tacticTable.GetString("reference");
+ var tacticTable = (TomlTable)threatTable["tactic"];
return new DetectionRuleTactic
{
- Id = id,
- Name = name,
- Reference = reference
+ Id = GetString(tacticTable, "id"),
+ Name = GetString(tacticTable, "name"),
+ Reference = GetString(tacticTable, "reference")
};
}
+ private static string GetString(TomlTable table, string key) =>
+ (string)table[key];
+
private static string[]? TryGetStringArray(TomlTable table, string key) =>
- table.TryGetValue(key, out var node) && node is TomlArray t ? t.ArrayValues.Select(value => value.StringValue).ToArray() : null;
+ table.TryGetValue(key, out var node) && node is TomlArray t ? t.OfType().ToArray() : null;
private static string? TryGetString(TomlTable table, string key) =>
- table.TryGetValue(key, out var node) && node is TomlString t ? t.Value : null;
+ table.TryGetValue(key, out var node) && node is string s ? s : null;
private static int? TryGetInt(TomlTable table, string key) =>
- table.TryGetValue(key, out var node) && node is TomlLong t ? (int)t.Value : null;
+ table.TryGetValue(key, out var node) && node is long l ? (int)l : null;
}
diff --git a/tests/Elastic.Markdown.Tests/DetectionRules/DetectionRuleParsingTests.cs b/tests/Elastic.Markdown.Tests/DetectionRules/DetectionRuleParsingTests.cs
new file mode 100644
index 0000000000..e9fc329e03
--- /dev/null
+++ b/tests/Elastic.Markdown.Tests/DetectionRules/DetectionRuleParsingTests.cs
@@ -0,0 +1,274 @@
+// Licensed to Elasticsearch B.V under one or more agreements.
+// Elasticsearch B.V licenses this file to you under the Apache 2.0 License.
+// See the LICENSE file in the project root for more information
+
+using AwesomeAssertions;
+using Elastic.Markdown.Extensions.DetectionRules;
+
+namespace Elastic.Markdown.Tests.DetectionRules;
+
+public class DetectionRuleParsingTests
+{
+ private const string MinimalRule = """
+ [metadata]
+ creation_date = "2024/08/01"
+ maturity = "production"
+
+ [rule]
+ author = ["Elastic"]
+ description = "Test rule"
+ name = "Test Rule"
+ rule_id = "abc-123"
+ risk_score = 47
+ severity = "medium"
+ type = "query"
+ license = "Elastic License v2"
+ language = "kuery"
+ query = "process.name : evil.exe"
+ """;
+
+ [Fact]
+ public void FromToml_MinimalRule_ParsesCorrectly()
+ {
+ var rule = DetectionRule.FromToml(MinimalRule);
+
+ rule.Name.Should().Be("Test Rule");
+ rule.RuleId.Should().Be("abc-123");
+ rule.Type.Should().Be("query");
+ rule.RiskScore.Should().Be(47);
+ rule.Severity.Should().Be("medium");
+ rule.Authors.Should().ContainSingle().Which.Should().Be("Elastic");
+ }
+
+ [Fact]
+ public void FromToml_ImplicitIntermediateTable_ParsesTransformInvestigate()
+ {
+ var toml = MinimalRule + """
+
+ [[transform.investigate]]
+ label = "Alerts associated with the user"
+ description = ""
+ providers = []
+
+ [[transform.investigate]]
+ label = "Alerts associated with the host"
+ description = ""
+ providers = []
+
+ [[rule.threat]]
+ framework = "MITRE ATT&CK"
+ [rule.threat.tactic]
+ id = "TA0011"
+ name = "Command and Control"
+ reference = "https://attack.mitre.org/tactics/TA0011/"
+ [[rule.threat.technique]]
+ id = "T1071"
+ name = "Application Layer Protocol"
+ reference = "https://attack.mitre.org/techniques/T1071/"
+ """;
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.Threats.Should().HaveCount(1);
+ rule.Threats[0].Tactic.Id.Should().Be("TA0011");
+ rule.Threats[0].Techniques.Should().HaveCount(1);
+ rule.Threats[0].Techniques[0].Id.Should().Be("T1071");
+ }
+
+ [Fact]
+ public void FromToml_MultiLineStringWithMarkdownLinks_ParsesCorrectly()
+ {
+ // TOML uses """ for multi-line strings; use 4-quote C# raw literals to embed them
+ var toml = """"
+ [metadata]
+ creation_date = "2024/08/01"
+ maturity = "production"
+
+ [rule]
+ author = ["Elastic"]
+ description = "Test rule"
+ name = "Test Rule With Setup"
+ rule_id = "abc-456"
+ risk_score = 73
+ severity = "high"
+ type = "esql"
+ license = "Elastic License v2"
+ setup = """
+ ## Setup
+
+ Follow the [helper guide](https://www.elastic.co/docs/some/path#anchor) to configure.
+
+ Also see [another link](https://example.com).
+ """
+ """";
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.Name.Should().Be("Test Rule With Setup");
+ rule.Setup.Should().Contain("[helper guide]");
+ rule.Setup.Should().Contain("elastic.co/docs");
+ }
+
+ [Fact]
+ public void FromToml_MixedMultiLineDelimiters_ParsesCorrectly()
+ {
+ // Triple-quoted """ appears inside a '''-delimited multi-line string
+ var toml = """"
+ [metadata]
+ creation_date = "2024/08/01"
+ maturity = "production"
+
+ [rule]
+ author = ["Elastic"]
+ description = "Test rule"
+ name = "Mixed Delimiters"
+ rule_id = "abc-789"
+ risk_score = 50
+ severity = "medium"
+ type = "esql"
+ license = "Elastic License v2"
+ query = '''
+ from logs-endpoint.events.process-*
+ | grok process.command_line """e=Access&y=Guest&h=(?[^&]+)&p"""
+ | where Esql.server is not null
+ '''
+ note = """## Triage
+ Check the process tree."""
+ """";
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.Query.Should().Contain("grok process.command_line");
+ rule.Query.Should().Contain("e=Access");
+ rule.Note.Should().Contain("Triage");
+ }
+
+ [Fact]
+ public void FromToml_DeprecatedRule_ParsesDeprecationDate()
+ {
+ var toml = """
+ [metadata]
+ creation_date = "2024/08/01"
+ deprecation_date = "2025/03/15"
+ maturity = "deprecated"
+
+ [rule]
+ author = ["Elastic"]
+ description = "Deprecated rule"
+ name = "Old Rule"
+ rule_id = "dep-001"
+ risk_score = 21
+ severity = "low"
+ type = "query"
+ license = "Elastic License v2"
+ """;
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.DeprecationDate.Should().Be("2025/03/15");
+ rule.Maturity.Should().Be("deprecated");
+ }
+
+ [Fact]
+ public void FromToml_ThreatWithSubTechniques_ParsesFullHierarchy()
+ {
+ var toml = MinimalRule + """
+
+ [[rule.threat]]
+ framework = "MITRE ATT&CK"
+ [rule.threat.tactic]
+ id = "TA0001"
+ name = "Initial Access"
+ reference = "https://attack.mitre.org/tactics/TA0001/"
+ [[rule.threat.technique]]
+ id = "T1566"
+ name = "Phishing"
+ reference = "https://attack.mitre.org/techniques/T1566/"
+ [[rule.threat.technique.subtechnique]]
+ id = "T1566.001"
+ name = "Spearphishing Attachment"
+ reference = "https://attack.mitre.org/techniques/T1566/001/"
+ [[rule.threat.technique.subtechnique]]
+ id = "T1566.002"
+ name = "Spearphishing Link"
+ reference = "https://attack.mitre.org/techniques/T1566/002/"
+ """;
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.Threats.Should().HaveCount(1);
+ var technique = rule.Threats[0].Techniques[0];
+ technique.Id.Should().Be("T1566");
+ technique.SubTechniques.Should().HaveCount(2);
+ technique.SubTechniques[0].Id.Should().Be("T1566.001");
+ technique.SubTechniques[1].Id.Should().Be("T1566.002");
+ }
+
+ [Fact]
+ public void FromToml_MultipleThreats_ParsesAll()
+ {
+ var toml = MinimalRule + """
+
+ [[rule.threat]]
+ framework = "MITRE ATT&CK"
+ [rule.threat.tactic]
+ id = "TA0011"
+ name = "Command and Control"
+ reference = "https://attack.mitre.org/tactics/TA0011/"
+
+ [[rule.threat]]
+ framework = "MITRE ATT&CK"
+ [rule.threat.tactic]
+ id = "TA0005"
+ name = "Defense Evasion"
+ reference = "https://attack.mitre.org/tactics/TA0005/"
+ """;
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.Threats.Should().HaveCount(2);
+ rule.Threats[0].Tactic.Id.Should().Be("TA0011");
+ rule.Threats[1].Tactic.Id.Should().Be("TA0005");
+ }
+
+ [Fact]
+ public void FromToml_OptionalFieldsMissing_DefaultsCorrectly()
+ {
+ var rule = DetectionRule.FromToml(MinimalRule);
+
+ rule.Setup.Should().BeNull();
+ rule.Note.Should().BeNull();
+ rule.References.Should().BeNull();
+ rule.Indices.Should().BeNull();
+ rule.RunsEvery.Should().BeNull();
+ rule.IndicesFromDateMath.Should().BeNull();
+ rule.DeprecationDate.Should().BeNull();
+ rule.MaximumAlertsPerExecution.Should().Be(100);
+ rule.Threats.Should().BeEmpty();
+ }
+
+ [Fact]
+ public void FromToml_DomainTag_ExtractedCorrectly()
+ {
+ var toml = """
+ [metadata]
+ creation_date = "2024/08/01"
+ maturity = "production"
+
+ [rule]
+ author = ["Elastic"]
+ description = "Test"
+ name = "Domain Test"
+ rule_id = "dom-001"
+ risk_score = 50
+ severity = "medium"
+ type = "query"
+ license = "Elastic License v2"
+ tags = ["Domain: Endpoint", "Use Case: Threat Detection"]
+ """;
+
+ var rule = DetectionRule.FromToml(toml);
+
+ rule.Domain.Should().Be("Endpoint");
+ }
+}