Skip to content

Swap Tomlet with Tomlyn, add tests #3214

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Mpdreamz merged 1 commit into from
Apr 30, 2026
Merged

Swap Tomlet with Tomlyn, add tests #3214

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Directory.Packages.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
<PackageVersion Include="MartinCostello.Logging.XUnit.v3" Version="0.7.1" />
<PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="10.0.3" />
<PackageVersion Include="Microsoft.OpenApi" Version="3.1.1" />
<PackageVersion Include="Tomlyn" Version="2.3.2" />
<PackageVersion Include="TUnit" Version="0.25.21" />
<PackageVersion Include="xunit.v3.extensibility.core" Version="2.0.2" />
<PackageVersion Include="WireMock.Net" Version="1.6.11" />
Expand Down Expand Up @@ -86,7 +87,6 @@
<PackageVersion Include="Riok.Mapperly" Version="4.2.1" PrivateAssets="all" IncludeAssets="runtime; build; native; contentfiles; analyzers; buildtransitive" />
<PackageVersion Include="Proc" Version="0.13.0" />
<PackageVersion Include="RazorSlices" Version="0.9.5" />
<PackageVersion Include="Samboy063.Tomlet" Version="6.0.0" />
<PackageVersion Include="Sep" Version="0.11.0" />
<PackageVersion Include="Slugify.Core" Version="4.0.1" />
<PackageVersion Include="SoftCircuits.IniFileParser" Version="2.7.0" />
Expand Down
2 changes: 1 addition & 1 deletion src/Elastic.Documentation.Configuration/Elastic.Documentation.Configuration.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
<PackageReference Include="NetEscapades.EnumGenerators" />
<PackageReference Include="Nullean.ScopedFileSystem" />
<PackageReference Include="System.IO.Abstractions.TestingHelpers" />
<PackageReference Include="Samboy063.Tomlet" />
<PackageReference Include="Tomlyn" />
<PackageReference Include="Vecc.YamlDotNet.Analyzers.StaticGenerator" />
<PackageReference Include="YamlDotNet" />
</ItemGroup>
Expand Down
4 changes: 4 additions & 0 deletions src/Elastic.Markdown/Elastic.Markdown.csproj
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@
<ProjectReference Include="..\Elastic.Documentation.Svg\Elastic.Documentation.Svg.csproj" />
</ItemGroup>

<ItemGroup>
<InternalsVisibleTo Include="Elastic.Markdown.Tests"/>
</ItemGroup>

<ItemGroup>
<UpToDateCheckInput Remove="Myst\Directives\AppliesSwitch\AppliesItemView.cshtml" />
<UpToDateCheckInput Remove="Myst\Directives\AppliesSwitch\AppliesSwitchView.cshtml" />
Expand Down
137 changes: 72 additions & 65 deletions src/Elastic.Markdown/Extensions/DetectionRules/DetectionRule.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,8 +9,9 @@
using System.Text.Json;
using System.Text.Json.Serialization;
using Cysharp.IO;
using Tomlet;
using Tomlet.Models;
using Tomlyn;
using Tomlyn.Model;
using Tomlyn.Serialization;

namespace Elastic.Markdown.Extensions.DetectionRules;

Expand All @@ -35,6 +36,9 @@ public record VersionLockEntry
[JsonSerializable(typeof(Dictionary<string, VersionLockEntry>))]
internal sealed partial class VersionLockJsonContext : JsonSerializerContext;

[TomlSerializable(typeof(TomlTable))]
internal sealed partial class DetectionRuleTomlContext : TomlSerializerContext;

public record DetectionRuleThreat
{
public required string Framework { get; init; }
Expand Down Expand Up @@ -63,8 +67,6 @@ public record DetectionRuleTechnique : DetectionRuleSubTechnique

public record DetectionRule
{
// TomlParser is stateful across calls; create a new instance per parse to avoid accumulation bugs in serve/reload mode

// Cached version lock data, loaded once per build
private static FrozenDictionary<string, VersionLockEntry>? VersionLock;

Expand Down Expand Up @@ -141,27 +143,46 @@ public static void InitializeVersionLock(IFileSystem fileSystem, IDirectoryInfo?
[SuppressMessage("Reliability", "CA2012:Use ValueTasks correctly")]
public static DetectionRule From(IFileInfo source)
{
TomlDocument model;
TomlTable model;
try
{
// Use optimized Utf8StreamReader for better I/O performance
using var reader = new Utf8StreamReader(source.FullName, fileOpenMode: FileOpenMode.Throughput);
var sourceText = Encoding.UTF8.GetString(reader.ReadToEndAsync().GetAwaiter().GetResult());
model = new TomlParser().Parse(sourceText);
model = TomlSerializer.Deserialize(sourceText, DetectionRuleTomlContext.Default.TomlTable)!;
}
catch (Exception e)
{
throw new Exception($"Could not parse toml in: {source.FullName}", e);
}

if (!model.TryGetValue("metadata", out var node) || node is not TomlTable metadata)
if (!model.TryGetValue("metadata", out var metadataObj) || metadataObj is not TomlTable metadata)
throw new Exception($"Could not find metadata section in {source.FullName}");

if (!model.TryGetValue("rule", out node) || node is not TomlTable rule)
if (!model.TryGetValue("rule", out var ruleObj) || ruleObj is not TomlTable rule)
throw new Exception($"Could not find rule section in {source.FullName}");

try
{
return BuildRule(metadata, rule);
}
catch (Exception e)
{
throw new Exception($"Could not read fields from: {source.FullName}", e);
}
}

internal static DetectionRule FromToml(string toml)
{
var model = TomlSerializer.Deserialize(toml, DetectionRuleTomlContext.Default.TomlTable)!;
var metadata = (TomlTable)model["metadata"]!;
var rule = (TomlTable)model["rule"]!;
return BuildRule(metadata, rule);
}

private static DetectionRule BuildRule(TomlTable metadata, TomlTable rule)
{
var threats = GetThreats(rule);
var ruleId = rule.GetString("rule_id");
var ruleId = GetString(rule, "rule_id");

// Get max_signals from TOML, default to 100 if not specified
var maxSignals = TryGetInt(rule, "max_signals") ?? 100;
Expand All @@ -174,21 +195,21 @@ public static DetectionRule From(IFileInfo source)
return new DetectionRule
{
Authors = TryGetStringArray(rule, "author"),
Description = rule.GetString("description"),
Type = rule.GetString("type"),
Description = GetString(rule, "description"),
Type = GetString(rule, "type"),
Language = TryGetString(rule, "language"),
License = rule.GetString("license"),
License = GetString(rule, "license"),
RiskScore = TryGetInt(rule, "risk_score") ?? 0,
RuleId = ruleId,
Severity = rule.GetString("severity"),
Severity = GetString(rule, "severity"),
Tags = TryGetStringArray(rule, "tags"),
Indices = TryGetStringArray(rule, "index"),
References = TryGetStringArray(rule, "references"),
IndicesFromDateMath = TryGetString(rule, "from"),
Setup = TryGetString(rule, "setup"),
Query = TryGetString(rule, "query"),
Note = TryGetString(rule, "note"),
Name = rule.GetString("name"),
Name = GetString(rule, "name"),
RunsEvery = TryGetString(rule, "interval"),
MaximumAlertsPerExecution = maxSignals,
Version = version,
Expand All @@ -200,97 +221,83 @@ public static DetectionRule From(IFileInfo source)

private static DetectionRuleThreat[] GetThreats(TomlTable model)
{
if (!model.TryGetValue("threat", out var node) || node is not TomlArray threats)
if (!model.TryGetValue("threat", out var node) || node is not TomlTableArray threats)
return [];

var threatsList = new List<DetectionRuleThreat>(threats.ArrayValues.Count);
foreach (var value in threats)
var threatsList = new List<DetectionRuleThreat>(threats.Count);
foreach (var threatTable in threats.OfType<TomlTable>())
{
if (value is not TomlTable threatTable)
continue;

var framework = threatTable.GetString("framework");
var framework = GetString(threatTable, "framework");
var techniques = ReadTechniques(threatTable);

var tactic = ReadTactic(threatTable);
var threat = new DetectionRuleThreat
threatsList.Add(new DetectionRuleThreat
{
Framework = framework,
Techniques = techniques.ToArray(),
Techniques = techniques,
Tactic = tactic
};
threatsList.Add(threat);
});
}

return threatsList.ToArray();
return [.. threatsList];
}

private static IReadOnlyCollection<DetectionRuleTechnique> ReadTechniques(TomlTable threatTable)
private static DetectionRuleTechnique[] ReadTechniques(TomlTable threatTable)
{
var techniquesArray = threatTable.TryGetValue("technique", out var node) && node is TomlArray ta ? ta : null;
if (techniquesArray is null)
if (!threatTable.TryGetValue("technique", out var node) || node is not TomlTableArray techniquesArray)
return [];

var techniques = new List<DetectionRuleTechnique>(techniquesArray.Count);
foreach (var t in techniquesArray)
foreach (var techniqueTable in techniquesArray.OfType<TomlTable>())
{
if (t is not TomlTable techniqueTable)
continue;
var id = techniqueTable.GetString("id");
var name = techniqueTable.GetString("name");
var reference = techniqueTable.GetString("reference");
techniques.Add(new DetectionRuleTechnique
{
Id = id,
Name = name,
Reference = reference,
SubTechniques = ReadSubTechniques(techniqueTable).ToArray()
Id = GetString(techniqueTable, "id"),
Name = GetString(techniqueTable, "name"),
Reference = GetString(techniqueTable, "reference"),
SubTechniques = ReadSubTechniques(techniqueTable)
});
}
return techniques;
return [.. techniques];
}
private static IReadOnlyCollection<DetectionRuleSubTechnique> ReadSubTechniques(TomlTable techniqueTable)

private static DetectionRuleSubTechnique[] ReadSubTechniques(TomlTable techniqueTable)
{
var subArray = techniqueTable.TryGetValue("subtechnique", out var node) && node is TomlArray ta ? ta : null;
if (subArray is null)
if (!techniqueTable.TryGetValue("subtechnique", out var node) || node is not TomlTableArray subArray)
return [];

var subTechniques = new List<DetectionRuleSubTechnique>(subArray.Count);
foreach (var t in subArray)
foreach (var subTable in subArray.OfType<TomlTable>())
{
if (t is not TomlTable subTechniqueTable)
continue;
var id = subTechniqueTable.GetString("id");
var name = subTechniqueTable.GetString("name");
var reference = subTechniqueTable.GetString("reference");
subTechniques.Add(new DetectionRuleSubTechnique
{
Id = id,
Name = name,
Reference = reference
Id = GetString(subTable, "id"),
Name = GetString(subTable, "name"),
Reference = GetString(subTable, "reference")
});
}
return subTechniques;
return [.. subTechniques];
}

private static DetectionRuleTactic ReadTactic(TomlTable threatTable)
{
var tacticTable = threatTable.GetSubTable("tactic");
var id = tacticTable.GetString("id");
var name = tacticTable.GetString("name");
var reference = tacticTable.GetString("reference");
var tacticTable = (TomlTable)threatTable["tactic"];
return new DetectionRuleTactic
{
Id = id,
Name = name,
Reference = reference
Id = GetString(tacticTable, "id"),
Name = GetString(tacticTable, "name"),
Reference = GetString(tacticTable, "reference")
};
}

private static string GetString(TomlTable table, string key) =>
(string)table[key];

private static string[]? TryGetStringArray(TomlTable table, string key) =>
table.TryGetValue(key, out var node) && node is TomlArray t ? t.ArrayValues.Select(value => value.StringValue).ToArray() : null;
table.TryGetValue(key, out var node) && node is TomlArray t ? t.OfType<string>().ToArray() : null;

private static string? TryGetString(TomlTable table, string key) =>
table.TryGetValue(key, out var node) && node is TomlString t ? t.Value : null;
table.TryGetValue(key, out var node) && node is string s ? s : null;

private static int? TryGetInt(TomlTable table, string key) =>
table.TryGetValue(key, out var node) && node is TomlLong t ? (int)t.Value : null;
table.TryGetValue(key, out var node) && node is long l ? (int)l : null;
}
Loading
Loading