Initial shell of new algorithm to enable sign-tool signing outside of arcade

.github/copilot-instructions.md

@@ -20,7 +20,6 @@ Arcade SDK is the core infrastructure tooling used across the .NET ecosystem for
 **NEVER CANCEL BUILDS OR TESTS** - they may take 90+ minutes. Always use appropriate timeouts.
 
 ```bash
-
 # Full restore, build, and test - TAKES 90+ MINUTES - NEVER CANCEL
 timeout 6000 ./build.sh --restore --build
 # Set timeout to 100+ minutes (6000 seconds) for build commands
@@ -35,8 +34,6 @@ timeout 1800 ./build.sh --restore
 
 # Clean build artifacts
 ./build.sh --clean
-```
-
 ### Platform-Specific Commands
 - **Linux/macOS**: `./build.sh`, `./test.sh`, `./restore.sh`
 - **Windows**: `Build.cmd`, `Test.cmd`, `Restore.cmd`
@@ -62,15 +59,13 @@ timeout 1800 ./build.sh --restore
 - **Microsoft.DotNet.PackageTesting**: Automated package validation testing
 
 ## Build Artifacts Structure
-```
 artifacts/
 ├── bin/                    # Compiled binaries by project/configuration
 ├── packages/               # Generated NuGet packages (Shipping/NonShipping)
 ├── TestResults/            # Unit and integration test results
 ├── log/                    # Build logs and binary logs (.binlog)
 ├── tmp/                    # Temporary build artifacts
 └── toolset/               # Downloaded build tools and dependencies
-```
 
 ## Validation and Testing
 
@@ -88,6 +83,9 @@ timeout 6000 ./build.sh --restore --build --configuration Release --test
 - **SDK Tests**: Validate Arcade SDK works in sample projects
 - **Packaging Tests**: Ensure generated packages are valid
 
+### Writing Tests
+- When writing tests in this repo, **do not implement new mock file system wrappers**; use the existing `Microsoft.Arcade.Test.Common.MockFileSystem`. Update the shared `IFileSystem`/`MockFileSystem` to support binary-safe operations needed for in-place updates, rather than adding separate binary storage dictionaries or per-test filesystem adapters.
+
 ## Common Development Tasks
 
 ### Adding New Build Tasks
@@ -166,3 +164,9 @@ timeout 6000 ./build.sh --restore --build --configuration Release --test
 - **Discussions**: Use dotnet/arcade discussions for questions
 - **Documentation**: See `/Documentation/` folder for detailed guides
 - **Contact**: @dotnet/dnceng team for infrastructure issues
+
+## Project-Specific Notes
+- Microsoft.DotNet.RecursiveSigning architecture and design references now live in `src/Microsoft.DotNet.RecursiveSigning/docs/`. Consult the markdown files in that directory for component guidance before editing related code.
+
+### Iterative Signing Workflow
+- Sign all nodes ready for signing, update the graph, then repack containers whose signable children are signed; proceed round-by-round.

Arcade.slnx

@@ -29,6 +29,7 @@
     <Project Path="src/Microsoft.DotNet.NuGetRepack/tests/Microsoft.DotNet.NuGetRepack.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.PackageTesting.Tests/Microsoft.DotNet.PackageTesting.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.RemoteExecutor/tests/Microsoft.DotNet.RemoteExecutor.Tests.csproj" />
+    <Project Path="src/Microsoft.DotNet.RecursiveSigning.Tests/Microsoft.DotNet.RecursiveSigning.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.SetupNugetSources.Tests/Microsoft.DotNet.SetupNugetSources.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.SignTool.Tests/Microsoft.DotNet.SignTool.Tests.csproj" />
     <Project Path="src/Microsoft.DotNet.SourceBuild/tests/Microsoft.DotNet.SourceBuild.Tasks.Tests.csproj" />
@@ -71,6 +72,8 @@
   <Project Path="src/Microsoft.DotNet.RemoteExecutor/src/Microsoft.DotNet.RemoteExecutor.csproj" />
   <Project Path="src/Microsoft.DotNet.SharedFramework.Sdk/Microsoft.DotNet.SharedFramework.Sdk.csproj" />
   <Project Path="src/Microsoft.DotNet.SignTool/Microsoft.DotNet.SignTool.csproj" />
+  <Project Path="src/Microsoft.DotNet.RecursiveSigning/Microsoft.DotNet.RecursiveSigning.csproj" />
+  <Project Path="src/Microsoft.DotNet.RecursiveSigning.Cli/Microsoft.DotNet.RecursiveSigning.Cli.csproj" />
   <Project Path="src/Microsoft.DotNet.SourceBuild/tasks/Microsoft.DotNet.SourceBuild.Tasks.csproj" />
   <Project Path="src/Microsoft.DotNet.StrongName/Microsoft.DotNet.StrongName.csproj" />
   <Project Path="src/Microsoft.DotNet.Tar/Microsoft.DotNet.Tar.csproj" />

Directory.Packages.props

@@ -63,6 +63,7 @@
     <PackageVersion Include="Microsoft.Extensions.Http" Version="$(MicrosoftExtensionsHttpVersion)" />
     <PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="$(MicrosoftExtensionsFileProvidersAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.Extensions.FileSystemGlobbing" Version="$(MicrosoftExtensionsFileSystemGlobbingVersion)" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsLoggingAbstractionsVersion)" />
     <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsLoggingConsoleVersion)" />
     <PackageVersion Include="System.Collections.Immutable" Version="$(SystemCollectionsImmutableVersion)" />
     <PackageVersion Include="System.Composition" Version="$(SystemCompositionVersion)" />
@@ -92,6 +93,7 @@
     <PackageVersion Include="Microsoft.Data.OData" Version="5.8.4" />
     <PackageVersion Include="Microsoft.DataServices.Client" Version="$(MicrosoftDataServicesClientVersion)" />
     <PackageVersion Include="Microsoft.Diagnostics.Runtime" Version="1.0.5" />
+    <PackageVersion Include="Microsoft.Ess.EsrpCli" Version="5.1.20250306.4" />
     <PackageVersion Include="Microsoft.Identity.Client" Version="4.73.1" />
     <PackageVersion Include="Microsoft.OpenApi" Version="1.3.2" />
     <PackageVersion Include="Microsoft.OpenApi.Readers" Version="1.3.2" />

NuGet.config

@@ -16,6 +16,7 @@
     <add key="dotnet10-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet10-transport/nuget/v3/index.json" />
     <add key="dotnet11" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet11/nuget/v3/index.json" />
     <add key="dotnet11-transport" value="https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet11-transport/nuget/v3/index.json" />
+    <add key="MicroBuildToolset" value="https://pkgs.dev.azure.com/devdiv/_packaging/Engineering/nuget/v3/index.json" />
   </packageSources>
   <packageSourceMapping>
     <clear />
@@ -59,6 +60,9 @@
     <packageSource key="dotnet11-transport">
       <package pattern="microsoft.*" />
     </packageSource>
+    <packageSource key="MicroBuildToolset">
+      <package pattern="microsoft.*" />
+    </packageSource>
   </packageSourceMapping>
   <disabledPackageSources>
     <clear />

eng/BuildTask.Packages.props

@@ -17,6 +17,7 @@
     <PackageVersion Update="Microsoft.Extensions.DependencyModel" Version="8.0.2" />
     <PackageVersion Update="Microsoft.Extensions.Http" Version="8.0.0" />
     <PackageVersion Update="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
+    <PackageVersion Update="Microsoft.Extensions.Logging.Abstractions" Version="8.0.3" />
   </ItemGroup>
 
 </Project>

eng/Version.Details.props

@@ -34,6 +34,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>9.0.0</MicrosoftExtensionsFileProvidersAbstractionsPackageVersion>
     <MicrosoftExtensionsFileSystemGlobbingPackageVersion>9.0.0</MicrosoftExtensionsFileSystemGlobbingPackageVersion>
     <MicrosoftExtensionsHttpPackageVersion>9.0.0</MicrosoftExtensionsHttpPackageVersion>
+    <MicrosoftExtensionsLoggingAbstractionsPackageVersion>9.0.0</MicrosoftExtensionsLoggingAbstractionsPackageVersion>
     <MicrosoftExtensionsLoggingConsolePackageVersion>9.0.0</MicrosoftExtensionsLoggingConsolePackageVersion>
     <SystemCollectionsImmutablePackageVersion>9.0.0</SystemCollectionsImmutablePackageVersion>
     <SystemFormatsAsn1PackageVersion>9.0.0</SystemFormatsAsn1PackageVersion>
@@ -88,6 +89,7 @@ This file should be imported by eng/Versions.props
     <MicrosoftExtensionsFileProvidersAbstractionsVersion>$(MicrosoftExtensionsFileProvidersAbstractionsPackageVersion)</MicrosoftExtensionsFileProvidersAbstractionsVersion>
     <MicrosoftExtensionsFileSystemGlobbingVersion>$(MicrosoftExtensionsFileSystemGlobbingPackageVersion)</MicrosoftExtensionsFileSystemGlobbingVersion>
     <MicrosoftExtensionsHttpVersion>$(MicrosoftExtensionsHttpPackageVersion)</MicrosoftExtensionsHttpVersion>
+    <MicrosoftExtensionsLoggingAbstractionsVersion>$(MicrosoftExtensionsLoggingAbstractionsPackageVersion)</MicrosoftExtensionsLoggingAbstractionsVersion>
     <MicrosoftExtensionsLoggingConsoleVersion>$(MicrosoftExtensionsLoggingConsolePackageVersion)</MicrosoftExtensionsLoggingConsoleVersion>
     <SystemCollectionsImmutableVersion>$(SystemCollectionsImmutablePackageVersion)</SystemCollectionsImmutableVersion>
     <SystemFormatsAsn1Version>$(SystemFormatsAsn1PackageVersion)</SystemFormatsAsn1Version>

eng/Version.Details.xml

@@ -102,6 +102,10 @@
       <Uri>https://github.com/dotnet/runtime</Uri>
       <Sha>d3981726bc8b0e179db50301daf9f22d42393096</Sha>
     </Dependency>
+    <Dependency Name="Microsoft.Extensions.Logging.Abstractions" Version="9.0.0">
+      <Uri>https://github.com/dotnet/runtime</Uri>
+      <Sha>d3981726bc8b0e179db50301daf9f22d42393096</Sha>
+    </Dependency>
     <Dependency Name="System.Text.Encodings.Web" Version="9.0.0">
       <Uri>https://github.com/dotnet/runtime</Uri>
       <Sha>d3981726bc8b0e179db50301daf9f22d42393096</Sha>

src/Common/Microsoft.Arcade.Common/FileSystem.cs

@@ -53,5 +53,19 @@ public void WriteToFile(string path, string content)
         /// <param name="targetPath">Target path that is relative to base path.</param>
         /// <exception cref="NotImplementedException"></exception>
         public virtual string GetRelativePath(string basePath, string targetPath) => throw new NotImplementedException("Not supported in default FileSystem implementation");
+
+        public byte[] ReadAllBytes(string path) => File.ReadAllBytes(path);
+
+        public void WriteAllBytes(string path, byte[] bytes)
+        {
+            string? dirPath = Path.GetDirectoryName(path);
+            if (!string.IsNullOrEmpty(dirPath))
+            {
+                Directory.CreateDirectory(dirPath);
+            }
+            File.WriteAllBytes(path, bytes);
+        }
+
+        public long GetFileLength(string path) => new FileInfo(path).Length;
     }
 }

src/Common/Microsoft.Arcade.Common/IFileSystem.cs

@@ -39,5 +39,11 @@ public interface IFileSystem
         FileAttributes GetAttributes(string path);
 
         string GetRelativePath(string basePath, string targetPath);
+
+        byte[] ReadAllBytes(string path);
+
+        void WriteAllBytes(string path, byte[] bytes);
+
+        long GetFileLength(string path);
     }
 }

src/Common/Microsoft.Arcade.Test.Common/MockFileSystem.cs

@@ -4,18 +4,23 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
+using System.Text.Unicode;
 using Microsoft.Arcade.Common;
 
 #nullable enable
 namespace Microsoft.Arcade.Test.Common
 {
+
     public class MockFileSystem : IFileSystem
     {
         #region File system state
 
-        public HashSet<string> Directories { get; }    
+        public HashSet<string> Directories { get; }
 
-        public Dictionary<string, string> Files { get; }
+        public Dictionary<string, byte[]> Files { get; }
+
+
 
         public List<string> RemovedFiles { get; } = new();
 
@@ -29,7 +34,7 @@ public MockFileSystem(
             string directorySeparator = "/")
         {
             Directories = new(directories ?? new string[0]);
-            Files = files ?? new();
+            Files = files?.ToDictionary(f => f.Key, f => System.Text.Encoding.UTF8.GetBytes(f.Value)) ?? new();
             DirectorySeparator = directorySeparator;
         }
 
@@ -59,12 +64,17 @@ public void DeleteFile(string path)
 
         public string PathCombine(string path1, string path2, string path3) => path1 + DirectorySeparator + path2 + DirectorySeparator + path3;
 
-        public void WriteToFile(string path, string content) => Files[path] = content;
+        public void WriteToFile(string path, string content) => Files[path] = System.Text.Encoding.UTF8.GetBytes(content);
+
+        public string ReadToString(string path) => System.Text.Encoding.UTF8.GetString(Files[path]);
 
         public void CopyFile(string sourceFileName, string destFileName, bool overwrite = false) => Files[destFileName] = Files[sourceFileName];
 
         public Stream GetFileStream(string path, FileMode mode, FileAccess access)
-            => FileExists(path) ? new MemoryStream() : new MockFileStream(this, path);
+        {
+            // Always use MockFileStream which handles both read and write correctly
+            return new MockFileStream(this, path, mode, access);
+        }
 
         public FileAttributes GetAttributes(string path)
         {
@@ -90,34 +100,86 @@ public string GetRelativePath(string basePath, string targetPath)
             return targetPath.Replace(basePath, "").TrimStart('/', '\\');
         }
 
+        public byte[] ReadAllBytes(string path)
+        {
+            if (!Files.ContainsKey(path))
+            {
+                throw new FileNotFoundException($"File not found: {path}");
+            }
+            return Files[path];
+        }
+
+        public void WriteAllBytes(string path, byte[] bytes)
+        {
+            Files[path] = bytes;
+        }
+
+        public long GetFileLength(string path)
+        {
+            if (!Files.ContainsKey(path))
+            {
+                throw new FileNotFoundException($"File not found: {path}");
+            }
+            return Files[path].LongLength;
+        }
+
         #endregion
 
         /// <summary>
-        /// Allows to write to a stream that will end up in the MockFileSystem.
+        /// Allows to read and write to a stream that will end up in the MockFileSystem.
         /// </summary>
         private class MockFileStream : MemoryStream
         {
             private readonly MockFileSystem _fileSystem;
             private readonly string _path;
+            private readonly FileAccess _access;
             private bool _disposed = false;
 
-            public MockFileStream(MockFileSystem fileSystem, string path)
-                : base(fileSystem.FileExists(path) ? System.Text.Encoding.UTF8.GetBytes(fileSystem.Files[path]) : new byte[2048])
+            public MockFileStream(MockFileSystem fileSystem, string path, FileMode mode, FileAccess access)
             {
                 _fileSystem = fileSystem;
                 _path = path;
+                _access = access;
+
+                // Initialize the stream based on mode and existing file content
+                if (mode == FileMode.Open || mode == FileMode.Append)
+                {
+                    if (fileSystem.FileExists(path))
+                    {
+                        byte[] existingContent = fileSystem.Files[path];
+                        Write(existingContent, 0, existingContent.Length);
+
+                        if (mode == FileMode.Open)
+                        {
+                            Seek(0, SeekOrigin.Begin); // Reset to beginning for read
+                        }
+                        // For Append mode, position is already at the end
+                    }
+                    else if (mode == FileMode.Open)
+                    {
+                        throw new FileNotFoundException($"File not found: {path}");
+                    }
+                }
+                else if (mode == FileMode.Create || mode == FileMode.CreateNew || mode == FileMode.Truncate)
+                {
+                    // Start with an empty stream
+                    if (mode == FileMode.CreateNew && fileSystem.FileExists(path))
+                    {
+                        throw new IOException($"File already exists: {path}");
+                    }
+                }
             }
 
             protected override void Dispose(bool disposing)
             {
-                // flush file to our system
-                if (!_disposed)
+                // Flush to file system if we have write access
+                if (!_disposed && (_access == FileAccess.Write || _access == FileAccess.ReadWrite))
                 {
                     _disposed = true;
-                    using var sr = new StreamReader(this);
-                    Seek(0, SeekOrigin.Begin);
-                    _fileSystem.WriteToFile(_path, sr.ReadToEnd().Replace("\0", ""));
+                    _fileSystem.WriteAllBytes(_path, ToArray());
                 }
+
+                base.Dispose(disposing);
             }
         }
     }