From f572663dba3a6954f687e4fc544b982decc71221 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Mon, 4 Apr 2022 11:50:37 +0200 Subject: [PATCH 001/145] added defaults shell and working-directory to composite --- src/Runner.Worker/ActionManager.cs | 1 + src/Runner.Worker/ActionManifestManager.cs | 9 ++++- .../Handlers/CompositeActionHandler.cs | 39 +++++++++++++++++-- src/Runner.Worker/Handlers/ScriptHandler.cs | 10 +++-- src/Runner.Worker/action_yaml.json | 38 +++++++++++++++++- 5 files changed, 86 insertions(+), 11 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 954d9a691f6..ce2b649bf59 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -1232,6 +1232,7 @@ public sealed class CompositeActionExecutionData : ActionExecutionData public List Steps { get; set; } public Stack PostSteps { get; set; } public MappingToken Outputs { get; set; } + public MappingToken JobDefaults { get; set; } } public abstract class ActionExecutionData diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 3740128443f..dc76fb298fc 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -372,6 +372,7 @@ private ActionExecutionData ConvertRuns( var postEntrypointToken = default(StringToken); var postIfToken = default(StringToken); var steps = default(List); + var defaultsToken = default(MappingToken); foreach (var run in runsMapping) { @@ -422,6 +423,9 @@ private ActionExecutionData ConvertRuns( steps = PipelineTemplateConverter.ConvertToSteps(templateContext, stepsToken); templateContext.Errors.Check(); break; + case "defaults": + defaultsToken = run.Value.AssertMapping("defaults"); + break; default: Trace.Info($"Ignore run property {runsKey}."); break; @@ -451,7 +455,7 @@ private ActionExecutionData ConvertRuns( }; } } - else if (string.Equals(usingToken.Value, "node12", StringComparison.OrdinalIgnoreCase)|| + else if (string.Equals(usingToken.Value, "node12", StringComparison.OrdinalIgnoreCase) || string.Equals(usingToken.Value, "node16", StringComparison.OrdinalIgnoreCase)) { if (string.IsNullOrEmpty(mainToken?.Value)) @@ -486,7 +490,8 @@ private ActionExecutionData ConvertRuns( PostSteps = new Stack(), InitCondition = "always()", CleanupCondition = "always()", - Outputs = outputs + Outputs = outputs, + JobDefaults = defaultsToken, }; } } diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index 425d1989204..66be3140ac0 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.Expressions2; @@ -13,7 +11,6 @@ using GitHub.Runner.Common; using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; -using GitHub.Runner.Worker; using GitHub.Runner.Worker.Expressions; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -86,13 +83,21 @@ public async Task RunAsync(ActionRunStage stage) ExecutionContext.StepTelemetry.HasPreStep = Data.HasPre; ExecutionContext.StepTelemetry.HasPostStep = Data.HasPost; - + ExecutionContext.StepTelemetry.HasRunsStep = hasRunsStep; ExecutionContext.StepTelemetry.HasUsesStep = hasUsesStep; ExecutionContext.StepTelemetry.StepCount = steps.Count; } ExecutionContext.StepTelemetry.Type = "composite"; + // save job defaults run to be restored after the composite run + IDictionary jobDefaultsRun = new Dictionary(); + if (ExecutionContext.Global.JobDefaults.ContainsKey("run")) + { + jobDefaultsRun = ExecutionContext.Global.JobDefaults["run"]; + } + + var isJobDefaultsRunChanged = false; try { // Inputs of the composite step @@ -163,6 +168,25 @@ public async Task RunAsync(ActionRunStage stage) embeddedSteps.Add(step); } + if (Data.JobDefaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase))) + { + isJobDefaultsRunChanged = true; + + ExecutionContext.Global.JobDefaults["run"] = new Dictionary(StringComparer.OrdinalIgnoreCase); + var defaultsRun = Data.JobDefaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)); + + var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); + var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsRun.Value, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions); + + foreach (var pair in jobDefaults) + { + if (!string.IsNullOrEmpty(pair.Value)) + { + ExecutionContext.Global.JobDefaults["run"][pair.Key] = pair.Value; + } + } + } + // Run embedded steps await RunStepsAsync(embeddedSteps, stage); @@ -178,6 +202,13 @@ public async Task RunAsync(ActionRunStage stage) ExecutionContext.Error(ex); ExecutionContext.Result = TaskResult.Failed; } + finally + { + if (isJobDefaultsRunChanged) + { + ExecutionContext.Global.JobDefaults["run"] = jobDefaultsRun; + } + } } private void ProcessOutputs() diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index 598a28a2f1a..d3cd55b0f66 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -70,7 +70,7 @@ protected override void PrintActionDetails(ActionRunStage stage) { // TODO: figure out how defaults interact with template later // for now, we won't check job.defaults if we are inside a template. - if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) + if (ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) { runDefaults.TryGetValue("shell", out shell); } @@ -153,7 +153,7 @@ public async Task RunAsync(ActionRunStage stage) string workingDirectory = null; if (!Inputs.TryGetValue("workingDirectory", out workingDirectory)) { - if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) + if (ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) { if (runDefaults.TryGetValue("working-directory", out workingDirectory)) { @@ -162,12 +162,16 @@ public async Task RunAsync(ActionRunStage stage) } } var workspaceDir = githubContext["workspace"] as StringContextData; + if (githubContext.ContainsKey("action_path")) + { + workspaceDir = githubContext["action_path"] as StringContextData; + } workingDirectory = Path.Combine(workspaceDir, workingDirectory ?? string.Empty); string shell = null; if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) { - if (string.IsNullOrEmpty(ExecutionContext.ScopeName) && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) + if (ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) { if (runDefaults.TryGetValue("shell", out shell)) { diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index b01d913482f..579feaba4b9 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -103,7 +103,11 @@ "mapping": { "properties": { "using": "non-empty-string", - "steps": "composite-steps" + "steps": "composite-steps", + "defaults": { + "type": "job-defaults", + "required": false + } } } }, @@ -132,7 +136,7 @@ "working-directory": "string-steps-context", "shell": { "type": "non-empty-string", - "required": true + "required": false } } } @@ -252,6 +256,36 @@ "loose-key-type": "non-empty-string", "loose-value-type": "string" } + }, + + "job-defaults": { + "mapping": { + "properties": { + "run": "job-defaults-run" + } + } + }, + + "job-defaults-run": { + "context": [ + "github", + "strategy", + "matrix", + "needs", + "env" + ], + "mapping": { + "properties": { + "shell": { + "type": "non-empty-string", + "required": false + }, + "working-directory": { + "type": "non-empty-string", + "required": false + } + } + } } } } \ No newline at end of file From af1d9c06c02aba9b5e617ef84f8c316ecd1e6943 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Mon, 4 Apr 2022 11:56:28 +0200 Subject: [PATCH 002/145] removed extra spaces in action_yaml --- src/Runner.Worker/action_yaml.json | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 579feaba4b9..89c33ec8e00 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -257,7 +257,6 @@ "loose-value-type": "string" } }, - "job-defaults": { "mapping": { "properties": { @@ -265,7 +264,6 @@ } } }, - "job-defaults-run": { "context": [ "github", From e87a3113024396c340bf9dafa9d343720dc379ca Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Thu, 28 Apr 2022 13:26:27 +0200 Subject: [PATCH 003/145] added check if default is set or shell is set --- src/Runner.Worker/ActionManifestManager.cs | 27 ++++++++++++++++--- src/Runner.Worker/ActionRunner.cs | 1 - .../Handlers/CompositeActionHandler.cs | 1 - 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 928b706f922..907fae6890e 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -10,9 +10,6 @@ using GitHub.DistributedTask.ObjectTemplating; using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.Pipelines.ContextData; -using YamlDotNet.Core; -using YamlDotNet.Core.Events; -using System.Globalization; using System.Linq; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -483,9 +480,31 @@ private ActionExecutionData ConvertRuns( } else { + var compositeActionsExecutionDataSteps = steps.Cast().ToList(); + if (defaultsToken == null) + { + foreach (var step in compositeActionsExecutionDataSteps) + { + var mapping = step.Inputs.AssertMapping("inputs"); + var foundShell = false; + foreach (var pair in mapping) + { + // Expression key + if (pair.Key is StringToken && pair.Key.AssertString("inputs key").Value.Equals("shell")) + { + foundShell = true; + break; + } + } + if (!foundShell) + { + throw new TemplateValidationException("Composite action must set the default shell, or the shell parameter in each step is required."); + } + } + } return new CompositeActionExecutionData() { - Steps = steps.Cast().ToList(), + Steps = compositeActionsExecutionDataSteps, PreSteps = new List(), PostSteps = new Stack(), InitCondition = "always()", diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 18b266e2c90..b997112d266 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Linq; using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating; using GitHub.DistributedTask.ObjectTemplating.Tokens; diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index f68da8ee7d7..6bc7658b999 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -167,7 +167,6 @@ public async Task RunAsync(ActionRunStage stage) if (Data.CompositeDefaults != null && Data.CompositeDefaults.Any(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase))) { - var defaultsToken = Data.CompositeDefaults.First(x => string.Equals(x.Key.AssertString("defaults key").Value, "run", StringComparison.OrdinalIgnoreCase)); var templateEvaluator = ExecutionContext.ToPipelineTemplateEvaluator(); var jobDefaults = templateEvaluator.EvaluateJobDefaultsRun(defaultsToken.Value, ExecutionContext.ExpressionValues, ExecutionContext.ExpressionFunctions); From 238565ad04df3737689be0bc7e0948f5ff4ff94b Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Thu, 28 Apr 2022 15:16:58 +0200 Subject: [PATCH 004/145] added missing check on shell only if action source type is script --- src/Runner.Worker/ActionManifestManager.cs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 907fae6890e..6127153bf7a 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -485,6 +485,10 @@ private ActionExecutionData ConvertRuns( { foreach (var step in compositeActionsExecutionDataSteps) { + if (step.Reference.Type != Pipelines.ActionSourceType.Script) + { + continue; + } var mapping = step.Inputs.AssertMapping("inputs"); var foundShell = false; foreach (var pair in mapping) From a79599070a41e1eb54701174c1abcf4487902d8f Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Tue, 16 Aug 2022 17:17:42 +0200 Subject: [PATCH 005/145] Improved error logs for missing 'using' configuration in metadata file (#2052) Co-authored-by: Octavia Stancu --- src/Runner.Worker/ActionManifestManager.cs | 2 +- src/Test/L0/Worker/ActionManifestManagerL0.cs | 25 +++++++++++++++++++ .../composite_action_without_using_token.yml | 9 +++++++ 3 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 src/Test/TestData/composite_action_without_using_token.yml diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 6127153bf7a..e446ea34c25 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -531,7 +531,7 @@ private ActionExecutionData ConvertRuns( }; } - throw new NotSupportedException(nameof(ConvertRuns)); + throw new NotSupportedException("Missing 'using' value. 'using' requires 'composite', 'docker', 'node12' or 'node16'."); } private void ConvertInputs( diff --git a/src/Test/L0/Worker/ActionManifestManagerL0.cs b/src/Test/L0/Worker/ActionManifestManagerL0.cs index d11f3c1e85c..b6e3f855b34 100644 --- a/src/Test/L0/Worker/ActionManifestManagerL0.cs +++ b/src/Test/L0/Worker/ActionManifestManagerL0.cs @@ -698,6 +698,31 @@ public void Load_ConditionalCompositeAction() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void Load_CompositeActionNoUsing() + { + try + { + //Arrange + Setup(); + + var actionManifest = new ActionManifestManager(); + actionManifest.Initialize(_hc); + var action_path = Path.Combine(TestUtil.GetTestDataPath(), "composite_action_without_using_token.yml"); + + //Assert + var err = Assert.Throws(() => actionManifest.Load(_ec.Object, action_path)); + Assert.Contains($"Fail to load {action_path}", err.Message); + _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Missing 'using' value. 'using' requires 'composite', 'docker', 'node12' or 'node16'.")), It.IsAny()), Times.Once); + } + finally + { + Teardown(); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] diff --git a/src/Test/TestData/composite_action_without_using_token.yml b/src/Test/TestData/composite_action_without_using_token.yml new file mode 100644 index 00000000000..6f8a4b5fe22 --- /dev/null +++ b/src/Test/TestData/composite_action_without_using_token.yml @@ -0,0 +1,9 @@ +name: "composite action" +description: "test composite action without value for the 'using' token in 'runs'" + +runs: + steps: + - id: mystep + shell: bash + run: | + echo "hello world" From 703a54ffb788c74da0e64771255fb4d59b97c416 Mon Sep 17 00:00:00 2001 From: Nicholas Bergesen Date: Tue, 23 Aug 2022 09:20:58 +0900 Subject: [PATCH 006/145] Display full job name and nested workflow details in log (#2049) --- src/Runner.Worker/JobExtension.cs | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 030cc84ed0a..09bbdd479d2 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -316,6 +316,29 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } + if (message.Variables.TryGetValue("system.workflowFileFullPath", out VariableValue workflowFileFullPath)) + { + context.Output($"Uses: {workflowFileFullPath.Value}"); + if (message.ContextData.TryGetValue("inputs", out var pipelineContextData)) + { + var inputs = pipelineContextData.AssertDictionary("inputs"); + if (inputs.Any()) + { + context.Output($"##[group] Inputs"); + foreach (var input in inputs) + { + context.Output($" {input.Key}: {input.Value}"); + } + context.Output("##[endgroup]"); + } + } + + if (!string.IsNullOrWhiteSpace(message.JobDisplayName)) + { + context.Output($"Complete job name: {message.JobDisplayName}"); + } + } + var intraActionStates = new Dictionary>(); foreach (var preStep in prepareResult.PreStepTracker) { From 5731c0ea5d71feaaa8ddef5ad223ed3e0627556b Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 22 Aug 2022 18:26:52 -0700 Subject: [PATCH 007/145] Include step context name and start/finish time in step telemetry (#2069) * Include step context name in telemetry. * . --- src/Runner.Worker/ExecutionContext.cs | 3 +++ src/Sdk/DTWebApi/WebApi/ActionsStepTelemetry.cs | 9 +++++++++ 2 files changed, 12 insertions(+) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index d46aefb65f4..79e0335fda9 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -369,6 +369,7 @@ public IExecutionContext CreateChild( child.StepTelemetry.StepId = recordId; child.StepTelemetry.Stage = stage.ToString(); child.StepTelemetry.IsEmbedded = isEmbedded; + child.StepTelemetry.StepContextName = child.GetFullyQualifiedContextName(); ; return child; } @@ -959,6 +960,8 @@ public void PublishStepTelemetry() _record.StartTime != null) { StepTelemetry.ExecutionTimeInSeconds = (int)Math.Ceiling((_record.FinishTime - _record.StartTime)?.TotalSeconds ?? 0); + StepTelemetry.StartTime = _record.StartTime; + StepTelemetry.FinishTime = _record.FinishTime; } if (!IsEmbedded && diff --git a/src/Sdk/DTWebApi/WebApi/ActionsStepTelemetry.cs b/src/Sdk/DTWebApi/WebApi/ActionsStepTelemetry.cs index 71384013922..e6ee7f6472a 100644 --- a/src/Sdk/DTWebApi/WebApi/ActionsStepTelemetry.cs +++ b/src/Sdk/DTWebApi/WebApi/ActionsStepTelemetry.cs @@ -30,6 +30,9 @@ public ActionsStepTelemetry() [DataMember(EmitDefaultValue = false)] public Guid StepId { get; set; } + [DataMember(EmitDefaultValue = false)] + public string StepContextName { get; set; } + [DataMember(EmitDefaultValue = false)] public bool? HasRunsStep { get; set; } @@ -57,6 +60,12 @@ public ActionsStepTelemetry() [DataMember(EmitDefaultValue = false)] public int? ExecutionTimeInSeconds { get; set; } + [DataMember(EmitDefaultValue = false)] + public DateTime? StartTime { get; set; } + + [DataMember(EmitDefaultValue = false)] + public DateTime? FinishTime { get; set; } + [DataMember(EmitDefaultValue = false)] public string ContainerHookData { get; set; } } From a2fb048be75f798b0b6cae30a9d882338c6e2c8b Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Tue, 23 Aug 2022 07:42:29 -0700 Subject: [PATCH 008/145] Escaping key and quoting it to avoid key based command injection (#2062) * escaping key and quoting it to avoid key based command injection * extracted creation of flags to DockerUtil, with testing included --- .../Container/DockerCommandManager.cs | 6 +-- src/Runner.Worker/Container/DockerUtil.cs | 25 ++++++++- src/Runner.Worker/Handlers/StepHost.cs | 4 +- src/Test/L0/Container/DockerUtilL0.cs | 54 +++++++++++++++++++ 4 files changed, 83 insertions(+), 6 deletions(-) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index 86cf0eeedf2..a0c158bdf68 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -131,11 +131,11 @@ public async Task DockerCreate(IExecutionContext context, ContainerInfo { if (String.IsNullOrEmpty(env.Value)) { - dockerOptions.Add($"-e \"{env.Key}\""); + dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key)); } else { - dockerOptions.Add($"-e \"{env.Key}={env.Value.Replace("\"", "\\\"")}\""); + dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key, env.Value)); } } @@ -202,7 +202,7 @@ public async Task DockerRun(IExecutionContext context, ContainerInfo contai { // e.g. -e MY_SECRET maps the value into the exec'ed process without exposing // the value directly in the command - dockerOptions.Add($"-e {env.Key}"); + dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key)); } // Watermark for GitHub Action environment diff --git a/src/Runner.Worker/Container/DockerUtil.cs b/src/Runner.Worker/Container/DockerUtil.cs index 02c2ece5b79..bde59f5b641 100644 --- a/src/Runner.Worker/Container/DockerUtil.cs +++ b/src/Runner.Worker/Container/DockerUtil.cs @@ -17,7 +17,7 @@ public static List ParseDockerPort(IList portMappingLines) string pattern = $"^(?<{targetPort}>\\d+)/(?<{proto}>\\w+) -> (?<{host}>.+):(?<{hostPort}>\\d+)$"; List portMappings = new List(); - foreach(var line in portMappingLines) + foreach (var line in portMappingLines) { Match m = Regex.Match(line, pattern, RegexOptions.None, TimeSpan.FromSeconds(1)); if (m.Success) @@ -61,5 +61,28 @@ public static string ParseRegistryHostnameFromImageName(string name) } return ""; } + + public static string CreateEscapedOption(string flag, string key) + { + if (String.IsNullOrEmpty(key)) + { + return ""; + } + return $"{flag} \"{EscapeString(key)}\""; + } + + public static string CreateEscapedOption(string flag, string key, string value) + { + if (String.IsNullOrEmpty(key)) + { + return ""; + } + return $"{flag} \"{EscapeString(key)}={EscapeString(value)}\""; + } + + private static string EscapeString(string value) + { + return value.Replace("\\", "\\\\").Replace("\"", "\\\""); + } } } diff --git a/src/Runner.Worker/Handlers/StepHost.cs b/src/Runner.Worker/Handlers/StepHost.cs index 95feaca9aef..1270dd90e6f 100644 --- a/src/Runner.Worker/Handlers/StepHost.cs +++ b/src/Runner.Worker/Handlers/StepHost.cs @@ -193,7 +193,7 @@ public async Task ExecuteAsync(IExecutionContext context, TranslateToContainerPath(environment); await containerHookManager.RunScriptStepAsync(context, Container, - workingDirectory, + workingDirectory, fileName, arguments, environment, @@ -216,7 +216,7 @@ await containerHookManager.RunScriptStepAsync(context, { // e.g. -e MY_SECRET maps the value into the exec'ed process without exposing // the value directly in the command - dockerCommandArgs.Add($"-e {env.Key}"); + dockerCommandArgs.Add(DockerUtil.CreateEscapedOption("-e", env.Key)); } if (!string.IsNullOrEmpty(PrependPath)) { diff --git a/src/Test/L0/Container/DockerUtilL0.cs b/src/Test/L0/Container/DockerUtilL0.cs index c069255a85c..e5ae05fda72 100644 --- a/src/Test/L0/Container/DockerUtilL0.cs +++ b/src/Test/L0/Container/DockerUtilL0.cs @@ -144,5 +144,59 @@ public void ParseRegistryHostnameFromImageName(string input, string expected) var actual = DockerUtil.ParseRegistryHostnameFromImageName(input); Assert.Equal(expected, actual); } + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + [InlineData("", "")] + [InlineData("HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #")] + [InlineData("HOME \"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\\\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \"\"alpine:3.8 sh -c id #", "HOME \\\"\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \\\"\"alpine:3.8 sh -c id #", "HOME \\\\\\\"\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \"\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\\\\\"alpine:3.8 sh -c id #")] + public void CreateEscapedOption_keyOnly(string input, string escaped) + { + var flag = "--example"; + var actual = DockerUtil.CreateEscapedOption(flag, input); + string expected; + if (String.IsNullOrEmpty(input)) + { + expected = ""; + } + else + { + expected = $"{flag} \"{escaped}\""; + } + Assert.Equal(expected, actual); + } + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + [InlineData("HOME", "", "HOME", "")] + [InlineData("HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #")] + [InlineData("HOME \"alpine:3.8 sh -c id #", "HOME \"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \\\"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \\\\\"alpine:3.8 sh -c id #", "HOME \\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\\\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \"\"alpine:3.8 sh -c id #", "HOME \"\"alpine:3.8 sh -c id #", "HOME \\\"\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \\\"\"alpine:3.8 sh -c id #", "HOME \\\"\"alpine:3.8 sh -c id #", "HOME \\\\\\\"\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"\\\"alpine:3.8 sh -c id #")] + [InlineData("HOME \"\\\"alpine:3.8 sh -c id #", "HOME \"\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\\\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\\\\\"alpine:3.8 sh -c id #")] + public void CreateEscapedOption_keyValue(string keyInput, string valueInput, string escapedKey, string escapedValue) + { + var flag = "--example"; + var actual = DockerUtil.CreateEscapedOption(flag, keyInput, valueInput); + string expected; + if (String.IsNullOrEmpty(keyInput)) + { + expected = ""; + } + else + { + expected = $"{flag} \"{escapedKey}={escapedValue}\""; + } + Assert.Equal(expected, actual); + } } } From 2363d32faf444f66381baf325b5d8525d1a47e5e Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Tue, 23 Aug 2022 16:42:40 +0200 Subject: [PATCH 009/145] Release notes for 2.296.0 (#2078) * Update releaseNote.md * Update runnerversion --- releaseNote.md | 12 +++++------- src/runnerversion | 2 +- 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index ae05e6dc92f..91255a46fae 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,11 +1,9 @@ -## Features -- GHES: Support connecting to GitHub Enterprise Server Actions Service on a subdomain ## Bugs -- Fixed a bug where GITHUB_ENV would not update correctly between composite action steps (#1794) -- Fixed runner update bug caused by `update.sh|cmd` running too long (#2044) +- Avoid key based command injection via Docker command arguments (#2062) ## Misc -- Bump Newtonsoft.Json from 11.0.2 to 13.0.1 (#2012) -- Change a periodic token expiry log message level from `WARNING` to `VERBOSE` (#2021) +- Added step context name and start/finish time in step telemetry (#2069) +- Improved error logs when there is a missing 'using' token configuration in the metadata file (#2052) +- Added full job name and nested workflow details in log (#2049) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. @@ -32,7 +30,7 @@ curl -O -L https://github.com/actions/runner/releases/download/v tar xzf ./actions-runner-osx-x64-.tar.gz ``` -## [Pre-release] OSX arm64 (Apple silicon) +## OSX arm64 (Apple silicon) ``` bash # Create a folder diff --git a/src/runnerversion b/src/runnerversion index ea0fe6c4e2c..d1ad399cbf4 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.295.0 +2.296.0 From a47f49c4c0000f8ba5e457f148e12380a4a7450f Mon Sep 17 00:00:00 2001 From: Konrad Pabjan Date: Wed, 24 Aug 2022 16:02:51 -0400 Subject: [PATCH 010/145] Validate lines and columns for Annotations (#2082) --- src/Runner.Worker/ActionCommandManager.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/Runner.Worker/ActionCommandManager.cs b/src/Runner.Worker/ActionCommandManager.cs index 0bcde5f5354..a705d67ed40 100644 --- a/src/Runner.Worker/ActionCommandManager.cs +++ b/src/Runner.Worker/ActionCommandManager.cs @@ -585,6 +585,8 @@ public abstract class IssueCommandExtension : RunnerService, IActionCommandExten public void ProcessCommand(IExecutionContext context, string inputLine, ActionCommand command, ContainerInfo container) { + ValidateLinesAndColumns(command, context); + command.Properties.TryGetValue(IssueCommandProperties.File, out string file); command.Properties.TryGetValue(IssueCommandProperties.Line, out string line); command.Properties.TryGetValue(IssueCommandProperties.Column, out string column); From 7c7cd7fde5feee0ea953ffbf0b6335e536caf344 Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Wed, 31 Aug 2022 10:39:58 -0700 Subject: [PATCH 011/145] docker: escape key-value pair as -e KEY and VALUE being environment var (#2091) * docker: escape key-value pair as -e KEY and VALUE being environment var * removed code duplication, removed unused method and test --- .../Container/DockerCommandManager.cs | 13 ++++++--- src/Runner.Worker/Container/DockerUtil.cs | 9 ------- src/Test/L0/Container/DockerUtilL0.cs | 27 ------------------- 3 files changed, 10 insertions(+), 39 deletions(-) diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index a0c158bdf68..4d4940c9752 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -107,6 +107,7 @@ public async Task DockerBuild(IExecutionContext context, string workingDire public async Task DockerCreate(IExecutionContext context, ContainerInfo container) { IList dockerOptions = new List(); + IDictionary environment = new Dictionary(); // OPTIONS dockerOptions.Add($"--name {container.ContainerDisplayName}"); dockerOptions.Add($"--label {DockerInstanceLabel}"); @@ -135,7 +136,8 @@ public async Task DockerCreate(IExecutionContext context, ContainerInfo } else { - dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key, env.Value)); + environment.Add(env.Key, env.Value); + dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key)); } } @@ -183,7 +185,7 @@ public async Task DockerCreate(IExecutionContext context, ContainerInfo dockerOptions.Add($"{container.ContainerEntryPointArgs}"); var optionsString = string.Join(" ", dockerOptions); - List outputStrings = await ExecuteDockerCommandAsync(context, "create", optionsString); + List outputStrings = await ExecuteDockerCommandAsync(context, "create", optionsString, environment); return outputStrings.FirstOrDefault(); } @@ -443,6 +445,11 @@ public Task DockerLogin(IExecutionContext context, string configFileDirecto } private async Task> ExecuteDockerCommandAsync(IExecutionContext context, string command, string options) + { + return await ExecuteDockerCommandAsync(context, command, options, null); + } + + private async Task> ExecuteDockerCommandAsync(IExecutionContext context, string command, string options, IDictionary environment) { string arg = $"{command} {options}".Trim(); context.Command($"{DockerPath} {arg}"); @@ -470,7 +477,7 @@ await processInvoker.ExecuteAsync( workingDirectory: context.GetGitHubContext("workspace"), fileName: DockerPath, arguments: arg, - environment: null, + environment: environment, requireExitCodeZero: true, outputEncoding: null, cancellationToken: CancellationToken.None); diff --git a/src/Runner.Worker/Container/DockerUtil.cs b/src/Runner.Worker/Container/DockerUtil.cs index bde59f5b641..e1ae6d3eeb4 100644 --- a/src/Runner.Worker/Container/DockerUtil.cs +++ b/src/Runner.Worker/Container/DockerUtil.cs @@ -71,15 +71,6 @@ public static string CreateEscapedOption(string flag, string key) return $"{flag} \"{EscapeString(key)}\""; } - public static string CreateEscapedOption(string flag, string key, string value) - { - if (String.IsNullOrEmpty(key)) - { - return ""; - } - return $"{flag} \"{EscapeString(key)}={EscapeString(value)}\""; - } - private static string EscapeString(string value) { return value.Replace("\\", "\\\\").Replace("\"", "\\\""); diff --git a/src/Test/L0/Container/DockerUtilL0.cs b/src/Test/L0/Container/DockerUtilL0.cs index e5ae05fda72..843aaf5efd1 100644 --- a/src/Test/L0/Container/DockerUtilL0.cs +++ b/src/Test/L0/Container/DockerUtilL0.cs @@ -171,32 +171,5 @@ public void CreateEscapedOption_keyOnly(string input, string escaped) } Assert.Equal(expected, actual); } - - [Theory] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - [InlineData("HOME", "", "HOME", "")] - [InlineData("HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #")] - [InlineData("HOME \"alpine:3.8 sh -c id #", "HOME \"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \\\"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \\\\\"alpine:3.8 sh -c id #", "HOME \\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\\\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \"\"alpine:3.8 sh -c id #", "HOME \"\"alpine:3.8 sh -c id #", "HOME \\\"\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \\\"\"alpine:3.8 sh -c id #", "HOME \\\"\"alpine:3.8 sh -c id #", "HOME \\\\\\\"\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \"\\\"alpine:3.8 sh -c id #", "HOME \"\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\\\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\\\\\"alpine:3.8 sh -c id #")] - public void CreateEscapedOption_keyValue(string keyInput, string valueInput, string escapedKey, string escapedValue) - { - var flag = "--example"; - var actual = DockerUtil.CreateEscapedOption(flag, keyInput, valueInput); - string expected; - if (String.IsNullOrEmpty(keyInput)) - { - expected = ""; - } - else - { - expected = $"{flag} \"{escapedKey}={escapedValue}\""; - } - Assert.Equal(expected, actual); - } } } From a815fa953875624f6be94d4bcdd322c25139cae1 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Fri, 2 Sep 2022 17:43:22 +0200 Subject: [PATCH 012/145] 2.296.1 Release (#2092) (#2099) * docker: escape key-value pair as -e KEY and VALUE being environment var * removed code duplication, removed unused method and test * add release notes Co-authored-by: Nikola Jokic Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> Co-authored-by: Nikola Jokic --- releaseNote.md | 5 +---- src/runnerversion | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 91255a46fae..a6cb9c027f8 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,9 +1,6 @@ ## Bugs -- Avoid key based command injection via Docker command arguments (#2062) +- Fixed an issue where job and service container envs were corrupted (#2091) ## Misc -- Added step context name and start/finish time in step telemetry (#2069) -- Improved error logs when there is a missing 'using' token configuration in the metadata file (#2052) -- Added full job name and nested workflow details in log (#2049) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index d1ad399cbf4..f7b71d4819b 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.296.0 +2.296.1 From a419d6cd47835297d1bfdb1c0f9f758e0d152b8e Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Tue, 6 Sep 2022 07:45:00 -0700 Subject: [PATCH 013/145] fix ACTIONS_RUNNER_CONTAINER_HOOKS name in ADR (#2098) --- docs/adrs/1891-container-hooks.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/adrs/1891-container-hooks.md b/docs/adrs/1891-container-hooks.md index 1ab14d85fcd..5c905e40588 100644 --- a/docs/adrs/1891-container-hooks.md +++ b/docs/adrs/1891-container-hooks.md @@ -16,7 +16,7 @@ We should give them that option, and publish examples how how they can create th - For example, the current runner overrides `HOME`, we can do that in the hook, but we shouldn't pass that hook as an ENV with the other env's the user has set, as that is not user input, it is how the runner invokes containers ## Interface -- You will set the variable `ACTIONS_RUNNER_CONTAINER_HOOK=/Users/foo/runner/hooks.js` which is the entrypoint to your hook handler. +- You will set the variable `ACTIONS_RUNNER_CONTAINER_HOOKS=/Users/foo/runner/hooks.js` which is the entrypoint to your hook handler. - There is no partial opt in, you must handle every hook - We will pass a command and some args via `stdin` - An exit code of 0 is a success, every other exit code is a failure From 43853e5c97e1f125e362561e5e6f44cc2f70e157 Mon Sep 17 00:00:00 2001 From: Thomas Boop <52323235+thboop@users.noreply.github.com> Date: Fri, 9 Sep 2022 14:32:07 -0400 Subject: [PATCH 014/145] Port hotfix to main branch (#2108) * fix issue with env's overwriting environment * add release notes * handle value escaping * compile regex for runtime perf improvements --- releaseNote.md | 2 +- .../Container/DockerCommandManager.cs | 13 ++----- src/Runner.Worker/Container/DockerUtil.cs | 32 +++++++++++++++-- src/Test/L0/Container/DockerUtilL0.cs | 36 +++++++++++++++---- src/runnerversion | 2 +- 5 files changed, 64 insertions(+), 21 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index a6cb9c027f8..b70766e7098 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,5 +1,5 @@ ## Bugs -- Fixed an issue where job and service container envs were corrupted (#2091) +- Fixed an issue where self hosted environments had their docker env's overwritten (#2107) ## Misc ## Windows x64 diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index 4d4940c9752..a0c158bdf68 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -107,7 +107,6 @@ public async Task DockerBuild(IExecutionContext context, string workingDire public async Task DockerCreate(IExecutionContext context, ContainerInfo container) { IList dockerOptions = new List(); - IDictionary environment = new Dictionary(); // OPTIONS dockerOptions.Add($"--name {container.ContainerDisplayName}"); dockerOptions.Add($"--label {DockerInstanceLabel}"); @@ -136,8 +135,7 @@ public async Task DockerCreate(IExecutionContext context, ContainerInfo } else { - environment.Add(env.Key, env.Value); - dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key)); + dockerOptions.Add(DockerUtil.CreateEscapedOption("-e", env.Key, env.Value)); } } @@ -185,7 +183,7 @@ public async Task DockerCreate(IExecutionContext context, ContainerInfo dockerOptions.Add($"{container.ContainerEntryPointArgs}"); var optionsString = string.Join(" ", dockerOptions); - List outputStrings = await ExecuteDockerCommandAsync(context, "create", optionsString, environment); + List outputStrings = await ExecuteDockerCommandAsync(context, "create", optionsString); return outputStrings.FirstOrDefault(); } @@ -445,11 +443,6 @@ public Task DockerLogin(IExecutionContext context, string configFileDirecto } private async Task> ExecuteDockerCommandAsync(IExecutionContext context, string command, string options) - { - return await ExecuteDockerCommandAsync(context, command, options, null); - } - - private async Task> ExecuteDockerCommandAsync(IExecutionContext context, string command, string options, IDictionary environment) { string arg = $"{command} {options}".Trim(); context.Command($"{DockerPath} {arg}"); @@ -477,7 +470,7 @@ await processInvoker.ExecuteAsync( workingDirectory: context.GetGitHubContext("workspace"), fileName: DockerPath, arguments: arg, - environment: environment, + environment: null, requireExitCodeZero: true, outputEncoding: null, cancellationToken: CancellationToken.None); diff --git a/src/Runner.Worker/Container/DockerUtil.cs b/src/Runner.Worker/Container/DockerUtil.cs index e1ae6d3eeb4..97a53759bc7 100644 --- a/src/Runner.Worker/Container/DockerUtil.cs +++ b/src/Runner.Worker/Container/DockerUtil.cs @@ -6,6 +6,9 @@ namespace GitHub.Runner.Worker.Container { public class DockerUtil { + private static readonly Regex QuoteEscape = new Regex(@"(\\*)" + "\"", RegexOptions.Compiled); + private static readonly Regex EndOfStringEscape = new Regex(@"(\\+)$", RegexOptions.Compiled); + public static List ParseDockerPort(IList portMappingLines) { const string targetPort = "targetPort"; @@ -68,12 +71,37 @@ public static string CreateEscapedOption(string flag, string key) { return ""; } - return $"{flag} \"{EscapeString(key)}\""; + return $"{flag} {EscapeString(key)}"; + } + + public static string CreateEscapedOption(string flag, string key, string value) + { + if (String.IsNullOrEmpty(key)) + { + return ""; + } + var escapedString = EscapeString($"{key}={value}"); + return $"{flag} {escapedString}"; } private static string EscapeString(string value) { - return value.Replace("\\", "\\\\").Replace("\"", "\\\""); + if (String.IsNullOrEmpty(value)) + { + return ""; + } + // Dotnet escaping rules are weird here, we can only escape \ if it precedes a " + // If a double quotation mark follows two or an even number of backslashes, each proceeding backslash pair is replaced with one backslash and the double quotation mark is removed. + // If a double quotation mark follows an odd number of backslashes, including just one, each preceding pair is replaced with one backslash and the remaining backslash is removed; however, in this case the double quotation mark is not removed. + // https://docs.microsoft.com/en-us/dotnet/api/system.environment.getcommandlineargs?redirectedfrom=MSDN&view=net-6.0#remarks + + // First, find any \ followed by a " and double the number of \ + 1. + value = QuoteEscape.Replace(value, @"$1$1\" + "\""); + // Next, what if it ends in `\`, it would escape the end quote. So, we need to detect that at the end of the string and perform the same escape + // Luckily, we can just use the $ character with detects the end of string in regex + value = EndOfStringEscape.Replace(value, @"$1$1"); + // Finally, wrap it in quotes + return $"\"{value}\""; } } } diff --git a/src/Test/L0/Container/DockerUtilL0.cs b/src/Test/L0/Container/DockerUtilL0.cs index 843aaf5efd1..1e45d598e89 100644 --- a/src/Test/L0/Container/DockerUtilL0.cs +++ b/src/Test/L0/Container/DockerUtilL0.cs @@ -149,13 +149,12 @@ public void ParseRegistryHostnameFromImageName(string input, string expected) [Trait("Level", "L0")] [Trait("Category", "Worker")] [InlineData("", "")] - [InlineData("HOME alpine:3.8 sh -c id #", "HOME alpine:3.8 sh -c id #")] - [InlineData("HOME \"alpine:3.8 sh -c id #", "HOME \\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \\\"alpine:3.8 sh -c id #", "HOME \\\\\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \\\\\"alpine:3.8 sh -c id #", "HOME \\\\\\\\\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \"\"alpine:3.8 sh -c id #", "HOME \\\"\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \\\"\"alpine:3.8 sh -c id #", "HOME \\\\\\\"\\\"alpine:3.8 sh -c id #")] - [InlineData("HOME \"\\\"alpine:3.8 sh -c id #", "HOME \\\"\\\\\\\"alpine:3.8 sh -c id #")] + [InlineData("foo", "foo")] + [InlineData("foo \\ bar", "foo \\ bar")] + [InlineData("foo \\", "foo \\\\")] + [InlineData("foo \\\\", "foo \\\\\\\\")] + [InlineData("foo \\\" bar", "foo \\\\\\\" bar")] + [InlineData("foo \\\\\" bar", "foo \\\\\\\\\\\" bar")] public void CreateEscapedOption_keyOnly(string input, string escaped) { var flag = "--example"; @@ -171,5 +170,28 @@ public void CreateEscapedOption_keyOnly(string input, string escaped) } Assert.Equal(expected, actual); } + + [Theory] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + [InlineData("foo", "bar", "foo=bar")] + [InlineData("foo\\", "bar", "foo\\=bar")] + [InlineData("foo\\", "bar\\", "foo\\=bar\\\\")] + [InlineData("foo \\","bar \\", "foo \\=bar \\\\")] + public void CreateEscapedOption_keyValue(string keyInput, string valueInput, string escapedString) + { + var flag = "--example"; + var actual = DockerUtil.CreateEscapedOption(flag, keyInput, valueInput); + string expected; + if (String.IsNullOrEmpty(keyInput)) + { + expected = ""; + } + else + { + expected = $"{flag} \"{escapedString}\""; + } + Assert.Equal(expected, actual); + } } } diff --git a/src/runnerversion b/src/runnerversion index f7b71d4819b..ffb30427c2f 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.296.1 +2.296.2 From 69f84e99251fac57454b9aeb8c0f0daf39187380 Mon Sep 17 00:00:00 2001 From: Stefan Ruvceski <96768603+ruvceskistefan@users.noreply.github.com> Date: Mon, 12 Sep 2022 20:51:36 +0200 Subject: [PATCH 015/145] fix for issue #2009 - composite summary file (#2077) --- src/Runner.Worker/FileCommandManager.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index a55050de13b..145291ccd4f 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -68,7 +68,7 @@ public void ProcessFiles(IExecutionContext context, ContainerInfo container) { try { - fileCommand.ProcessCommand(context, Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix),container); + fileCommand.ProcessCommand(context, Path.Combine(_fileCommandDirectory, fileCommand.FilePrefix + _fileSuffix), container); } catch (Exception ex) { @@ -118,7 +118,7 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container if (File.Exists(filePath)) { var lines = File.ReadAllLines(filePath, Encoding.UTF8); - foreach(var line in lines) + foreach (var line in lines) { if (line == string.Empty) { @@ -312,7 +312,9 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container } } - var attachmentName = context.Id.ToString(); + var attachmentName = !context.IsEmbedded + ? context.Id.ToString() + : context.EmbeddedId.ToString(); Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); // Attachments must be added to the parent context (job), not the current context (step) From df0ac57c9d8dccfb955af11ae4ff5d3f7b121b4e Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Thu, 15 Sep 2022 10:43:28 +0100 Subject: [PATCH 016/145] Bump @actions/core from 1.2.6 to 1.9.1 in /src/Misc/expressionFunc/hashFiles (#2123) --- src/Misc/expressionFunc/hashFiles/README.md | 5 +- .../hashFiles/package-lock.json | 62 +- src/Misc/layoutbin/hashFiles/index.js | 2618 ++++++++++++++++- 3 files changed, 2526 insertions(+), 159 deletions(-) diff --git a/src/Misc/expressionFunc/hashFiles/README.md b/src/Misc/expressionFunc/hashFiles/README.md index 6283ee8b0d4..c23d01e6efb 100644 --- a/src/Misc/expressionFunc/hashFiles/README.md +++ b/src/Misc/expressionFunc/hashFiles/README.md @@ -1 +1,4 @@ -To update hashFiles under `Misc/layoutbin` run `npm install && npm run all` \ No newline at end of file +To compile this package (output will be stored in `Misc/layoutbin`) run `npm install && npm run all`. + +> Note: this package also needs to be recompiled for dependabot PRs updating one of +> its dependencies. diff --git a/src/Misc/expressionFunc/hashFiles/package-lock.json b/src/Misc/expressionFunc/hashFiles/package-lock.json index 272830a94d7..c584d01b315 100644 --- a/src/Misc/expressionFunc/hashFiles/package-lock.json +++ b/src/Misc/expressionFunc/hashFiles/package-lock.json @@ -22,9 +22,13 @@ } }, "node_modules/@actions/core": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.6.tgz", - "integrity": "sha512-ZQYitnqiyBc3D+k7LsgSBmMDVkOVidaagDG7j3fOym77jNunWRuYx7VSHa9GNfFZh+zh61xsCjRj4JxMZlDqTA==" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", + "dependencies": { + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" + } }, "node_modules/@actions/glob": { "version": "0.1.0", @@ -35,6 +39,14 @@ "minimatch": "^3.0.4" } }, + "node_modules/@actions/http-client": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "dependencies": { + "tunnel": "^0.0.6" + } + }, "node_modules/@eslint/eslintrc": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.2.1.tgz", @@ -2381,6 +2393,14 @@ "typescript": ">=2.8.0 || >= 3.2.0-dev || >= 3.3.0-dev || >= 3.4.0-dev || >= 3.5.0-dev || >= 3.6.0-dev || >= 3.6.0-beta || >= 3.7.0-dev || >= 3.7.0-beta" } }, + "node_modules/tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==", + "engines": { + "node": ">=0.6.11 <=0.7.0 || >=0.7.3" + } + }, "node_modules/type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -2442,6 +2462,14 @@ "punycode": "^2.1.0" } }, + "node_modules/uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==", + "bin": { + "uuid": "dist/bin/uuid" + } + }, "node_modules/v8-compile-cache": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", @@ -2503,9 +2531,13 @@ }, "dependencies": { "@actions/core": { - "version": "1.2.6", - "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.2.6.tgz", - "integrity": "sha512-ZQYitnqiyBc3D+k7LsgSBmMDVkOVidaagDG7j3fOym77jNunWRuYx7VSHa9GNfFZh+zh61xsCjRj4JxMZlDqTA==" + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/@actions/core/-/core-1.9.1.tgz", + "integrity": "sha512-5ad+U2YGrmmiw6du20AQW5XuWo7UKN2052FjSV7MX+Wfjf8sCqcsZe62NfgHys4QI4/Y+vQvLKYL8jWtA1ZBTA==", + "requires": { + "@actions/http-client": "^2.0.1", + "uuid": "^8.3.2" + } }, "@actions/glob": { "version": "0.1.0", @@ -2516,6 +2548,14 @@ "minimatch": "^3.0.4" } }, + "@actions/http-client": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/@actions/http-client/-/http-client-2.0.1.tgz", + "integrity": "sha512-PIXiMVtz6VvyaRsGY268qvj57hXQEpsYogYOu2nrQhlf+XCGmZstmuZBbAybUl1nQGnvS1k1eEsQ69ZoD7xlSw==", + "requires": { + "tunnel": "^0.0.6" + } + }, "@eslint/eslintrc": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/@eslint/eslintrc/-/eslintrc-1.2.1.tgz", @@ -4189,6 +4229,11 @@ "tslib": "^1.8.1" } }, + "tunnel": { + "version": "0.0.6", + "resolved": "https://registry.npmjs.org/tunnel/-/tunnel-0.0.6.tgz", + "integrity": "sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==" + }, "type-check": { "version": "0.4.0", "resolved": "https://registry.npmjs.org/type-check/-/type-check-0.4.0.tgz", @@ -4231,6 +4276,11 @@ "punycode": "^2.1.0" } }, + "uuid": { + "version": "8.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz", + "integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==" + }, "v8-compile-cache": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/v8-compile-cache/-/v8-compile-cache-2.1.0.tgz", diff --git a/src/Misc/layoutbin/hashFiles/index.js b/src/Misc/layoutbin/hashFiles/index.js index 912b3c45be4..e0bf2f96b47 100644 --- a/src/Misc/layoutbin/hashFiles/index.js +++ b/src/Misc/layoutbin/hashFiles/index.js @@ -43,6 +43,199 @@ module.exports = /************************************************************************/ /******/ ({ +/***/ 16: +/***/ (function(module) { + +module.exports = require("tls"); + +/***/ }), + +/***/ 22: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _validate = _interopRequireDefault(__webpack_require__(78)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function parse(uuid) { + if (!(0, _validate.default)(uuid)) { + throw TypeError('Invalid UUID'); + } + + let v; + const arr = new Uint8Array(16); // Parse ########-....-....-....-............ + + arr[0] = (v = parseInt(uuid.slice(0, 8), 16)) >>> 24; + arr[1] = v >>> 16 & 0xff; + arr[2] = v >>> 8 & 0xff; + arr[3] = v & 0xff; // Parse ........-####-....-....-............ + + arr[4] = (v = parseInt(uuid.slice(9, 13), 16)) >>> 8; + arr[5] = v & 0xff; // Parse ........-....-####-....-............ + + arr[6] = (v = parseInt(uuid.slice(14, 18), 16)) >>> 8; + arr[7] = v & 0xff; // Parse ........-....-....-####-............ + + arr[8] = (v = parseInt(uuid.slice(19, 23), 16)) >>> 8; + arr[9] = v & 0xff; // Parse ........-....-....-....-############ + // (Use "/" to avoid 32-bit truncation when bit-shifting high-order bytes) + + arr[10] = (v = parseInt(uuid.slice(24, 36), 16)) / 0x10000000000 & 0xff; + arr[11] = v / 0x100000000 & 0xff; + arr[12] = v >>> 24 & 0xff; + arr[13] = v >>> 16 & 0xff; + arr[14] = v >>> 8 & 0xff; + arr[15] = v & 0xff; + return arr; +} + +var _default = parse; +exports.default = _default; + +/***/ }), + +/***/ 36: +/***/ (function(__unusedmodule, exports) { + +"use strict"; + +Object.defineProperty(exports, "__esModule", { value: true }); +/** + * Indicates whether a pattern matches a path + */ +var MatchKind; +(function (MatchKind) { + /** Not matched */ + MatchKind[MatchKind["None"] = 0] = "None"; + /** Matched if the path is a directory */ + MatchKind[MatchKind["Directory"] = 1] = "Directory"; + /** Matched if the path is a regular file */ + MatchKind[MatchKind["File"] = 2] = "File"; + /** Matched */ + MatchKind[MatchKind["All"] = 3] = "All"; +})(MatchKind = exports.MatchKind || (exports.MatchKind = {})); +//# sourceMappingURL=internal-match-kind.js.map + +/***/ }), + +/***/ 62: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +Object.defineProperty(exports, "v1", { + enumerable: true, + get: function () { + return _v.default; + } +}); +Object.defineProperty(exports, "v3", { + enumerable: true, + get: function () { + return _v2.default; + } +}); +Object.defineProperty(exports, "v4", { + enumerable: true, + get: function () { + return _v3.default; + } +}); +Object.defineProperty(exports, "v5", { + enumerable: true, + get: function () { + return _v4.default; + } +}); +Object.defineProperty(exports, "NIL", { + enumerable: true, + get: function () { + return _nil.default; + } +}); +Object.defineProperty(exports, "version", { + enumerable: true, + get: function () { + return _version.default; + } +}); +Object.defineProperty(exports, "validate", { + enumerable: true, + get: function () { + return _validate.default; + } +}); +Object.defineProperty(exports, "stringify", { + enumerable: true, + get: function () { + return _stringify.default; + } +}); +Object.defineProperty(exports, "parse", { + enumerable: true, + get: function () { + return _parse.default; + } +}); + +var _v = _interopRequireDefault(__webpack_require__(893)); + +var _v2 = _interopRequireDefault(__webpack_require__(209)); + +var _v3 = _interopRequireDefault(__webpack_require__(733)); + +var _v4 = _interopRequireDefault(__webpack_require__(384)); + +var _nil = _interopRequireDefault(__webpack_require__(327)); + +var _version = _interopRequireDefault(__webpack_require__(695)); + +var _validate = _interopRequireDefault(__webpack_require__(78)); + +var _stringify = _interopRequireDefault(__webpack_require__(411)); + +var _parse = _interopRequireDefault(__webpack_require__(22)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +/***/ }), + +/***/ 78: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _regex = _interopRequireDefault(__webpack_require__(456)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function validate(uuid) { + return typeof uuid === 'string' && _regex.default.test(uuid); +} + +var _default = validate; +exports.default = _default; + +/***/ }), + /***/ 82: /***/ (function(__unusedmodule, exports) { @@ -51,6 +244,7 @@ module.exports = // We use any as a valid input type /* eslint-disable @typescript-eslint/no-explicit-any */ Object.defineProperty(exports, "__esModule", { value: true }); +exports.toCommandProperties = exports.toCommandValue = void 0; /** * Sanitizes an input into a string so it can be passed into issueCommand safely * @param input input to sanitize into a string @@ -65,6 +259,26 @@ function toCommandValue(input) { return JSON.stringify(input); } exports.toCommandValue = toCommandValue; +/** + * + * @param annotationProperties + * @returns The command properties to send with the actual annotation command + * See IssueCommandProperties: https://github.com/actions/runner/blob/main/src/Runner.Worker/ActionCommandManager.cs#L646 + */ +function toCommandProperties(annotationProperties) { + if (!Object.keys(annotationProperties).length) { + return {}; + } + return { + title: annotationProperties.title, + file: annotationProperties.file, + line: annotationProperties.startLine, + endLine: annotationProperties.endLine, + col: annotationProperties.startColumn, + endColumn: annotationProperties.endColumn + }; +} +exports.toCommandProperties = toCommandProperties; //# sourceMappingURL=utils.js.map /***/ }), @@ -1012,14 +1226,27 @@ function regExpEscape (s) { "use strict"; // For internal use, subject to change. +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); var __importStar = (this && this.__importStar) || function (mod) { if (mod && mod.__esModule) return mod; var result = {}; - if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; - result["default"] = mod; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); return result; }; Object.defineProperty(exports, "__esModule", { value: true }); +exports.issueCommand = void 0; // We use any as a valid input type /* eslint-disable @typescript-eslint/no-explicit-any */ const fs = __importStar(__webpack_require__(747)); @@ -1042,101 +1269,556 @@ exports.issueCommand = issueCommand; /***/ }), -/***/ 281: +/***/ 141: /***/ (function(__unusedmodule, exports, __webpack_require__) { "use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); + +var net = __webpack_require__(631); +var tls = __webpack_require__(16); +var http = __webpack_require__(605); +var https = __webpack_require__(211); +var events = __webpack_require__(614); +var assert = __webpack_require__(357); +var util = __webpack_require__(669); + + +exports.httpOverHttp = httpOverHttp; +exports.httpsOverHttp = httpsOverHttp; +exports.httpOverHttps = httpOverHttps; +exports.httpsOverHttps = httpsOverHttps; + + +function httpOverHttp(options) { + var agent = new TunnelingAgent(options); + agent.request = http.request; + return agent; +} + +function httpsOverHttp(options) { + var agent = new TunnelingAgent(options); + agent.request = http.request; + agent.createSocket = createSecureSocket; + agent.defaultPort = 443; + return agent; +} + +function httpOverHttps(options) { + var agent = new TunnelingAgent(options); + agent.request = https.request; + return agent; +} + +function httpsOverHttps(options) { + var agent = new TunnelingAgent(options); + agent.request = https.request; + agent.createSocket = createSecureSocket; + agent.defaultPort = 443; + return agent; +} + + +function TunnelingAgent(options) { + var self = this; + self.options = options || {}; + self.proxyOptions = self.options.proxy || {}; + self.maxSockets = self.options.maxSockets || http.Agent.defaultMaxSockets; + self.requests = []; + self.sockets = []; + + self.on('free', function onFree(socket, host, port, localAddress) { + var options = toOptions(host, port, localAddress); + for (var i = 0, len = self.requests.length; i < len; ++i) { + var pending = self.requests[i]; + if (pending.host === options.host && pending.port === options.port) { + // Detect the request to connect same origin server, + // reuse the connection. + self.requests.splice(i, 1); + pending.request.onSocket(socket); + return; + } + } + socket.destroy(); + self.removeSocket(socket); + }); +} +util.inherits(TunnelingAgent, events.EventEmitter); + +TunnelingAgent.prototype.addRequest = function addRequest(req, host, port, localAddress) { + var self = this; + var options = mergeOptions({request: req}, self.options, toOptions(host, port, localAddress)); + + if (self.sockets.length >= this.maxSockets) { + // We are over limit so we'll add it to the queue. + self.requests.push(options); + return; + } + + // If we are under maxSockets create a new one. + self.createSocket(options, function(socket) { + socket.on('free', onFree); + socket.on('close', onCloseOrRemove); + socket.on('agentRemove', onCloseOrRemove); + req.onSocket(socket); + + function onFree() { + self.emit('free', socket, options); + } + + function onCloseOrRemove(err) { + self.removeSocket(socket); + socket.removeListener('free', onFree); + socket.removeListener('close', onCloseOrRemove); + socket.removeListener('agentRemove', onCloseOrRemove); + } + }); +}; + +TunnelingAgent.prototype.createSocket = function createSocket(options, cb) { + var self = this; + var placeholder = {}; + self.sockets.push(placeholder); + + var connectOptions = mergeOptions({}, self.proxyOptions, { + method: 'CONNECT', + path: options.host + ':' + options.port, + agent: false, + headers: { + host: options.host + ':' + options.port + } + }); + if (options.localAddress) { + connectOptions.localAddress = options.localAddress; + } + if (connectOptions.proxyAuth) { + connectOptions.headers = connectOptions.headers || {}; + connectOptions.headers['Proxy-Authorization'] = 'Basic ' + + new Buffer(connectOptions.proxyAuth).toString('base64'); + } + + debug('making CONNECT request'); + var connectReq = self.request(connectOptions); + connectReq.useChunkedEncodingByDefault = false; // for v0.6 + connectReq.once('response', onResponse); // for v0.6 + connectReq.once('upgrade', onUpgrade); // for v0.6 + connectReq.once('connect', onConnect); // for v0.7 or later + connectReq.once('error', onError); + connectReq.end(); + + function onResponse(res) { + // Very hacky. This is necessary to avoid http-parser leaks. + res.upgrade = true; + } + + function onUpgrade(res, socket, head) { + // Hacky. + process.nextTick(function() { + onConnect(res, socket, head); }); + } + + function onConnect(res, socket, head) { + connectReq.removeAllListeners(); + socket.removeAllListeners(); + + if (res.statusCode !== 200) { + debug('tunneling socket could not be established, statusCode=%d', + res.statusCode); + socket.destroy(); + var error = new Error('tunneling socket could not be established, ' + + 'statusCode=' + res.statusCode); + error.code = 'ECONNRESET'; + options.request.emit('error', error); + self.removeSocket(placeholder); + return; + } + if (head.length > 0) { + debug('got illegal response body from proxy'); + socket.destroy(); + var error = new Error('got illegal response body from proxy'); + error.code = 'ECONNRESET'; + options.request.emit('error', error); + self.removeSocket(placeholder); + return; + } + debug('tunneling connection has established'); + self.sockets[self.sockets.indexOf(placeholder)] = socket; + return cb(socket); + } + + function onError(cause) { + connectReq.removeAllListeners(); + + debug('tunneling socket could not be established, cause=%s\n', + cause.message, cause.stack); + var error = new Error('tunneling socket could not be established, ' + + 'cause=' + cause.message); + error.code = 'ECONNRESET'; + options.request.emit('error', error); + self.removeSocket(placeholder); + } }; -Object.defineProperty(exports, "__esModule", { value: true }); -const internal_globber_1 = __webpack_require__(297); -/** - * Constructs a globber - * - * @param patterns Patterns separated by newlines - * @param options Glob options - */ -function create(patterns, options) { - return __awaiter(this, void 0, void 0, function* () { - return yield internal_globber_1.DefaultGlobber.create(patterns, options); + +TunnelingAgent.prototype.removeSocket = function removeSocket(socket) { + var pos = this.sockets.indexOf(socket) + if (pos === -1) { + return; + } + this.sockets.splice(pos, 1); + + var pending = this.requests.shift(); + if (pending) { + // If we have pending requests and a socket gets closed a new one + // needs to be created to take over in the pool for the one that closed. + this.createSocket(pending, function(socket) { + pending.request.onSocket(socket); + }); + } +}; + +function createSecureSocket(options, cb) { + var self = this; + TunnelingAgent.prototype.createSocket.call(self, options, function(socket) { + var hostHeader = options.request.getHeader('host'); + var tlsOptions = mergeOptions({}, self.options, { + socket: socket, + servername: hostHeader ? hostHeader.replace(/:.*$/, '') : options.host }); + + // 0 is dummy port for v0.6 + var secureSocket = tls.connect(0, tlsOptions); + self.sockets[self.sockets.indexOf(socket)] = secureSocket; + cb(secureSocket); + }); } -exports.create = create; -//# sourceMappingURL=glob.js.map + + +function toOptions(host, port, localAddress) { + if (typeof host === 'string') { // since v0.10 + return { + host: host, + port: port, + localAddress: localAddress + }; + } + return host; // for v0.11 or later +} + +function mergeOptions(target) { + for (var i = 1, len = arguments.length; i < len; ++i) { + var overrides = arguments[i]; + if (typeof overrides === 'object') { + var keys = Object.keys(overrides); + for (var j = 0, keyLen = keys.length; j < keyLen; ++j) { + var k = keys[j]; + if (overrides[k] !== undefined) { + target[k] = overrides[k]; + } + } + } + } + return target; +} + + +var debug; +if (process.env.NODE_DEBUG && /\btunnel\b/.test(process.env.NODE_DEBUG)) { + debug = function() { + var args = Array.prototype.slice.call(arguments); + if (typeof args[0] === 'string') { + args[0] = 'TUNNEL: ' + args[0]; + } else { + args.unshift('TUNNEL:'); + } + console.error.apply(console, args); + } +} else { + debug = function() {}; +} +exports.debug = debug; // for test + /***/ }), -/***/ 297: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 177: +/***/ (function(__unusedmodule, exports) { "use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __asyncValues = (this && this.__asyncValues) || function (o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator], i; - return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); - function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } - function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } -}; -var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } -var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var g = generator.apply(thisArg, _arguments || []), i, q = []; - return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; - function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } - function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } - function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } - function fulfill(value) { resume("next", value); } - function reject(value) { resume("throw", value); } - function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } -}; Object.defineProperty(exports, "__esModule", { value: true }); -const core = __webpack_require__(470); -const fs = __webpack_require__(747); -const globOptionsHelper = __webpack_require__(601); -const path = __webpack_require__(622); -const patternHelper = __webpack_require__(597); -const internal_match_kind_1 = __webpack_require__(327); -const internal_pattern_1 = __webpack_require__(923); -const internal_search_state_1 = __webpack_require__(728); -const IS_WINDOWS = process.platform === 'win32'; -class DefaultGlobber { - constructor(options) { - this.patterns = []; - this.searchPaths = []; - this.options = globOptionsHelper.getOptions(options); +exports.checkBypass = exports.getProxyUrl = void 0; +function getProxyUrl(reqUrl) { + const usingSsl = reqUrl.protocol === 'https:'; + if (checkBypass(reqUrl)) { + return undefined; } - getSearchPaths() { - // Return a copy - return this.searchPaths.slice(); + const proxyVar = (() => { + if (usingSsl) { + return process.env['https_proxy'] || process.env['HTTPS_PROXY']; + } + else { + return process.env['http_proxy'] || process.env['HTTP_PROXY']; + } + })(); + if (proxyVar) { + return new URL(proxyVar); } - glob() { - var e_1, _a; - return __awaiter(this, void 0, void 0, function* () { - const result = []; - try { - for (var _b = __asyncValues(this.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { - const itemPath = _c.value; - result.push(itemPath); - } - } + else { + return undefined; + } +} +exports.getProxyUrl = getProxyUrl; +function checkBypass(reqUrl) { + if (!reqUrl.hostname) { + return false; + } + const noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; + if (!noProxy) { + return false; + } + // Determine the request port + let reqPort; + if (reqUrl.port) { + reqPort = Number(reqUrl.port); + } + else if (reqUrl.protocol === 'http:') { + reqPort = 80; + } + else if (reqUrl.protocol === 'https:') { + reqPort = 443; + } + // Format the request hostname and hostname with port + const upperReqHosts = [reqUrl.hostname.toUpperCase()]; + if (typeof reqPort === 'number') { + upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); + } + // Compare request host against noproxy + for (const upperNoProxyItem of noProxy + .split(',') + .map(x => x.trim().toUpperCase()) + .filter(x => x)) { + if (upperReqHosts.some(x => x === upperNoProxyItem)) { + return true; + } + } + return false; +} +exports.checkBypass = checkBypass; +//# sourceMappingURL=proxy.js.map + +/***/ }), + +/***/ 209: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _v = _interopRequireDefault(__webpack_require__(212)); + +var _md = _interopRequireDefault(__webpack_require__(803)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +const v3 = (0, _v.default)('v3', 0x30, _md.default); +var _default = v3; +exports.default = _default; + +/***/ }), + +/***/ 211: +/***/ (function(module) { + +module.exports = require("https"); + +/***/ }), + +/***/ 212: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = _default; +exports.URL = exports.DNS = void 0; + +var _stringify = _interopRequireDefault(__webpack_require__(411)); + +var _parse = _interopRequireDefault(__webpack_require__(22)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function stringToBytes(str) { + str = unescape(encodeURIComponent(str)); // UTF8 escape + + const bytes = []; + + for (let i = 0; i < str.length; ++i) { + bytes.push(str.charCodeAt(i)); + } + + return bytes; +} + +const DNS = '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; +exports.DNS = DNS; +const URL = '6ba7b811-9dad-11d1-80b4-00c04fd430c8'; +exports.URL = URL; + +function _default(name, version, hashfunc) { + function generateUUID(value, namespace, buf, offset) { + if (typeof value === 'string') { + value = stringToBytes(value); + } + + if (typeof namespace === 'string') { + namespace = (0, _parse.default)(namespace); + } + + if (namespace.length !== 16) { + throw TypeError('Namespace must be array-like (16 iterable integer values, 0-255)'); + } // Compute hash of namespace and value, Per 4.3 + // Future: Use spread syntax when supported on all platforms, e.g. `bytes = + // hashfunc([...namespace, ... value])` + + + let bytes = new Uint8Array(16 + value.length); + bytes.set(namespace); + bytes.set(value, namespace.length); + bytes = hashfunc(bytes); + bytes[6] = bytes[6] & 0x0f | version; + bytes[8] = bytes[8] & 0x3f | 0x80; + + if (buf) { + offset = offset || 0; + + for (let i = 0; i < 16; ++i) { + buf[offset + i] = bytes[i]; + } + + return buf; + } + + return (0, _stringify.default)(bytes); + } // Function#name is not settable on some platforms (#270) + + + try { + generateUUID.name = name; // eslint-disable-next-line no-empty + } catch (err) {} // For CommonJS default export support + + + generateUUID.DNS = DNS; + generateUUID.URL = URL; + return generateUUID; +} + +/***/ }), + +/***/ 281: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const internal_globber_1 = __webpack_require__(297); +/** + * Constructs a globber + * + * @param patterns Patterns separated by newlines + * @param options Glob options + */ +function create(patterns, options) { + return __awaiter(this, void 0, void 0, function* () { + return yield internal_globber_1.DefaultGlobber.create(patterns, options); + }); +} +exports.create = create; +//# sourceMappingURL=glob.js.map + +/***/ }), + +/***/ 297: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); } +var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var g = generator.apply(thisArg, _arguments || []), i, q = []; + return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i; + function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; } + function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } } + function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); } + function fulfill(value) { resume("next", value); } + function reject(value) { resume("throw", value); } + function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } +}; +Object.defineProperty(exports, "__esModule", { value: true }); +const core = __webpack_require__(470); +const fs = __webpack_require__(747); +const globOptionsHelper = __webpack_require__(601); +const path = __webpack_require__(622); +const patternHelper = __webpack_require__(597); +const internal_match_kind_1 = __webpack_require__(36); +const internal_pattern_1 = __webpack_require__(923); +const internal_search_state_1 = __webpack_require__(728); +const IS_WINDOWS = process.platform === 'win32'; +class DefaultGlobber { + constructor(options) { + this.patterns = []; + this.searchPaths = []; + this.options = globOptionsHelper.getOptions(options); + } + getSearchPaths() { + // Return a copy + return this.searchPaths.slice(); + } + glob() { + var e_1, _a; + return __awaiter(this, void 0, void 0, function* () { + const result = []; + try { + for (var _b = __asyncValues(this.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { + const itemPath = _c.value; + result.push(itemPath); + } + } catch (e_1_1) { e_1 = { error: e_1_1 }; } finally { try { @@ -1509,22 +2191,13 @@ function expand(str, isTop) { "use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -/** - * Indicates whether a pattern matches a path - */ -var MatchKind; -(function (MatchKind) { - /** Not matched */ - MatchKind[MatchKind["None"] = 0] = "None"; - /** Matched if the path is a directory */ - MatchKind[MatchKind["Directory"] = 1] = "Directory"; - /** Matched if the path is a regular file */ - MatchKind[MatchKind["File"] = 2] = "File"; - /** Matched */ - MatchKind[MatchKind["All"] = 3] = "All"; -})(MatchKind = exports.MatchKind || (exports.MatchKind = {})); -//# sourceMappingURL=internal-match-kind.js.map + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; +var _default = '00000000-0000-0000-0000-000000000000'; +exports.default = _default; /***/ }), @@ -1561,7 +2234,7 @@ const crypto = __importStar(__webpack_require__(417)); const fs = __importStar(__webpack_require__(747)); const glob = __importStar(__webpack_require__(281)); const path = __importStar(__webpack_require__(622)); -const stream = __importStar(__webpack_require__(413)); +const stream = __importStar(__webpack_require__(794)); const util = __importStar(__webpack_require__(669)); function run() { var e_1, _a; @@ -1738,10 +2411,80 @@ exports.Path = Path; /***/ }), +/***/ 384: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _v = _interopRequireDefault(__webpack_require__(212)); + +var _sha = _interopRequireDefault(__webpack_require__(498)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +const v5 = (0, _v.default)('v5', 0x50, _sha.default); +var _default = v5; +exports.default = _default; + +/***/ }), + +/***/ 411: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _validate = _interopRequireDefault(__webpack_require__(78)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +/** + * Convert array of 16 byte values to UUID string format of the form: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + */ +const byteToHex = []; + +for (let i = 0; i < 256; ++i) { + byteToHex.push((i + 0x100).toString(16).substr(1)); +} + +function stringify(arr, offset = 0) { + // Note: Be careful editing this code! It's been tuned for performance + // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 + const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + '-' + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + '-' + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + '-' + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + '-' + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase(); // Consistency check for valid UUID. If this throws, it's likely due to one + // of the following: + // - One or more input array values don't map to a hex octet (leading to + // "undefined" in the uuid) + // - Invalid input values for the RFC `version` or `variant` fields + + if (!(0, _validate.default)(uuid)) { + throw TypeError('Stringified UUID is invalid'); + } + + return uuid; +} + +var _default = stringify; +exports.default = _default; + +/***/ }), + /***/ 413: -/***/ (function(module) { +/***/ (function(module, __unusedexports, __webpack_require__) { + +module.exports = __webpack_require__(141); -module.exports = require("stream"); /***/ }), @@ -1752,43 +2495,668 @@ module.exports = require("crypto"); /***/ }), -/***/ 431: +/***/ 425: /***/ (function(__unusedmodule, exports, __webpack_require__) { "use strict"; +/* eslint-disable @typescript-eslint/no-explicit-any */ +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); var __importStar = (this && this.__importStar) || function (mod) { if (mod && mod.__esModule) return mod; var result = {}; - if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; - result["default"] = mod; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); return result; }; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; Object.defineProperty(exports, "__esModule", { value: true }); -const os = __importStar(__webpack_require__(87)); -const utils_1 = __webpack_require__(82); +exports.HttpClient = exports.isHttps = exports.HttpClientResponse = exports.HttpClientError = exports.getProxyUrl = exports.MediaTypes = exports.Headers = exports.HttpCodes = void 0; +const http = __importStar(__webpack_require__(605)); +const https = __importStar(__webpack_require__(211)); +const pm = __importStar(__webpack_require__(177)); +const tunnel = __importStar(__webpack_require__(413)); +var HttpCodes; +(function (HttpCodes) { + HttpCodes[HttpCodes["OK"] = 200] = "OK"; + HttpCodes[HttpCodes["MultipleChoices"] = 300] = "MultipleChoices"; + HttpCodes[HttpCodes["MovedPermanently"] = 301] = "MovedPermanently"; + HttpCodes[HttpCodes["ResourceMoved"] = 302] = "ResourceMoved"; + HttpCodes[HttpCodes["SeeOther"] = 303] = "SeeOther"; + HttpCodes[HttpCodes["NotModified"] = 304] = "NotModified"; + HttpCodes[HttpCodes["UseProxy"] = 305] = "UseProxy"; + HttpCodes[HttpCodes["SwitchProxy"] = 306] = "SwitchProxy"; + HttpCodes[HttpCodes["TemporaryRedirect"] = 307] = "TemporaryRedirect"; + HttpCodes[HttpCodes["PermanentRedirect"] = 308] = "PermanentRedirect"; + HttpCodes[HttpCodes["BadRequest"] = 400] = "BadRequest"; + HttpCodes[HttpCodes["Unauthorized"] = 401] = "Unauthorized"; + HttpCodes[HttpCodes["PaymentRequired"] = 402] = "PaymentRequired"; + HttpCodes[HttpCodes["Forbidden"] = 403] = "Forbidden"; + HttpCodes[HttpCodes["NotFound"] = 404] = "NotFound"; + HttpCodes[HttpCodes["MethodNotAllowed"] = 405] = "MethodNotAllowed"; + HttpCodes[HttpCodes["NotAcceptable"] = 406] = "NotAcceptable"; + HttpCodes[HttpCodes["ProxyAuthenticationRequired"] = 407] = "ProxyAuthenticationRequired"; + HttpCodes[HttpCodes["RequestTimeout"] = 408] = "RequestTimeout"; + HttpCodes[HttpCodes["Conflict"] = 409] = "Conflict"; + HttpCodes[HttpCodes["Gone"] = 410] = "Gone"; + HttpCodes[HttpCodes["TooManyRequests"] = 429] = "TooManyRequests"; + HttpCodes[HttpCodes["InternalServerError"] = 500] = "InternalServerError"; + HttpCodes[HttpCodes["NotImplemented"] = 501] = "NotImplemented"; + HttpCodes[HttpCodes["BadGateway"] = 502] = "BadGateway"; + HttpCodes[HttpCodes["ServiceUnavailable"] = 503] = "ServiceUnavailable"; + HttpCodes[HttpCodes["GatewayTimeout"] = 504] = "GatewayTimeout"; +})(HttpCodes = exports.HttpCodes || (exports.HttpCodes = {})); +var Headers; +(function (Headers) { + Headers["Accept"] = "accept"; + Headers["ContentType"] = "content-type"; +})(Headers = exports.Headers || (exports.Headers = {})); +var MediaTypes; +(function (MediaTypes) { + MediaTypes["ApplicationJson"] = "application/json"; +})(MediaTypes = exports.MediaTypes || (exports.MediaTypes = {})); /** - * Commands - * - * Command Format: - * ::name key=value,key=value::message - * - * Examples: - * ::warning::This is the message - * ::set-env name=MY_VAR::some value + * Returns the proxy URL, depending upon the supplied url and proxy environment variables. + * @param serverUrl The server URL where the request will be sent. For example, https://api.github.com */ -function issueCommand(command, properties, message) { - const cmd = new Command(command, properties, message); - process.stdout.write(cmd.toString() + os.EOL); +function getProxyUrl(serverUrl) { + const proxyUrl = pm.getProxyUrl(new URL(serverUrl)); + return proxyUrl ? proxyUrl.href : ''; } -exports.issueCommand = issueCommand; -function issue(name, message = '') { - issueCommand(name, {}, message); +exports.getProxyUrl = getProxyUrl; +const HttpRedirectCodes = [ + HttpCodes.MovedPermanently, + HttpCodes.ResourceMoved, + HttpCodes.SeeOther, + HttpCodes.TemporaryRedirect, + HttpCodes.PermanentRedirect +]; +const HttpResponseRetryCodes = [ + HttpCodes.BadGateway, + HttpCodes.ServiceUnavailable, + HttpCodes.GatewayTimeout +]; +const RetryableHttpVerbs = ['OPTIONS', 'GET', 'DELETE', 'HEAD']; +const ExponentialBackoffCeiling = 10; +const ExponentialBackoffTimeSlice = 5; +class HttpClientError extends Error { + constructor(message, statusCode) { + super(message); + this.name = 'HttpClientError'; + this.statusCode = statusCode; + Object.setPrototypeOf(this, HttpClientError.prototype); + } } -exports.issue = issue; -const CMD_STRING = '::'; -class Command { - constructor(command, properties, message) { +exports.HttpClientError = HttpClientError; +class HttpClientResponse { + constructor(message) { + this.message = message; + } + readBody() { + return __awaiter(this, void 0, void 0, function* () { + return new Promise((resolve) => __awaiter(this, void 0, void 0, function* () { + let output = Buffer.alloc(0); + this.message.on('data', (chunk) => { + output = Buffer.concat([output, chunk]); + }); + this.message.on('end', () => { + resolve(output.toString()); + }); + })); + }); + } +} +exports.HttpClientResponse = HttpClientResponse; +function isHttps(requestUrl) { + const parsedUrl = new URL(requestUrl); + return parsedUrl.protocol === 'https:'; +} +exports.isHttps = isHttps; +class HttpClient { + constructor(userAgent, handlers, requestOptions) { + this._ignoreSslError = false; + this._allowRedirects = true; + this._allowRedirectDowngrade = false; + this._maxRedirects = 50; + this._allowRetries = false; + this._maxRetries = 1; + this._keepAlive = false; + this._disposed = false; + this.userAgent = userAgent; + this.handlers = handlers || []; + this.requestOptions = requestOptions; + if (requestOptions) { + if (requestOptions.ignoreSslError != null) { + this._ignoreSslError = requestOptions.ignoreSslError; + } + this._socketTimeout = requestOptions.socketTimeout; + if (requestOptions.allowRedirects != null) { + this._allowRedirects = requestOptions.allowRedirects; + } + if (requestOptions.allowRedirectDowngrade != null) { + this._allowRedirectDowngrade = requestOptions.allowRedirectDowngrade; + } + if (requestOptions.maxRedirects != null) { + this._maxRedirects = Math.max(requestOptions.maxRedirects, 0); + } + if (requestOptions.keepAlive != null) { + this._keepAlive = requestOptions.keepAlive; + } + if (requestOptions.allowRetries != null) { + this._allowRetries = requestOptions.allowRetries; + } + if (requestOptions.maxRetries != null) { + this._maxRetries = requestOptions.maxRetries; + } + } + } + options(requestUrl, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('OPTIONS', requestUrl, null, additionalHeaders || {}); + }); + } + get(requestUrl, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('GET', requestUrl, null, additionalHeaders || {}); + }); + } + del(requestUrl, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('DELETE', requestUrl, null, additionalHeaders || {}); + }); + } + post(requestUrl, data, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('POST', requestUrl, data, additionalHeaders || {}); + }); + } + patch(requestUrl, data, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('PATCH', requestUrl, data, additionalHeaders || {}); + }); + } + put(requestUrl, data, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('PUT', requestUrl, data, additionalHeaders || {}); + }); + } + head(requestUrl, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request('HEAD', requestUrl, null, additionalHeaders || {}); + }); + } + sendStream(verb, requestUrl, stream, additionalHeaders) { + return __awaiter(this, void 0, void 0, function* () { + return this.request(verb, requestUrl, stream, additionalHeaders); + }); + } + /** + * Gets a typed object from an endpoint + * Be aware that not found returns a null. Other errors (4xx, 5xx) reject the promise + */ + getJson(requestUrl, additionalHeaders = {}) { + return __awaiter(this, void 0, void 0, function* () { + additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson); + const res = yield this.get(requestUrl, additionalHeaders); + return this._processResponse(res, this.requestOptions); + }); + } + postJson(requestUrl, obj, additionalHeaders = {}) { + return __awaiter(this, void 0, void 0, function* () { + const data = JSON.stringify(obj, null, 2); + additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson); + additionalHeaders[Headers.ContentType] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.ContentType, MediaTypes.ApplicationJson); + const res = yield this.post(requestUrl, data, additionalHeaders); + return this._processResponse(res, this.requestOptions); + }); + } + putJson(requestUrl, obj, additionalHeaders = {}) { + return __awaiter(this, void 0, void 0, function* () { + const data = JSON.stringify(obj, null, 2); + additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson); + additionalHeaders[Headers.ContentType] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.ContentType, MediaTypes.ApplicationJson); + const res = yield this.put(requestUrl, data, additionalHeaders); + return this._processResponse(res, this.requestOptions); + }); + } + patchJson(requestUrl, obj, additionalHeaders = {}) { + return __awaiter(this, void 0, void 0, function* () { + const data = JSON.stringify(obj, null, 2); + additionalHeaders[Headers.Accept] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.Accept, MediaTypes.ApplicationJson); + additionalHeaders[Headers.ContentType] = this._getExistingOrDefaultHeader(additionalHeaders, Headers.ContentType, MediaTypes.ApplicationJson); + const res = yield this.patch(requestUrl, data, additionalHeaders); + return this._processResponse(res, this.requestOptions); + }); + } + /** + * Makes a raw http request. + * All other methods such as get, post, patch, and request ultimately call this. + * Prefer get, del, post and patch + */ + request(verb, requestUrl, data, headers) { + return __awaiter(this, void 0, void 0, function* () { + if (this._disposed) { + throw new Error('Client has already been disposed.'); + } + const parsedUrl = new URL(requestUrl); + let info = this._prepareRequest(verb, parsedUrl, headers); + // Only perform retries on reads since writes may not be idempotent. + const maxTries = this._allowRetries && RetryableHttpVerbs.includes(verb) + ? this._maxRetries + 1 + : 1; + let numTries = 0; + let response; + do { + response = yield this.requestRaw(info, data); + // Check if it's an authentication challenge + if (response && + response.message && + response.message.statusCode === HttpCodes.Unauthorized) { + let authenticationHandler; + for (const handler of this.handlers) { + if (handler.canHandleAuthentication(response)) { + authenticationHandler = handler; + break; + } + } + if (authenticationHandler) { + return authenticationHandler.handleAuthentication(this, info, data); + } + else { + // We have received an unauthorized response but have no handlers to handle it. + // Let the response return to the caller. + return response; + } + } + let redirectsRemaining = this._maxRedirects; + while (response.message.statusCode && + HttpRedirectCodes.includes(response.message.statusCode) && + this._allowRedirects && + redirectsRemaining > 0) { + const redirectUrl = response.message.headers['location']; + if (!redirectUrl) { + // if there's no location to redirect to, we won't + break; + } + const parsedRedirectUrl = new URL(redirectUrl); + if (parsedUrl.protocol === 'https:' && + parsedUrl.protocol !== parsedRedirectUrl.protocol && + !this._allowRedirectDowngrade) { + throw new Error('Redirect from HTTPS to HTTP protocol. This downgrade is not allowed for security reasons. If you want to allow this behavior, set the allowRedirectDowngrade option to true.'); + } + // we need to finish reading the response before reassigning response + // which will leak the open socket. + yield response.readBody(); + // strip authorization header if redirected to a different hostname + if (parsedRedirectUrl.hostname !== parsedUrl.hostname) { + for (const header in headers) { + // header names are case insensitive + if (header.toLowerCase() === 'authorization') { + delete headers[header]; + } + } + } + // let's make the request with the new redirectUrl + info = this._prepareRequest(verb, parsedRedirectUrl, headers); + response = yield this.requestRaw(info, data); + redirectsRemaining--; + } + if (!response.message.statusCode || + !HttpResponseRetryCodes.includes(response.message.statusCode)) { + // If not a retry code, return immediately instead of retrying + return response; + } + numTries += 1; + if (numTries < maxTries) { + yield response.readBody(); + yield this._performExponentialBackoff(numTries); + } + } while (numTries < maxTries); + return response; + }); + } + /** + * Needs to be called if keepAlive is set to true in request options. + */ + dispose() { + if (this._agent) { + this._agent.destroy(); + } + this._disposed = true; + } + /** + * Raw request. + * @param info + * @param data + */ + requestRaw(info, data) { + return __awaiter(this, void 0, void 0, function* () { + return new Promise((resolve, reject) => { + function callbackForResult(err, res) { + if (err) { + reject(err); + } + else if (!res) { + // If `err` is not passed, then `res` must be passed. + reject(new Error('Unknown error')); + } + else { + resolve(res); + } + } + this.requestRawWithCallback(info, data, callbackForResult); + }); + }); + } + /** + * Raw request with callback. + * @param info + * @param data + * @param onResult + */ + requestRawWithCallback(info, data, onResult) { + if (typeof data === 'string') { + if (!info.options.headers) { + info.options.headers = {}; + } + info.options.headers['Content-Length'] = Buffer.byteLength(data, 'utf8'); + } + let callbackCalled = false; + function handleResult(err, res) { + if (!callbackCalled) { + callbackCalled = true; + onResult(err, res); + } + } + const req = info.httpModule.request(info.options, (msg) => { + const res = new HttpClientResponse(msg); + handleResult(undefined, res); + }); + let socket; + req.on('socket', sock => { + socket = sock; + }); + // If we ever get disconnected, we want the socket to timeout eventually + req.setTimeout(this._socketTimeout || 3 * 60000, () => { + if (socket) { + socket.end(); + } + handleResult(new Error(`Request timeout: ${info.options.path}`)); + }); + req.on('error', function (err) { + // err has statusCode property + // res should have headers + handleResult(err); + }); + if (data && typeof data === 'string') { + req.write(data, 'utf8'); + } + if (data && typeof data !== 'string') { + data.on('close', function () { + req.end(); + }); + data.pipe(req); + } + else { + req.end(); + } + } + /** + * Gets an http agent. This function is useful when you need an http agent that handles + * routing through a proxy server - depending upon the url and proxy environment variables. + * @param serverUrl The server URL where the request will be sent. For example, https://api.github.com + */ + getAgent(serverUrl) { + const parsedUrl = new URL(serverUrl); + return this._getAgent(parsedUrl); + } + _prepareRequest(method, requestUrl, headers) { + const info = {}; + info.parsedUrl = requestUrl; + const usingSsl = info.parsedUrl.protocol === 'https:'; + info.httpModule = usingSsl ? https : http; + const defaultPort = usingSsl ? 443 : 80; + info.options = {}; + info.options.host = info.parsedUrl.hostname; + info.options.port = info.parsedUrl.port + ? parseInt(info.parsedUrl.port) + : defaultPort; + info.options.path = + (info.parsedUrl.pathname || '') + (info.parsedUrl.search || ''); + info.options.method = method; + info.options.headers = this._mergeHeaders(headers); + if (this.userAgent != null) { + info.options.headers['user-agent'] = this.userAgent; + } + info.options.agent = this._getAgent(info.parsedUrl); + // gives handlers an opportunity to participate + if (this.handlers) { + for (const handler of this.handlers) { + handler.prepareRequest(info.options); + } + } + return info; + } + _mergeHeaders(headers) { + if (this.requestOptions && this.requestOptions.headers) { + return Object.assign({}, lowercaseKeys(this.requestOptions.headers), lowercaseKeys(headers || {})); + } + return lowercaseKeys(headers || {}); + } + _getExistingOrDefaultHeader(additionalHeaders, header, _default) { + let clientHeader; + if (this.requestOptions && this.requestOptions.headers) { + clientHeader = lowercaseKeys(this.requestOptions.headers)[header]; + } + return additionalHeaders[header] || clientHeader || _default; + } + _getAgent(parsedUrl) { + let agent; + const proxyUrl = pm.getProxyUrl(parsedUrl); + const useProxy = proxyUrl && proxyUrl.hostname; + if (this._keepAlive && useProxy) { + agent = this._proxyAgent; + } + if (this._keepAlive && !useProxy) { + agent = this._agent; + } + // if agent is already assigned use that agent. + if (agent) { + return agent; + } + const usingSsl = parsedUrl.protocol === 'https:'; + let maxSockets = 100; + if (this.requestOptions) { + maxSockets = this.requestOptions.maxSockets || http.globalAgent.maxSockets; + } + // This is `useProxy` again, but we need to check `proxyURl` directly for TypeScripts's flow analysis. + if (proxyUrl && proxyUrl.hostname) { + const agentOptions = { + maxSockets, + keepAlive: this._keepAlive, + proxy: Object.assign(Object.assign({}, ((proxyUrl.username || proxyUrl.password) && { + proxyAuth: `${proxyUrl.username}:${proxyUrl.password}` + })), { host: proxyUrl.hostname, port: proxyUrl.port }) + }; + let tunnelAgent; + const overHttps = proxyUrl.protocol === 'https:'; + if (usingSsl) { + tunnelAgent = overHttps ? tunnel.httpsOverHttps : tunnel.httpsOverHttp; + } + else { + tunnelAgent = overHttps ? tunnel.httpOverHttps : tunnel.httpOverHttp; + } + agent = tunnelAgent(agentOptions); + this._proxyAgent = agent; + } + // if reusing agent across request and tunneling agent isn't assigned create a new agent + if (this._keepAlive && !agent) { + const options = { keepAlive: this._keepAlive, maxSockets }; + agent = usingSsl ? new https.Agent(options) : new http.Agent(options); + this._agent = agent; + } + // if not using private agent and tunnel agent isn't setup then use global agent + if (!agent) { + agent = usingSsl ? https.globalAgent : http.globalAgent; + } + if (usingSsl && this._ignoreSslError) { + // we don't want to set NODE_TLS_REJECT_UNAUTHORIZED=0 since that will affect request for entire process + // http.RequestOptions doesn't expose a way to modify RequestOptions.agent.options + // we have to cast it to any and change it directly + agent.options = Object.assign(agent.options || {}, { + rejectUnauthorized: false + }); + } + return agent; + } + _performExponentialBackoff(retryNumber) { + return __awaiter(this, void 0, void 0, function* () { + retryNumber = Math.min(ExponentialBackoffCeiling, retryNumber); + const ms = ExponentialBackoffTimeSlice * Math.pow(2, retryNumber); + return new Promise(resolve => setTimeout(() => resolve(), ms)); + }); + } + _processResponse(res, options) { + return __awaiter(this, void 0, void 0, function* () { + return new Promise((resolve, reject) => __awaiter(this, void 0, void 0, function* () { + const statusCode = res.message.statusCode || 0; + const response = { + statusCode, + result: null, + headers: {} + }; + // not found leads to null obj returned + if (statusCode === HttpCodes.NotFound) { + resolve(response); + } + // get the result from the body + function dateTimeDeserializer(key, value) { + if (typeof value === 'string') { + const a = new Date(value); + if (!isNaN(a.valueOf())) { + return a; + } + } + return value; + } + let obj; + let contents; + try { + contents = yield res.readBody(); + if (contents && contents.length > 0) { + if (options && options.deserializeDates) { + obj = JSON.parse(contents, dateTimeDeserializer); + } + else { + obj = JSON.parse(contents); + } + response.result = obj; + } + response.headers = res.message.headers; + } + catch (err) { + // Invalid resource (contents not json); leaving result obj null + } + // note that 3xx redirects are handled by the http layer. + if (statusCode > 299) { + let msg; + // if exception/error in body, attempt to get better error + if (obj && obj.message) { + msg = obj.message; + } + else if (contents && contents.length > 0) { + // it may be the case that the exception is in the body message as string + msg = contents; + } + else { + msg = `Failed request: (${statusCode})`; + } + const err = new HttpClientError(msg, statusCode); + err.result = response.result; + reject(err); + } + else { + resolve(response); + } + })); + }); + } +} +exports.HttpClient = HttpClient; +const lowercaseKeys = (obj) => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCase()] = obj[k]), c), {}); +//# sourceMappingURL=index.js.map + +/***/ }), + +/***/ 431: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.issue = exports.issueCommand = void 0; +const os = __importStar(__webpack_require__(87)); +const utils_1 = __webpack_require__(82); +/** + * Commands + * + * Command Format: + * ::name key=value,key=value::message + * + * Examples: + * ::warning::This is the message + * ::set-env name=MY_VAR::some value + */ +function issueCommand(command, properties, message) { + const cmd = new Command(command, properties, message); + process.stdout.write(cmd.toString() + os.EOL); +} +exports.issueCommand = issueCommand; +function issue(name, message = '') { + issueCommand(name, {}, message); +} +exports.issue = issue; +const CMD_STRING = '::'; +class Command { + constructor(command, properties, message) { if (!command) { command = 'missing.command'; } @@ -1838,11 +3206,45 @@ function escapeProperty(s) { /***/ }), +/***/ 456: +/***/ (function(__unusedmodule, exports) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; +var _default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i; +exports.default = _default; + +/***/ }), + /***/ 470: /***/ (function(__unusedmodule, exports, __webpack_require__) { "use strict"; +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } return new (P || (P = Promise))(function (resolve, reject) { @@ -1852,19 +3254,15 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; - result["default"] = mod; - return result; -}; Object.defineProperty(exports, "__esModule", { value: true }); +exports.getIDToken = exports.getState = exports.saveState = exports.group = exports.endGroup = exports.startGroup = exports.info = exports.notice = exports.warning = exports.error = exports.debug = exports.isDebug = exports.setFailed = exports.setCommandEcho = exports.setOutput = exports.getBooleanInput = exports.getMultilineInput = exports.getInput = exports.addPath = exports.setSecret = exports.exportVariable = exports.ExitCode = void 0; const command_1 = __webpack_require__(431); const file_command_1 = __webpack_require__(102); const utils_1 = __webpack_require__(82); const os = __importStar(__webpack_require__(87)); const path = __importStar(__webpack_require__(622)); +const uuid_1 = __webpack_require__(62); +const oidc_utils_1 = __webpack_require__(742); /** * The code to exit an action */ @@ -1893,7 +3291,14 @@ function exportVariable(name, val) { process.env[name] = convertedVal; const filePath = process.env['GITHUB_ENV'] || ''; if (filePath) { - const delimiter = '_GitHubActionsFileCommandDelimeter_'; + const delimiter = `ghadelimiter_${uuid_1.v4()}`; + // These should realistically never happen, but just in case someone finds a way to exploit uuid generation let's not allow keys or values that contain the delimiter. + if (name.includes(delimiter)) { + throw new Error(`Unexpected input: name should not contain the delimiter "${delimiter}"`); + } + if (convertedVal.includes(delimiter)) { + throw new Error(`Unexpected input: value should not contain the delimiter "${delimiter}"`); + } const commandValue = `${name}<<${delimiter}${os.EOL}${convertedVal}${os.EOL}${delimiter}`; file_command_1.issueCommand('ENV', commandValue); } @@ -1926,7 +3331,9 @@ function addPath(inputPath) { } exports.addPath = addPath; /** - * Gets the value of an input. The value is also trimmed. + * Gets the value of an input. + * Unless trimWhitespace is set to false in InputOptions, the value is also trimmed. + * Returns an empty string if the value is not defined. * * @param name name of the input to get * @param options optional. See InputOptions. @@ -1937,9 +3344,49 @@ function getInput(name, options) { if (options && options.required && !val) { throw new Error(`Input required and not supplied: ${name}`); } + if (options && options.trimWhitespace === false) { + return val; + } return val.trim(); } exports.getInput = getInput; +/** + * Gets the values of an multiline input. Each value is also trimmed. + * + * @param name name of the input to get + * @param options optional. See InputOptions. + * @returns string[] + * + */ +function getMultilineInput(name, options) { + const inputs = getInput(name, options) + .split('\n') + .filter(x => x !== ''); + return inputs; +} +exports.getMultilineInput = getMultilineInput; +/** + * Gets the input value of the boolean type in the YAML 1.2 "core schema" specification. + * Support boolean input list: `true | True | TRUE | false | False | FALSE` . + * The return value is also in boolean type. + * ref: https://yaml.org/spec/1.2/spec.html#id2804923 + * + * @param name name of the input to get + * @param options optional. See InputOptions. + * @returns boolean + */ +function getBooleanInput(name, options) { + const trueValue = ['true', 'True', 'TRUE']; + const falseValue = ['false', 'False', 'FALSE']; + const val = getInput(name, options); + if (trueValue.includes(val)) + return true; + if (falseValue.includes(val)) + return false; + throw new TypeError(`Input does not meet YAML 1.2 "Core Schema" specification: ${name}\n` + + `Support boolean input list: \`true | True | TRUE | false | False | FALSE\``); +} +exports.getBooleanInput = getBooleanInput; /** * Sets the value of an output. * @@ -1948,6 +3395,7 @@ exports.getInput = getInput; */ // eslint-disable-next-line @typescript-eslint/no-explicit-any function setOutput(name, value) { + process.stdout.write(os.EOL); command_1.issueCommand('set-output', { name }, value); } exports.setOutput = setOutput; @@ -1994,19 +3442,30 @@ exports.debug = debug; /** * Adds an error issue * @param message error issue message. Errors will be converted to string via toString() + * @param properties optional properties to add to the annotation. */ -function error(message) { - command_1.issue('error', message instanceof Error ? message.toString() : message); +function error(message, properties = {}) { + command_1.issueCommand('error', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); } exports.error = error; /** - * Adds an warning issue + * Adds a warning issue * @param message warning issue message. Errors will be converted to string via toString() + * @param properties optional properties to add to the annotation. */ -function warning(message) { - command_1.issue('warning', message instanceof Error ? message.toString() : message); +function warning(message, properties = {}) { + command_1.issueCommand('warning', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); } exports.warning = warning; +/** + * Adds a notice issue + * @param message notice issue message. Errors will be converted to string via toString() + * @param properties optional properties to add to the annotation. + */ +function notice(message, properties = {}) { + command_1.issueCommand('notice', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); +} +exports.notice = notice; /** * Writes info to log with console.log. * @param message info message @@ -2079,29 +3538,235 @@ function getState(name) { return process.env[`STATE_${name}`] || ''; } exports.getState = getState; +function getIDToken(aud) { + return __awaiter(this, void 0, void 0, function* () { + return yield oidc_utils_1.OidcClient.getIDToken(aud); + }); +} +exports.getIDToken = getIDToken; +/** + * Summary exports + */ +var summary_1 = __webpack_require__(665); +Object.defineProperty(exports, "summary", { enumerable: true, get: function () { return summary_1.summary; } }); +/** + * @deprecated use core.summary + */ +var summary_2 = __webpack_require__(665); +Object.defineProperty(exports, "markdownSummary", { enumerable: true, get: function () { return summary_2.markdownSummary; } }); +/** + * Path exports + */ +var path_utils_1 = __webpack_require__(573); +Object.defineProperty(exports, "toPosixPath", { enumerable: true, get: function () { return path_utils_1.toPosixPath; } }); +Object.defineProperty(exports, "toWin32Path", { enumerable: true, get: function () { return path_utils_1.toWin32Path; } }); +Object.defineProperty(exports, "toPlatformPath", { enumerable: true, get: function () { return path_utils_1.toPlatformPath; } }); //# sourceMappingURL=core.js.map /***/ }), -/***/ 597: +/***/ 498: /***/ (function(__unusedmodule, exports, __webpack_require__) { "use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const pathHelper = __webpack_require__(972); -const internal_match_kind_1 = __webpack_require__(327); -const IS_WINDOWS = process.platform === 'win32'; -/** - * Given an array of patterns, returns an array of paths to search. - * Duplicates and paths under other included paths are filtered out. - */ -function getSearchPaths(patterns) { - // Ignore negate patterns - patterns = patterns.filter(x => !x.negate); - // Create a map of all search paths - const searchPathMap = {}; - for (const pattern of patterns) { + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _crypto = _interopRequireDefault(__webpack_require__(417)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function sha1(bytes) { + if (Array.isArray(bytes)) { + bytes = Buffer.from(bytes); + } else if (typeof bytes === 'string') { + bytes = Buffer.from(bytes, 'utf8'); + } + + return _crypto.default.createHash('sha1').update(bytes).digest(); +} + +var _default = sha1; +exports.default = _default; + +/***/ }), + +/***/ 554: +/***/ (function(__unusedmodule, exports) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.PersonalAccessTokenCredentialHandler = exports.BearerCredentialHandler = exports.BasicCredentialHandler = void 0; +class BasicCredentialHandler { + constructor(username, password) { + this.username = username; + this.password = password; + } + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString('base64')}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } +} +exports.BasicCredentialHandler = BasicCredentialHandler; +class BearerCredentialHandler { + constructor(token) { + this.token = token; + } + // currently implements pre-authorization + // TODO: support preAuth = false where it hooks on 401 + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Bearer ${this.token}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } +} +exports.BearerCredentialHandler = BearerCredentialHandler; +class PersonalAccessTokenCredentialHandler { + constructor(token) { + this.token = token; + } + // currently implements pre-authorization + // TODO: support preAuth = false where it hooks on 401 + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Basic ${Buffer.from(`PAT:${this.token}`).toString('base64')}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } +} +exports.PersonalAccessTokenCredentialHandler = PersonalAccessTokenCredentialHandler; +//# sourceMappingURL=auth.js.map + +/***/ }), + +/***/ 573: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.toPlatformPath = exports.toWin32Path = exports.toPosixPath = void 0; +const path = __importStar(__webpack_require__(622)); +/** + * toPosixPath converts the given path to the posix form. On Windows, \\ will be + * replaced with /. + * + * @param pth. Path to transform. + * @return string Posix path. + */ +function toPosixPath(pth) { + return pth.replace(/[\\]/g, '/'); +} +exports.toPosixPath = toPosixPath; +/** + * toWin32Path converts the given path to the win32 form. On Linux, / will be + * replaced with \\. + * + * @param pth. Path to transform. + * @return string Win32 path. + */ +function toWin32Path(pth) { + return pth.replace(/[/]/g, '\\'); +} +exports.toWin32Path = toWin32Path; +/** + * toPlatformPath converts the given path to a platform-specific path. It does + * this by replacing instances of / and \ with the platform-specific path + * separator. + * + * @param pth The path to platformize. + * @return string The platform-specific path. + */ +function toPlatformPath(pth) { + return pth.replace(/[/\\]/g, path.sep); +} +exports.toPlatformPath = toPlatformPath; +//# sourceMappingURL=path-utils.js.map + +/***/ }), + +/***/ 597: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +Object.defineProperty(exports, "__esModule", { value: true }); +const pathHelper = __webpack_require__(972); +const internal_match_kind_1 = __webpack_require__(36); +const IS_WINDOWS = process.platform === 'win32'; +/** + * Given an array of patterns, returns an array of paths to search. + * Duplicates and paths under other included paths are filtered out. + */ +function getSearchPaths(patterns) { + // Ignore negate patterns + patterns = patterns.filter(x => !x.negate); + // Create a map of all search paths + const searchPathMap = {}; + for (const pattern of patterns) { const key = IS_WINDOWS ? pattern.searchPath.toUpperCase() : pattern.searchPath; @@ -2201,6 +3866,20 @@ exports.getOptions = getOptions; /***/ }), +/***/ 605: +/***/ (function(module) { + +module.exports = require("http"); + +/***/ }), + +/***/ 614: +/***/ (function(module) { + +module.exports = require("events"); + +/***/ }), + /***/ 621: /***/ (function(module) { @@ -2275,6 +3954,303 @@ module.exports = require("path"); /***/ }), +/***/ 631: +/***/ (function(module) { + +module.exports = require("net"); + +/***/ }), + +/***/ 665: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.summary = exports.markdownSummary = exports.SUMMARY_DOCS_URL = exports.SUMMARY_ENV_VAR = void 0; +const os_1 = __webpack_require__(87); +const fs_1 = __webpack_require__(747); +const { access, appendFile, writeFile } = fs_1.promises; +exports.SUMMARY_ENV_VAR = 'GITHUB_STEP_SUMMARY'; +exports.SUMMARY_DOCS_URL = 'https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary'; +class Summary { + constructor() { + this._buffer = ''; + } + /** + * Finds the summary file path from the environment, rejects if env var is not found or file does not exist + * Also checks r/w permissions. + * + * @returns step summary file path + */ + filePath() { + return __awaiter(this, void 0, void 0, function* () { + if (this._filePath) { + return this._filePath; + } + const pathFromEnv = process.env[exports.SUMMARY_ENV_VAR]; + if (!pathFromEnv) { + throw new Error(`Unable to find environment variable for $${exports.SUMMARY_ENV_VAR}. Check if your runtime environment supports job summaries.`); + } + try { + yield access(pathFromEnv, fs_1.constants.R_OK | fs_1.constants.W_OK); + } + catch (_a) { + throw new Error(`Unable to access summary file: '${pathFromEnv}'. Check if the file has correct read/write permissions.`); + } + this._filePath = pathFromEnv; + return this._filePath; + }); + } + /** + * Wraps content in an HTML tag, adding any HTML attributes + * + * @param {string} tag HTML tag to wrap + * @param {string | null} content content within the tag + * @param {[attribute: string]: string} attrs key-value list of HTML attributes to add + * + * @returns {string} content wrapped in HTML element + */ + wrap(tag, content, attrs = {}) { + const htmlAttrs = Object.entries(attrs) + .map(([key, value]) => ` ${key}="${value}"`) + .join(''); + if (!content) { + return `<${tag}${htmlAttrs}>`; + } + return `<${tag}${htmlAttrs}>${content}`; + } + /** + * Writes text in the buffer to the summary buffer file and empties buffer. Will append by default. + * + * @param {SummaryWriteOptions} [options] (optional) options for write operation + * + * @returns {Promise} summary instance + */ + write(options) { + return __awaiter(this, void 0, void 0, function* () { + const overwrite = !!(options === null || options === void 0 ? void 0 : options.overwrite); + const filePath = yield this.filePath(); + const writeFunc = overwrite ? writeFile : appendFile; + yield writeFunc(filePath, this._buffer, { encoding: 'utf8' }); + return this.emptyBuffer(); + }); + } + /** + * Clears the summary buffer and wipes the summary file + * + * @returns {Summary} summary instance + */ + clear() { + return __awaiter(this, void 0, void 0, function* () { + return this.emptyBuffer().write({ overwrite: true }); + }); + } + /** + * Returns the current summary buffer as a string + * + * @returns {string} string of summary buffer + */ + stringify() { + return this._buffer; + } + /** + * If the summary buffer is empty + * + * @returns {boolen} true if the buffer is empty + */ + isEmptyBuffer() { + return this._buffer.length === 0; + } + /** + * Resets the summary buffer without writing to summary file + * + * @returns {Summary} summary instance + */ + emptyBuffer() { + this._buffer = ''; + return this; + } + /** + * Adds raw text to the summary buffer + * + * @param {string} text content to add + * @param {boolean} [addEOL=false] (optional) append an EOL to the raw text (default: false) + * + * @returns {Summary} summary instance + */ + addRaw(text, addEOL = false) { + this._buffer += text; + return addEOL ? this.addEOL() : this; + } + /** + * Adds the operating system-specific end-of-line marker to the buffer + * + * @returns {Summary} summary instance + */ + addEOL() { + return this.addRaw(os_1.EOL); + } + /** + * Adds an HTML codeblock to the summary buffer + * + * @param {string} code content to render within fenced code block + * @param {string} lang (optional) language to syntax highlight code + * + * @returns {Summary} summary instance + */ + addCodeBlock(code, lang) { + const attrs = Object.assign({}, (lang && { lang })); + const element = this.wrap('pre', this.wrap('code', code), attrs); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML list to the summary buffer + * + * @param {string[]} items list of items to render + * @param {boolean} [ordered=false] (optional) if the rendered list should be ordered or not (default: false) + * + * @returns {Summary} summary instance + */ + addList(items, ordered = false) { + const tag = ordered ? 'ol' : 'ul'; + const listItems = items.map(item => this.wrap('li', item)).join(''); + const element = this.wrap(tag, listItems); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML table to the summary buffer + * + * @param {SummaryTableCell[]} rows table rows + * + * @returns {Summary} summary instance + */ + addTable(rows) { + const tableBody = rows + .map(row => { + const cells = row + .map(cell => { + if (typeof cell === 'string') { + return this.wrap('td', cell); + } + const { header, data, colspan, rowspan } = cell; + const tag = header ? 'th' : 'td'; + const attrs = Object.assign(Object.assign({}, (colspan && { colspan })), (rowspan && { rowspan })); + return this.wrap(tag, data, attrs); + }) + .join(''); + return this.wrap('tr', cells); + }) + .join(''); + const element = this.wrap('table', tableBody); + return this.addRaw(element).addEOL(); + } + /** + * Adds a collapsable HTML details element to the summary buffer + * + * @param {string} label text for the closed state + * @param {string} content collapsable content + * + * @returns {Summary} summary instance + */ + addDetails(label, content) { + const element = this.wrap('details', this.wrap('summary', label) + content); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML image tag to the summary buffer + * + * @param {string} src path to the image you to embed + * @param {string} alt text description of the image + * @param {SummaryImageOptions} options (optional) addition image attributes + * + * @returns {Summary} summary instance + */ + addImage(src, alt, options) { + const { width, height } = options || {}; + const attrs = Object.assign(Object.assign({}, (width && { width })), (height && { height })); + const element = this.wrap('img', null, Object.assign({ src, alt }, attrs)); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML section heading element + * + * @param {string} text heading text + * @param {number | string} [level=1] (optional) the heading level, default: 1 + * + * @returns {Summary} summary instance + */ + addHeading(text, level) { + const tag = `h${level}`; + const allowedTag = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tag) + ? tag + : 'h1'; + const element = this.wrap(allowedTag, text); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML thematic break (
) to the summary buffer + * + * @returns {Summary} summary instance + */ + addSeparator() { + const element = this.wrap('hr', null); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML line break (
) to the summary buffer + * + * @returns {Summary} summary instance + */ + addBreak() { + const element = this.wrap('br', null); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML blockquote to the summary buffer + * + * @param {string} text quote text + * @param {string} cite (optional) citation url + * + * @returns {Summary} summary instance + */ + addQuote(text, cite) { + const attrs = Object.assign({}, (cite && { cite })); + const element = this.wrap('blockquote', text, attrs); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML anchor tag to the summary buffer + * + * @param {string} text link text/content + * @param {string} href hyperlink + * + * @returns {Summary} summary instance + */ + addLink(text, href) { + const element = this.wrap('a', text, { href }); + return this.addRaw(element).addEOL(); + } +} +const _summary = new Summary(); +/** + * @deprecated use `core.summary` + */ +exports.markdownSummary = _summary; +exports.summary = _summary; +//# sourceMappingURL=summary.js.map + +/***/ }), + /***/ 669: /***/ (function(module) { @@ -2282,6 +4258,34 @@ module.exports = require("util"); /***/ }), +/***/ 695: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _validate = _interopRequireDefault(__webpack_require__(78)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function version(uuid) { + if (!(0, _validate.default)(uuid)) { + throw TypeError('Invalid UUID'); + } + + return parseInt(uuid.substr(14, 1), 16); +} + +var _default = version; +exports.default = _default; + +/***/ }), + /***/ 728: /***/ (function(__unusedmodule, exports) { @@ -2299,6 +4303,134 @@ exports.SearchState = SearchState; /***/ }), +/***/ 733: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _rng = _interopRequireDefault(__webpack_require__(844)); + +var _stringify = _interopRequireDefault(__webpack_require__(411)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function v4(options, buf, offset) { + options = options || {}; + + const rnds = options.random || (options.rng || _rng.default)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` + + + rnds[6] = rnds[6] & 0x0f | 0x40; + rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided + + if (buf) { + offset = offset || 0; + + for (let i = 0; i < 16; ++i) { + buf[offset + i] = rnds[i]; + } + + return buf; + } + + return (0, _stringify.default)(rnds); +} + +var _default = v4; +exports.default = _default; + +/***/ }), + +/***/ 742: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.OidcClient = void 0; +const http_client_1 = __webpack_require__(425); +const auth_1 = __webpack_require__(554); +const core_1 = __webpack_require__(470); +class OidcClient { + static createHttpClient(allowRetry = true, maxRetry = 10) { + const requestOptions = { + allowRetries: allowRetry, + maxRetries: maxRetry + }; + return new http_client_1.HttpClient('actions/oidc-client', [new auth_1.BearerCredentialHandler(OidcClient.getRequestToken())], requestOptions); + } + static getRequestToken() { + const token = process.env['ACTIONS_ID_TOKEN_REQUEST_TOKEN']; + if (!token) { + throw new Error('Unable to get ACTIONS_ID_TOKEN_REQUEST_TOKEN env variable'); + } + return token; + } + static getIDTokenUrl() { + const runtimeUrl = process.env['ACTIONS_ID_TOKEN_REQUEST_URL']; + if (!runtimeUrl) { + throw new Error('Unable to get ACTIONS_ID_TOKEN_REQUEST_URL env variable'); + } + return runtimeUrl; + } + static getCall(id_token_url) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + const httpclient = OidcClient.createHttpClient(); + const res = yield httpclient + .getJson(id_token_url) + .catch(error => { + throw new Error(`Failed to get ID Token. \n + Error Code : ${error.statusCode}\n + Error Message: ${error.result.message}`); + }); + const id_token = (_a = res.result) === null || _a === void 0 ? void 0 : _a.value; + if (!id_token) { + throw new Error('Response json body do not have ID Token field'); + } + return id_token; + }); + } + static getIDToken(audience) { + return __awaiter(this, void 0, void 0, function* () { + try { + // New ID Token is requested from action service + let id_token_url = OidcClient.getIDTokenUrl(); + if (audience) { + const encodedAudience = encodeURIComponent(audience); + id_token_url = `${id_token_url}&audience=${encodedAudience}`; + } + core_1.debug(`ID token url is ${id_token_url}`); + const id_token = yield OidcClient.getCall(id_token_url); + core_1.setSecret(id_token); + return id_token; + } + catch (error) { + throw new Error(`Error message: ${error.message}`); + } + }); + } +} +exports.OidcClient = OidcClient; +//# sourceMappingURL=oidc-utils.js.map + +/***/ }), + /***/ 747: /***/ (function(module) { @@ -2306,6 +4438,188 @@ module.exports = require("fs"); /***/ }), +/***/ 794: +/***/ (function(module) { + +module.exports = require("stream"); + +/***/ }), + +/***/ 803: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _crypto = _interopRequireDefault(__webpack_require__(417)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function md5(bytes) { + if (Array.isArray(bytes)) { + bytes = Buffer.from(bytes); + } else if (typeof bytes === 'string') { + bytes = Buffer.from(bytes, 'utf8'); + } + + return _crypto.default.createHash('md5').update(bytes).digest(); +} + +var _default = md5; +exports.default = _default; + +/***/ }), + +/***/ 844: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = rng; + +var _crypto = _interopRequireDefault(__webpack_require__(417)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +const rnds8Pool = new Uint8Array(256); // # of random values to pre-allocate + +let poolPtr = rnds8Pool.length; + +function rng() { + if (poolPtr > rnds8Pool.length - 16) { + _crypto.default.randomFillSync(rnds8Pool); + + poolPtr = 0; + } + + return rnds8Pool.slice(poolPtr, poolPtr += 16); +} + +/***/ }), + +/***/ 893: +/***/ (function(__unusedmodule, exports, __webpack_require__) { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", { + value: true +}); +exports.default = void 0; + +var _rng = _interopRequireDefault(__webpack_require__(844)); + +var _stringify = _interopRequireDefault(__webpack_require__(411)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +// **`v1()` - Generate time-based UUID** +// +// Inspired by https://github.com/LiosK/UUID.js +// and http://docs.python.org/library/uuid.html +let _nodeId; + +let _clockseq; // Previous uuid creation time + + +let _lastMSecs = 0; +let _lastNSecs = 0; // See https://github.com/uuidjs/uuid for API details + +function v1(options, buf, offset) { + let i = buf && offset || 0; + const b = buf || new Array(16); + options = options || {}; + let node = options.node || _nodeId; + let clockseq = options.clockseq !== undefined ? options.clockseq : _clockseq; // node and clockseq need to be initialized to random values if they're not + // specified. We do this lazily to minimize issues related to insufficient + // system entropy. See #189 + + if (node == null || clockseq == null) { + const seedBytes = options.random || (options.rng || _rng.default)(); + + if (node == null) { + // Per 4.5, create and 48-bit node id, (47 random bits + multicast bit = 1) + node = _nodeId = [seedBytes[0] | 0x01, seedBytes[1], seedBytes[2], seedBytes[3], seedBytes[4], seedBytes[5]]; + } + + if (clockseq == null) { + // Per 4.2.2, randomize (14 bit) clockseq + clockseq = _clockseq = (seedBytes[6] << 8 | seedBytes[7]) & 0x3fff; + } + } // UUID timestamps are 100 nano-second units since the Gregorian epoch, + // (1582-10-15 00:00). JSNumbers aren't precise enough for this, so + // time is handled internally as 'msecs' (integer milliseconds) and 'nsecs' + // (100-nanoseconds offset from msecs) since unix epoch, 1970-01-01 00:00. + + + let msecs = options.msecs !== undefined ? options.msecs : Date.now(); // Per 4.2.1.2, use count of uuid's generated during the current clock + // cycle to simulate higher resolution clock + + let nsecs = options.nsecs !== undefined ? options.nsecs : _lastNSecs + 1; // Time since last uuid creation (in msecs) + + const dt = msecs - _lastMSecs + (nsecs - _lastNSecs) / 10000; // Per 4.2.1.2, Bump clockseq on clock regression + + if (dt < 0 && options.clockseq === undefined) { + clockseq = clockseq + 1 & 0x3fff; + } // Reset nsecs if clock regresses (new clockseq) or we've moved onto a new + // time interval + + + if ((dt < 0 || msecs > _lastMSecs) && options.nsecs === undefined) { + nsecs = 0; + } // Per 4.2.1.2 Throw error if too many uuids are requested + + + if (nsecs >= 10000) { + throw new Error("uuid.v1(): Can't create more than 10M uuids/sec"); + } + + _lastMSecs = msecs; + _lastNSecs = nsecs; + _clockseq = clockseq; // Per 4.1.4 - Convert from unix epoch to Gregorian epoch + + msecs += 12219292800000; // `time_low` + + const tl = ((msecs & 0xfffffff) * 10000 + nsecs) % 0x100000000; + b[i++] = tl >>> 24 & 0xff; + b[i++] = tl >>> 16 & 0xff; + b[i++] = tl >>> 8 & 0xff; + b[i++] = tl & 0xff; // `time_mid` + + const tmh = msecs / 0x100000000 * 10000 & 0xfffffff; + b[i++] = tmh >>> 8 & 0xff; + b[i++] = tmh & 0xff; // `time_high_and_version` + + b[i++] = tmh >>> 24 & 0xf | 0x10; // include version + + b[i++] = tmh >>> 16 & 0xff; // `clock_seq_hi_and_reserved` (Per 4.2.2 - include variant) + + b[i++] = clockseq >>> 8 | 0x80; // `clock_seq_low` + + b[i++] = clockseq & 0xff; // `node` + + for (let n = 0; n < 6; ++n) { + b[i + n] = node[n]; + } + + return buf || (0, _stringify.default)(b); +} + +var _default = v1; +exports.default = _default; + +/***/ }), + /***/ 896: /***/ (function(module) { @@ -2337,7 +4651,7 @@ const os = __webpack_require__(87); const path = __webpack_require__(622); const pathHelper = __webpack_require__(972); const minimatch_1 = __webpack_require__(93); -const internal_match_kind_1 = __webpack_require__(327); +const internal_match_kind_1 = __webpack_require__(36); const internal_path_1 = __webpack_require__(383); const IS_WINDOWS = process.platform === 'win32'; class Pattern { From 3c93e8e274ebe79b9812e23848b89af16a4136ef Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 12:14:10 +0200 Subject: [PATCH 017/145] Remove unused imports (#2124) --- src/Runner.Common/ActionCommand.cs | 1 - src/Runner.Common/ActionResult.cs | 2 -- src/Runner.Common/CommandLineParser.cs | 1 - src/Runner.Common/ConfigurationStore.cs | 1 - src/Runner.Common/ExtensionManager.cs | 1 - src/Runner.Common/HostTraceListener.cs | 1 - src/Runner.Common/JobNotification.cs | 5 +---- src/Runner.Common/JobServer.cs | 3 +-- .../Listener/Configuration/ConfigurationManagerL0.cs | 4 +--- src/Test/L0/Listener/RunnerL0.cs | 3 +-- src/Test/L0/PagingLoggerL0.cs | 1 - src/Test/L0/ProcessInvokerL0.cs | 3 +-- src/Test/L0/ServiceControlManagerL0.cs | 3 --- src/Test/L0/Util/ArgUtilL0.cs | 1 - src/Test/L0/Util/UrlUtilL0.cs | 3 +-- src/Test/L0/Util/VssUtilL0.cs | 4 ---- .../L0/Worker/Expressions/ConditionFunctionsL0.cs | 12 +++++------- src/Test/L0/Worker/JobExtensionL0.cs | 3 +-- src/Test/L0/Worker/JobRunnerL0.cs | 4 +--- src/Test/L0/Worker/OutputManagerL0.cs | 1 - src/Test/L0/Worker/StepHostL0.cs | 5 +---- src/Test/L0/Worker/VariablesL0.cs | 4 +--- 22 files changed, 15 insertions(+), 51 deletions(-) diff --git a/src/Runner.Common/ActionCommand.cs b/src/Runner.Common/ActionCommand.cs index 3e25176b840..68ded964c57 100644 --- a/src/Runner.Common/ActionCommand.cs +++ b/src/Runner.Common/ActionCommand.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.Collections.Generic; diff --git a/src/Runner.Common/ActionResult.cs b/src/Runner.Common/ActionResult.cs index db68b685249..bb1622d6780 100644 --- a/src/Runner.Common/ActionResult.cs +++ b/src/Runner.Common/ActionResult.cs @@ -1,5 +1,3 @@ -using System; - namespace GitHub.Runner.Common { public enum ActionResult diff --git a/src/Runner.Common/CommandLineParser.cs b/src/Runner.Common/CommandLineParser.cs index bca3b461ab8..abcd8bd4de9 100644 --- a/src/Runner.Common/CommandLineParser.cs +++ b/src/Runner.Common/CommandLineParser.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using System; using System.Collections.Generic; using GitHub.DistributedTask.Logging; diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index ca526c77fe1..7cce70f601d 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.IO; diff --git a/src/Runner.Common/ExtensionManager.cs b/src/Runner.Common/ExtensionManager.cs index 29f99b62834..c7fbab73bcd 100644 --- a/src/Runner.Common/ExtensionManager.cs +++ b/src/Runner.Common/ExtensionManager.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.Collections.Concurrent; diff --git a/src/Runner.Common/HostTraceListener.cs b/src/Runner.Common/HostTraceListener.cs index e11edc5b492..e022eb0e47b 100644 --- a/src/Runner.Common/HostTraceListener.cs +++ b/src/Runner.Common/HostTraceListener.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.Diagnostics; diff --git a/src/Runner.Common/JobNotification.cs b/src/Runner.Common/JobNotification.cs index ee20fb452f4..c171e4e69e9 100644 --- a/src/Runner.Common/JobNotification.cs +++ b/src/Runner.Common/JobNotification.cs @@ -1,10 +1,7 @@ -using System; -using System.IO; -using System.IO.Pipes; +using System; using System.Net; using System.Net.Sockets; using System.Text; -using System.Threading; using System.Threading.Tasks; namespace GitHub.Runner.Common diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 1cac64f65b6..4a37311ca03 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -13,7 +13,6 @@ using GitHub.Services.Common; using GitHub.Services.WebApi; using GitHub.Services.WebApi.Utilities.Internal; -using Newtonsoft.Json; namespace GitHub.Runner.Common { diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index 197cf7e5202..20ec96a784a 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -1,13 +1,11 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Listener; using GitHub.Runner.Listener.Configuration; using GitHub.Runner.Common.Util; using GitHub.Services.WebApi; using Moq; -using Newtonsoft.Json; using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Runtime.CompilerServices; using System.Security.Cryptography; diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index acaff40eb5f..a1c283a9f0a 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -1,4 +1,4 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Listener; using GitHub.Runner.Listener.Configuration; using Moq; @@ -9,7 +9,6 @@ using Xunit; using GitHub.Services.WebApi; using Pipelines = GitHub.DistributedTask.Pipelines; -using GitHub.Runner.Common.Util; namespace GitHub.Runner.Common.Tests.Listener { diff --git a/src/Test/L0/PagingLoggerL0.cs b/src/Test/L0/PagingLoggerL0.cs index 2c1c3b0ba42..799428aa6eb 100644 --- a/src/Test/L0/PagingLoggerL0.cs +++ b/src/Test/L0/PagingLoggerL0.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using Moq; using System; using System.IO; diff --git a/src/Test/L0/ProcessInvokerL0.cs b/src/Test/L0/ProcessInvokerL0.cs index 629abb0a010..11dcf8d2ae0 100644 --- a/src/Test/L0/ProcessInvokerL0.cs +++ b/src/Test/L0/ProcessInvokerL0.cs @@ -1,11 +1,10 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Threading; using System.Threading.Tasks; using Xunit; -using GitHub.Runner.Common.Util; using System.Threading.Channels; using GitHub.Runner.Sdk; using System.Linq; diff --git a/src/Test/L0/ServiceControlManagerL0.cs b/src/Test/L0/ServiceControlManagerL0.cs index c7fe84cda87..f94ded572c2 100644 --- a/src/Test/L0/ServiceControlManagerL0.cs +++ b/src/Test/L0/ServiceControlManagerL0.cs @@ -1,7 +1,4 @@ -using System; using System.Runtime.CompilerServices; -using System.Text.RegularExpressions; -using GitHub.Runner.Common; using GitHub.Runner.Listener.Configuration; using Xunit; diff --git a/src/Test/L0/Util/ArgUtilL0.cs b/src/Test/L0/Util/ArgUtilL0.cs index 7ede3bed7fd..e0e0b34545d 100644 --- a/src/Test/L0/Util/ArgUtilL0.cs +++ b/src/Test/L0/Util/ArgUtilL0.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using Xunit; diff --git a/src/Test/L0/Util/UrlUtilL0.cs b/src/Test/L0/Util/UrlUtilL0.cs index b84db224256..6100be5dd4f 100644 --- a/src/Test/L0/Util/UrlUtilL0.cs +++ b/src/Test/L0/Util/UrlUtilL0.cs @@ -1,5 +1,4 @@ -using System; -using GitHub.Runner.Common.Util; +using System; using GitHub.Runner.Sdk; using Xunit; diff --git a/src/Test/L0/Util/VssUtilL0.cs b/src/Test/L0/Util/VssUtilL0.cs index 62577eff8d5..30ed8569219 100644 --- a/src/Test/L0/Util/VssUtilL0.cs +++ b/src/Test/L0/Util/VssUtilL0.cs @@ -1,10 +1,6 @@ -using GitHub.Runner.Common.Util; using GitHub.Services.Common; using System; -using System.Collections.Generic; -using System.Net.Http.Headers; using Xunit; -using System.Text.RegularExpressions; using GitHub.Runner.Sdk; namespace GitHub.Runner.Common.Tests.Util diff --git a/src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs b/src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs index 2ca10bffdcc..e4c809c47d9 100644 --- a/src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs +++ b/src/Test/L0/Worker/Expressions/ConditionFunctionsL0.cs @@ -1,10 +1,8 @@ -using System; -using System.Collections.Generic; +using System; using System.Runtime.CompilerServices; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating; using GitHub.DistributedTask.Pipelines.ObjectTemplating; -using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; using GitHub.Runner.Worker.Expressions; using Moq; @@ -121,8 +119,8 @@ public void FailureFunctionComposite(ActionResult jobStatus, ActionResult? actio var executionContext = InitializeExecutionContext(hc); executionContext.Setup(x => x.GetGitHubContext("action_status")).Returns(actionStatus.ToString()); - executionContext.Setup( x=> x.IsEmbedded).Returns(true); - executionContext.Setup( x=> x.Stage).Returns(ActionRunStage.Main); + executionContext.Setup(x => x.IsEmbedded).Returns(true); + executionContext.Setup(x => x.Stage).Returns(ActionRunStage.Main); _jobContext.Status = jobStatus; @@ -181,8 +179,8 @@ public void SuccessFunctionComposite(ActionResult jobStatus, ActionResult? actio var executionContext = InitializeExecutionContext(hc); executionContext.Setup(x => x.GetGitHubContext("action_status")).Returns(actionStatus.ToString()); - executionContext.Setup( x=> x.IsEmbedded).Returns(true); - executionContext.Setup( x=> x.Stage).Returns(ActionRunStage.Main); + executionContext.Setup(x => x.IsEmbedded).Returns(true); + executionContext.Setup(x => x.Stage).Returns(ActionRunStage.Main); _jobContext.Status = jobStatus; diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index 9f7cbd2f51a..e3955c65398 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -1,4 +1,4 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; using Moq; using System; @@ -8,7 +8,6 @@ using System.Threading.Tasks; using Xunit; using System.Threading; -using GitHub.DistributedTask.ObjectTemplating.Tokens; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Common.Tests.Worker diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index 198d378b9b9..1bb7f1ac099 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -1,14 +1,12 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; using Moq; using System; using System.Collections.Generic; -using System.Linq; using System.Runtime.CompilerServices; using System.Threading.Tasks; using Xunit; using System.Threading; -using System.Collections.ObjectModel; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Common.Tests.Worker diff --git a/src/Test/L0/Worker/OutputManagerL0.cs b/src/Test/L0/Worker/OutputManagerL0.cs index 85045802a35..e0364e9f0fc 100644 --- a/src/Test/L0/Worker/OutputManagerL0.cs +++ b/src/Test/L0/Worker/OutputManagerL0.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Globalization; using System.IO; using System.Linq; using System.Runtime.CompilerServices; diff --git a/src/Test/L0/Worker/StepHostL0.cs b/src/Test/L0/Worker/StepHostL0.cs index 0130b84cbad..24f2600dc65 100644 --- a/src/Test/L0/Worker/StepHostL0.cs +++ b/src/Test/L0/Worker/StepHostL0.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Runtime.CompilerServices; using System.Threading.Tasks; @@ -7,9 +7,6 @@ using GitHub.Runner.Worker; using GitHub.Runner.Worker.Handlers; using GitHub.Runner.Worker.Container; -using GitHub.DistributedTask.Pipelines.ContextData; -using System.Linq; -using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; namespace GitHub.Runner.Common.Tests.Worker diff --git a/src/Test/L0/Worker/VariablesL0.cs b/src/Test/L0/Worker/VariablesL0.cs index b0e943aa4c0..5b7d50b4db3 100644 --- a/src/Test/L0/Worker/VariablesL0.cs +++ b/src/Test/L0/Worker/VariablesL0.cs @@ -1,6 +1,4 @@ -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common.Util; -using GitHub.Runner.Sdk; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker; using System.Collections.Generic; using System.Globalization; From bf2caf4f6c60106748107e9e29baf3a88891a14b Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 15 Sep 2022 15:55:45 +0200 Subject: [PATCH 018/145] Remove unused imports (#2126) --- src/Runner.Common/Logging.cs | 1 - src/Runner.Common/ProcessInvoker.cs | 2 +- src/Runner.Common/RunServer.cs | 5 +---- src/Runner.Common/RunnerServer.cs | 3 +-- src/Runner.Common/ThrottlingReportHandler.cs | 3 +-- src/Runner.Common/TraceManager.cs | 1 - src/Runner.Common/Tracing.cs | 2 -- src/Runner.Common/Util/VarUtil.cs | 5 +---- src/Runner.Listener/Checks/GitCheck.cs | 3 +-- src/Runner.Listener/Checks/NodeJsCheck.cs | 3 +-- src/Runner.Listener/CommandSettings.cs | 1 - src/Runner.Listener/Configuration/PromptManager.cs | 10 +++++----- src/Runner.Listener/Configuration/RSAFileKeyManager.cs | 1 - .../Configuration/SystemdControlManager.cs | 3 +-- src/Runner.Listener/Program.cs | 2 -- src/Runner.Listener/Runner.cs | 1 - src/Runner.PluginHost/Program.cs | 5 +---- src/Runner.Plugins/Artifact/PipelinesServer.cs | 2 -- src/Runner.Plugins/Repository/GitCliManager.cs | 2 -- .../Repository/v1.0/GitSourceProvider.cs | 2 -- src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs | 5 +---- .../Repository/v1.1/GitSourceProvider.cs | 2 -- src/Runner.Plugins/Repository/v1.1/RepositoryPlugin.cs | 6 +----- src/Runner.Sdk/ActionPlugin.cs | 4 +--- src/Runner.Sdk/Util/IOUtil.cs | 1 - src/Runner.Sdk/Util/PathUtil.cs | 1 - src/Runner.Sdk/Util/StringUtil.cs | 6 +----- src/Runner.Worker/ActionManifestManager.cs | 2 +- src/Runner.Worker/ActionRunner.cs | 2 +- src/Runner.Worker/ConditionTraceWriter.cs | 1 - src/Runner.Worker/FileCommandManager.cs | 4 +--- src/Runner.Worker/IEnvironmentContextData.cs | 1 - src/Runner.Worker/JobExtension.cs | 4 +--- src/Runner.Worker/JobExtensionRunner.cs | 3 +-- 34 files changed, 23 insertions(+), 76 deletions(-) diff --git a/src/Runner.Common/Logging.cs b/src/Runner.Common/Logging.cs index b4c8d6f3240..40be5cdcff0 100644 --- a/src/Runner.Common/Logging.cs +++ b/src/Runner.Common/Logging.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using System; using System.IO; diff --git a/src/Runner.Common/ProcessInvoker.cs b/src/Runner.Common/ProcessInvoker.cs index 021c6db5e02..7b5fbb7fcf2 100644 --- a/src/Runner.Common/ProcessInvoker.cs +++ b/src/Runner.Common/ProcessInvoker.cs @@ -1,4 +1,4 @@ -using GitHub.Runner.Common.Util; +using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.Collections.Generic; diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 7df8da7a8fa..981a61b7f0c 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -1,11 +1,8 @@ -using System; -using System.Collections.Generic; -using System.Runtime.Serialization; +using System; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Services.Common; using GitHub.Services.WebApi; diff --git a/src/Runner.Common/RunnerServer.cs b/src/Runner.Common/RunnerServer.cs index 0ab7a628503..6f335a4aa3b 100644 --- a/src/Runner.Common/RunnerServer.cs +++ b/src/Runner.Common/RunnerServer.cs @@ -1,9 +1,8 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.WebApi; using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; -using GitHub.Runner.Common.Util; using GitHub.Services.WebApi; using GitHub.Services.Common; using GitHub.Runner.Sdk; diff --git a/src/Runner.Common/ThrottlingReportHandler.cs b/src/Runner.Common/ThrottlingReportHandler.cs index 969df606f3a..7b08ae0db0e 100644 --- a/src/Runner.Common/ThrottlingReportHandler.cs +++ b/src/Runner.Common/ThrottlingReportHandler.cs @@ -1,10 +1,9 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Net.Http; using System.Threading; using System.Threading.Tasks; -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Services.Common.Internal; diff --git a/src/Runner.Common/TraceManager.cs b/src/Runner.Common/TraceManager.cs index 2500eb82a56..ebbaefa89ed 100644 --- a/src/Runner.Common/TraceManager.cs +++ b/src/Runner.Common/TraceManager.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using System; using System.Collections.Concurrent; using System.Diagnostics; diff --git a/src/Runner.Common/Tracing.cs b/src/Runner.Common/Tracing.cs index b72070103e1..a0d93b658bc 100644 --- a/src/Runner.Common/Tracing.cs +++ b/src/Runner.Common/Tracing.cs @@ -1,5 +1,3 @@ - -using GitHub.Runner.Common.Util; using Newtonsoft.Json; using System; using System.Diagnostics; diff --git a/src/Runner.Common/Util/VarUtil.cs b/src/Runner.Common/Util/VarUtil.cs index b9860e151b5..4fcd69d6158 100644 --- a/src/Runner.Common/Util/VarUtil.cs +++ b/src/Runner.Common/Util/VarUtil.cs @@ -1,7 +1,4 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using GitHub.Runner.Sdk; +using System; namespace GitHub.Runner.Common.Util { diff --git a/src/Runner.Listener/Checks/GitCheck.cs b/src/Runner.Listener/Checks/GitCheck.cs index 201210a7b9c..203abd3b4ea 100644 --- a/src/Runner.Listener/Checks/GitCheck.cs +++ b/src/Runner.Listener/Checks/GitCheck.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Threading; using System.Threading.Tasks; using GitHub.Runner.Common; @@ -168,4 +167,4 @@ await processInvoker.ExecuteAsync( return result; } } -} \ No newline at end of file +} diff --git a/src/Runner.Listener/Checks/NodeJsCheck.cs b/src/Runner.Listener/Checks/NodeJsCheck.cs index 931b248fe4a..29fb282e07b 100644 --- a/src/Runner.Listener/Checks/NodeJsCheck.cs +++ b/src/Runner.Listener/Checks/NodeJsCheck.cs @@ -2,7 +2,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Net; using System.Threading; using System.Threading.Tasks; using GitHub.Runner.Common; @@ -179,4 +178,4 @@ await processInvoker.ExecuteAsync( return result; } } -} \ No newline at end of file +} diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index 40c2cc27cd8..ca0ea1b7da5 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -4,7 +4,6 @@ using System.Collections; using System.Collections.Generic; using System.Linq; -using GitHub.DistributedTask.Logging; using GitHub.Runner.Common; using GitHub.Runner.Sdk; diff --git a/src/Runner.Listener/Configuration/PromptManager.cs b/src/Runner.Listener/Configuration/PromptManager.cs index 49d6259b098..f04f17069aa 100644 --- a/src/Runner.Listener/Configuration/PromptManager.cs +++ b/src/Runner.Listener/Configuration/PromptManager.cs @@ -1,5 +1,4 @@ using GitHub.Runner.Common; -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; @@ -72,7 +71,7 @@ public string ReadValue( { return defaultValue; } - else if (isOptional) + else if (isOptional) { return string.Empty; } @@ -87,11 +86,12 @@ public string ReadValue( // Write the message prompt. _terminal.Write($"{description} "); - if(!string.IsNullOrEmpty(defaultValue)) + if (!string.IsNullOrEmpty(defaultValue)) { _terminal.Write($"[press Enter for {defaultValue}] "); } - else if (isOptional){ + else if (isOptional) + { _terminal.Write($"[press Enter to skip] "); } @@ -112,7 +112,7 @@ public string ReadValue( return string.Empty; } } - + // Return the value if it is not empty and it is valid. // Otherwise try the loop again. if (!string.IsNullOrEmpty(value)) diff --git a/src/Runner.Listener/Configuration/RSAFileKeyManager.cs b/src/Runner.Listener/Configuration/RSAFileKeyManager.cs index 3e5ed40a06c..2552a9f2ea6 100644 --- a/src/Runner.Listener/Configuration/RSAFileKeyManager.cs +++ b/src/Runner.Listener/Configuration/RSAFileKeyManager.cs @@ -3,7 +3,6 @@ using System.IO; using System.Security.Cryptography; using System.Threading; -using GitHub.Runner.Common.Util; using GitHub.Runner.Common; using GitHub.Runner.Sdk; diff --git a/src/Runner.Listener/Configuration/SystemdControlManager.cs b/src/Runner.Listener/Configuration/SystemdControlManager.cs index 28eaa75715b..9024416a364 100644 --- a/src/Runner.Listener/Configuration/SystemdControlManager.cs +++ b/src/Runner.Listener/Configuration/SystemdControlManager.cs @@ -1,4 +1,4 @@ -#if OS_LINUX +#if OS_LINUX using System; using System.Collections.Generic; using System.IO; @@ -6,7 +6,6 @@ using System.Text; using GitHub.Runner.Common.Util; using GitHub.Runner.Common; -using GitHub.Runner.Sdk; namespace GitHub.Runner.Listener.Configuration { diff --git a/src/Runner.Listener/Program.cs b/src/Runner.Listener/Program.cs index 537cda1c71e..4d52bed8f68 100644 --- a/src/Runner.Listener/Program.cs +++ b/src/Runner.Listener/Program.cs @@ -1,12 +1,10 @@ using GitHub.Runner.Common; -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.Globalization; using System.IO; using System.Reflection; using System.Runtime.InteropServices; -using System.Threading; using System.Threading.Tasks; namespace GitHub.Runner.Listener diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index dae439595a7..13cc7fe97aa 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -1,7 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.IO.Compression; using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; diff --git a/src/Runner.PluginHost/Program.cs b/src/Runner.PluginHost/Program.cs index d823c6fa1af..2496132586c 100644 --- a/src/Runner.PluginHost/Program.cs +++ b/src/Runner.PluginHost/Program.cs @@ -1,13 +1,10 @@ -using System; -using System.Collections.Generic; +using System; using System.Globalization; using System.IO; -using System.Linq; using System.Reflection; using System.Runtime.Loader; using System.Text; using System.Threading; -using System.Threading.Tasks; using GitHub.Runner.Sdk; using GitHub.DistributedTask.WebApi; diff --git a/src/Runner.Plugins/Artifact/PipelinesServer.cs b/src/Runner.Plugins/Artifact/PipelinesServer.cs index 6f712404551..8b51dbd72c5 100644 --- a/src/Runner.Plugins/Artifact/PipelinesServer.cs +++ b/src/Runner.Plugins/Artifact/PipelinesServer.cs @@ -1,5 +1,3 @@ -using System; -using System.IO; using System.Threading; using System.Threading.Tasks; using GitHub.Actions.Pipelines.WebApi; diff --git a/src/Runner.Plugins/Repository/GitCliManager.cs b/src/Runner.Plugins/Repository/GitCliManager.cs index 0ed051645a7..338e17a6e76 100644 --- a/src/Runner.Plugins/Repository/GitCliManager.cs +++ b/src/Runner.Plugins/Repository/GitCliManager.cs @@ -1,6 +1,5 @@ using System; using System.Collections.Generic; -using System.Diagnostics; using System.Linq; using System.Text; using System.Text.RegularExpressions; @@ -9,7 +8,6 @@ using System.IO; using GitHub.Runner.Sdk; using GitHub.Services.Common; -using GitHub.DistributedTask.Pipelines.ContextData; namespace GitHub.Runner.Plugins.Repository { diff --git a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs index b762cafcbba..f567a1338b8 100644 --- a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs +++ b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs @@ -6,11 +6,9 @@ using System.IO; using System.Text.RegularExpressions; using System.Text; -using System.Diagnostics; using GitHub.Runner.Sdk; using System.Linq; using GitHub.DistributedTask.WebApi; -using GitHub.Services.WebApi; namespace GitHub.Runner.Plugins.Repository.v1_0 { diff --git a/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs b/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs index a216ac306ad..df8bd33639c 100644 --- a/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs +++ b/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs @@ -1,12 +1,9 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using System; using System.Threading; using System.Threading.Tasks; using GitHub.Runner.Sdk; using Pipelines = GitHub.DistributedTask.Pipelines; using System.IO; -using GitHub.DistributedTask.Pipelines.ContextData; using System.Text.RegularExpressions; using GitHub.DistributedTask.Pipelines.Expressions; using System.Text; diff --git a/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs b/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs index 539b2b763f1..b2f50f45813 100644 --- a/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs +++ b/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs @@ -4,9 +4,7 @@ using System.Threading; using System.Threading.Tasks; using System.IO; -using System.Text.RegularExpressions; using System.Text; -using System.Diagnostics; using GitHub.Runner.Sdk; using System.Linq; using GitHub.DistributedTask.WebApi; diff --git a/src/Runner.Plugins/Repository/v1.1/RepositoryPlugin.cs b/src/Runner.Plugins/Repository/v1.1/RepositoryPlugin.cs index 3d6c87422a5..04c62b30194 100644 --- a/src/Runner.Plugins/Repository/v1.1/RepositoryPlugin.cs +++ b/src/Runner.Plugins/Repository/v1.1/RepositoryPlugin.cs @@ -1,13 +1,9 @@ -using System; -using System.Collections.Generic; -using System.Linq; +using System; using System.Threading; using System.Threading.Tasks; using GitHub.Runner.Sdk; using Pipelines = GitHub.DistributedTask.Pipelines; using System.IO; -using GitHub.DistributedTask.Pipelines.ContextData; -using System.Text.RegularExpressions; using GitHub.DistributedTask.Pipelines.Expressions; using System.Text; diff --git a/src/Runner.Sdk/ActionPlugin.cs b/src/Runner.Sdk/ActionPlugin.cs index 2a9d87a0a06..0a0f33ab457 100644 --- a/src/Runner.Sdk/ActionPlugin.cs +++ b/src/Runner.Sdk/ActionPlugin.cs @@ -1,7 +1,6 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; -using System.Net.Http; using System.Net.Http.Headers; using System.Runtime.InteropServices; using System.Threading; @@ -11,7 +10,6 @@ using GitHub.Services.Common; using GitHub.Services.WebApi; using Newtonsoft.Json; -using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Sdk { diff --git a/src/Runner.Sdk/Util/IOUtil.cs b/src/Runner.Sdk/Util/IOUtil.cs index 1c8a0cd506c..94a674da3b4 100644 --- a/src/Runner.Sdk/Util/IOUtil.cs +++ b/src/Runner.Sdk/Util/IOUtil.cs @@ -3,7 +3,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection; using System.Security.Cryptography; using System.Text; using System.Threading; diff --git a/src/Runner.Sdk/Util/PathUtil.cs b/src/Runner.Sdk/Util/PathUtil.cs index ac1704b5c8b..7a7575d20f5 100644 --- a/src/Runner.Sdk/Util/PathUtil.cs +++ b/src/Runner.Sdk/Util/PathUtil.cs @@ -1,4 +1,3 @@ -using System; using System.IO; namespace GitHub.Runner.Sdk diff --git a/src/Runner.Sdk/Util/StringUtil.cs b/src/Runner.Sdk/Util/StringUtil.cs index f5111ec89d4..e55fd7f4624 100644 --- a/src/Runner.Sdk/Util/StringUtil.cs +++ b/src/Runner.Sdk/Util/StringUtil.cs @@ -1,11 +1,7 @@ -using GitHub.Services.WebApi; +using GitHub.Services.WebApi; using Newtonsoft.Json; -using Newtonsoft.Json.Linq; using System; -using System.Collections.Generic; using System.Globalization; -using System.IO; -using System.Reflection; using System.Text; namespace GitHub.Runner.Sdk diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index e446ea34c25..00cc36f6af9 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Threading; diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index be1f45723f0..257d6010942 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating; diff --git a/src/Runner.Worker/ConditionTraceWriter.cs b/src/Runner.Worker/ConditionTraceWriter.cs index 951e4b4b219..1f98987ccd9 100644 --- a/src/Runner.Worker/ConditionTraceWriter.cs +++ b/src/Runner.Worker/ConditionTraceWriter.cs @@ -1,6 +1,5 @@ using System; using System.Text; -using GitHub.DistributedTask.Pipelines; using GitHub.Runner.Common; using GitHub.Runner.Sdk; using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index 145291ccd4f..37aacfbe60a 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -1,9 +1,7 @@ -using GitHub.DistributedTask.WebApi; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker.Container; using GitHub.Runner.Common; -using GitHub.Runner.Sdk; using System; -using System.Collections; using System.Collections.Generic; using System.IO; using System.Text; diff --git a/src/Runner.Worker/IEnvironmentContextData.cs b/src/Runner.Worker/IEnvironmentContextData.cs index b07bb169a9f..52b43e52be1 100644 --- a/src/Runner.Worker/IEnvironmentContextData.cs +++ b/src/Runner.Worker/IEnvironmentContextData.cs @@ -1,4 +1,3 @@ -using System; using System.Collections.Generic; public interface IEnvironmentContextData diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 09bbdd479d2..14004ecaf25 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Diagnostics; using System.Globalization; @@ -7,7 +7,6 @@ using System.Runtime.Serialization; using System.Threading; using System.Threading.Tasks; -using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; @@ -16,7 +15,6 @@ using GitHub.Runner.Common; using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; -using GitHub.Runner.Worker; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker diff --git a/src/Runner.Worker/JobExtensionRunner.cs b/src/Runner.Worker/JobExtensionRunner.cs index 48c2c2358e3..62fc3287fa3 100644 --- a/src/Runner.Worker/JobExtensionRunner.cs +++ b/src/Runner.Worker/JobExtensionRunner.cs @@ -1,6 +1,5 @@ -using System; +using System; using System.Threading.Tasks; -using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; namespace GitHub.Runner.Worker From ef8719ede2c39466a2e653503192f2813a13ac5a Mon Sep 17 00:00:00 2001 From: Thomas Boop <52323235+thboop@users.noreply.github.com> Date: Mon, 19 Sep 2022 11:28:46 -0400 Subject: [PATCH 019/145] Add Release branches to pull request spec (#2134) --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 67aeb533cbb..59d693aa68e 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -10,7 +10,7 @@ on: - '**.md' pull_request: branches: - - '*' + - '**' paths-ignore: - '**.md' From 9176801d983b3d1be360676eac8925ae8f0370c6 Mon Sep 17 00:00:00 2001 From: Francesco Renzi Date: Mon, 26 Sep 2022 10:17:46 +0100 Subject: [PATCH 020/145] Add file commands for save-state and set-output (#2118) --- src/Runner.Common/ExtensionManager.cs | 2 + src/Runner.Worker/FileCommandManager.cs | 308 ++++++++----- src/Runner.Worker/GitHubContext.cs | 2 + src/Test/L0/Worker/SaveStateFileCommandL0.cs | 438 ++++++++++++++++++ src/Test/L0/Worker/SetOutputFileCommandL0.cs | 444 +++++++++++++++++++ 5 files changed, 1085 insertions(+), 109 deletions(-) create mode 100644 src/Test/L0/Worker/SaveStateFileCommandL0.cs create mode 100644 src/Test/L0/Worker/SetOutputFileCommandL0.cs diff --git a/src/Runner.Common/ExtensionManager.cs b/src/Runner.Common/ExtensionManager.cs index c7fbab73bcd..6f1bfc5d7b6 100644 --- a/src/Runner.Common/ExtensionManager.cs +++ b/src/Runner.Common/ExtensionManager.cs @@ -60,6 +60,8 @@ private List LoadExtensions() where T : class, IExtension Add(extensions, "GitHub.Runner.Worker.AddPathFileCommand, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.SetEnvFileCommand, Runner.Worker"); Add(extensions, "GitHub.Runner.Worker.CreateStepSummaryCommand, Runner.Worker"); + Add(extensions, "GitHub.Runner.Worker.SaveStateFileCommand, Runner.Worker"); + Add(extensions, "GitHub.Runner.Worker.SetOutputFileCommand, Runner.Worker"); break; case "GitHub.Runner.Listener.Check.ICheckExtension": Add(extensions, "GitHub.Runner.Listener.Check.InternetCheck, Runner.Listener"); diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index 37aacfbe60a..66362f6812a 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -138,74 +138,10 @@ public sealed class SetEnvFileCommand : RunnerService, IFileCommandExtension public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) { - try - { - var text = File.ReadAllText(filePath) ?? string.Empty; - var index = 0; - var line = ReadLine(text, ref index); - while (line != null) - { - if (!string.IsNullOrEmpty(line)) - { - var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); - var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); - - // Normal style NAME=VALUE - if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) - { - var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(line)) - { - throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty"); - } - SetEnvironmentVariable(context, split[0], split[1]); - } - // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) - { - var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); - if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) - { - throw new Exception($"Invalid environment variable format '{line}'. Environment variable name must not be empty and delimiter must not be empty"); - } - var name = split[0]; - var delimiter = split[1]; - var startIndex = index; // Start index of the value (inclusive) - var endIndex = index; // End index of the value (exclusive) - var tempLine = ReadLine(text, ref index, out var newline); - while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) - { - if (tempLine == null) - { - throw new Exception($"Invalid environment variable value. Matching delimiter not found '{delimiter}'"); - } - if (newline == null) - { - throw new Exception($"Invalid environment variable value. EOF marker missing new line."); - } - endIndex = index - newline.Length; - tempLine = ReadLine(text, ref index, out newline); - } - - var value = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; - SetEnvironmentVariable(context, name, value); - } - else - { - throw new Exception($"Invalid environment variable format '{line}'"); - } - } - - line = ReadLine(text, ref index); - } - } - catch (DirectoryNotFoundException) + var pairs = new EnvFileKeyValuePairs(context, filePath); + foreach (var pair in pairs) { - context.Debug($"Environment variables file does not exist '{filePath}'"); - } - catch (FileNotFoundException) - { - context.Debug($"Environment variables file does not exist '{filePath}'"); + SetEnvironmentVariable(context, pair.Key, pair.Value); } } @@ -218,48 +154,6 @@ private static void SetEnvironmentVariable( context.SetEnvContext(name, value); context.Debug($"{name}='{value}'"); } - - private static string ReadLine( - string text, - ref int index) - { - return ReadLine(text, ref index, out _); - } - - private static string ReadLine( - string text, - ref int index, - out string newline) - { - if (index >= text.Length) - { - newline = null; - return null; - } - - var originalIndex = index; - var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); - if (lfIndex < 0) - { - index = text.Length; - newline = null; - return text.Substring(originalIndex); - } - -#if OS_WINDOWS - var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); - if (crLFIndex >= 0 && crLFIndex < lfIndex) - { - index = crLFIndex + 2; // Skip over CRLF - newline = "\r\n"; - return text.Substring(originalIndex, crLFIndex - originalIndex); - } -#endif - - index = lfIndex + 1; // Skip over LF - newline = "\n"; - return text.Substring(originalIndex, lfIndex - originalIndex); - } } public sealed class CreateStepSummaryCommand : RunnerService, IFileCommandExtension @@ -325,4 +219,200 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container } } } + + public sealed class SaveStateFileCommand : RunnerService, IFileCommandExtension + { + public string ContextName => "state"; + public string FilePrefix => "save_state_"; + + public Type ExtensionType => typeof(IFileCommandExtension); + + public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) + { + var pairs = new EnvFileKeyValuePairs(context, filePath); + foreach (var pair in pairs) + { + // Embedded steps (composite) keep track of the state at the root level + if (context.IsEmbedded) + { + var id = context.EmbeddedId; + if (!context.Root.EmbeddedIntraActionState.ContainsKey(id)) + { + context.Root.EmbeddedIntraActionState[id] = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + context.Root.EmbeddedIntraActionState[id][pair.Key] = pair.Value; + } + // Otherwise modify the ExecutionContext + else + { + context.IntraActionState[pair.Key] = pair.Value; + } + + context.Debug($"Save intra-action state {pair.Key} = {pair.Value}"); + } + } + } + + public sealed class SetOutputFileCommand : RunnerService, IFileCommandExtension + { + public string ContextName => "output"; + public string FilePrefix => "set_output_"; + + public Type ExtensionType => typeof(IFileCommandExtension); + + public void ProcessCommand(IExecutionContext context, string filePath, ContainerInfo container) + { + var pairs = new EnvFileKeyValuePairs(context, filePath); + foreach (var pair in pairs) + { + context.SetOutput(pair.Key, pair.Value, out var reference); + context.Debug($"Set output {pair.Key} = {pair.Value}"); + } + } + } + + public sealed class EnvFileKeyValuePairs: IEnumerable> + { + private IExecutionContext _context; + private string _filePath; + + public EnvFileKeyValuePairs(IExecutionContext context, string filePath) + { + _context = context; + _filePath = filePath; + } + + public IEnumerator> GetEnumerator() + { + var text = string.Empty; + try + { + text = File.ReadAllText(_filePath) ?? string.Empty; + } + catch (DirectoryNotFoundException) + { + _context.Debug($"File does not exist '{_filePath}'"); + yield break; + } + catch (FileNotFoundException) + { + _context.Debug($"File does not exist '{_filePath}'"); + yield break; + } + + var index = 0; + var line = ReadLine(text, ref index); + while (line != null) + { + if (!string.IsNullOrEmpty(line)) + { + var key = string.Empty; + var output = string.Empty; + + var equalsIndex = line.IndexOf("=", StringComparison.Ordinal); + var heredocIndex = line.IndexOf("<<", StringComparison.Ordinal); + + // Normal style NAME=VALUE + if (equalsIndex >= 0 && (heredocIndex < 0 || equalsIndex < heredocIndex)) + { + var split = line.Split(new[] { '=' }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(line)) + { + throw new Exception($"Invalid format '{line}'. Name must not be empty"); + } + + key = split[0]; + output = split[1]; + } + + // Heredoc style NAME<= 0 && (equalsIndex < 0 || heredocIndex < equalsIndex)) + { + var split = line.Split(new[] { "<<" }, 2, StringSplitOptions.None); + if (string.IsNullOrEmpty(split[0]) || string.IsNullOrEmpty(split[1])) + { + throw new Exception($"Invalid format '{line}'. Name must not be empty and delimiter must not be empty"); + } + key = split[0]; + var delimiter = split[1]; + var startIndex = index; // Start index of the value (inclusive) + var endIndex = index; // End index of the value (exclusive) + var tempLine = ReadLine(text, ref index, out var newline); + while (!string.Equals(tempLine, delimiter, StringComparison.Ordinal)) + { + if (tempLine == null) + { + throw new Exception($"Invalid value. Matching delimiter not found '{delimiter}'"); + } + if (newline == null) + { + throw new Exception($"Invalid value. EOF marker missing new line."); + } + endIndex = index - newline.Length; + tempLine = ReadLine(text, ref index, out newline); + } + + output = endIndex > startIndex ? text.Substring(startIndex, endIndex - startIndex) : string.Empty; + } + else + { + throw new Exception($"Invalid format '{line}'"); + } + + yield return new KeyValuePair(key, output); + } + + line = ReadLine(text, ref index); + } + } + + System.Collections.IEnumerator + System.Collections.IEnumerable.GetEnumerator() + { + // Invoke IEnumerator> GetEnumerator() above. + return GetEnumerator(); + } + + private static string ReadLine( + string text, + ref int index) + { + return ReadLine(text, ref index, out _); + } + + private static string ReadLine( + string text, + ref int index, + out string newline) + { + if (index >= text.Length) + { + newline = null; + return null; + } + + var originalIndex = index; + var lfIndex = text.IndexOf("\n", index, StringComparison.Ordinal); + if (lfIndex < 0) + { + index = text.Length; + newline = null; + return text.Substring(originalIndex); + } + +#if OS_WINDOWS + var crLFIndex = text.IndexOf("\r\n", index, StringComparison.Ordinal); + if (crLFIndex >= 0 && crLFIndex < lfIndex) + { + index = crLFIndex + 2; // Skip over CRLF + newline = "\r\n"; + return text.Substring(originalIndex, crLFIndex - originalIndex); + } +#endif + + index = lfIndex + 1; // Skip over LF + newline = "\n"; + return text.Substring(originalIndex, lfIndex - originalIndex); + } + } } diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index cdefd45b405..f320705eba0 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -21,6 +21,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "graphql_url", "head_ref", "job", + "output", "path", "ref_name", "ref_protected", @@ -34,6 +35,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "run_number", "server_url", "sha", + "state", "step_summary", "triggering_actor", "workflow", diff --git a/src/Test/L0/Worker/SaveStateFileCommandL0.cs b/src/Test/L0/Worker/SaveStateFileCommandL0.cs new file mode 100644 index 00000000000..45296c70978 --- /dev/null +++ b/src/Test/L0/Worker/SaveStateFileCommandL0.cs @@ -0,0 +1,438 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; +using Moq; +using Xunit; +using DTWebApi = GitHub.DistributedTask.WebApi; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class SaveStateFileCommandL0 + { + private Mock _executionContext; + private List> _issues; + private string _rootDirectory; + private SaveStateFileCommand _saveStateFileCommand; + private Dictionary _intraActionState; + private ITraceWriter _trace; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_DirectoryNotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "directory-not-found", "env"); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _intraActionState.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_NotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "file-not-found"); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _intraActionState.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_EmptyFile() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "empty-file"); + var content = new List(); + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _intraActionState.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_STATE=MY VALUE", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _intraActionState.Count); + Assert.Equal("MY VALUE", _intraActionState["MY_STATE"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_SkipEmptyLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + string.Empty, + "MY_STATE=my value", + string.Empty, + "MY_STATE_2=my second value", + string.Empty, + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(2, _intraActionState.Count); + Assert.Equal("my value", _intraActionState["MY_STATE"]); + Assert.Equal("my second value", _intraActionState["MY_STATE_2"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_EmptyValue() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple-empty-value"); + var content = new List + { + "MY_STATE=", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _intraActionState.Count); + Assert.Equal(string.Empty, _intraActionState["MY_STATE"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_MultipleValues() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_STATE=my value", + "MY_STATE_2=", + "MY_STATE_3=my third value", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _intraActionState.Count); + Assert.Equal("my value", _intraActionState["MY_STATE"]); + Assert.Equal(string.Empty, _intraActionState["MY_STATE_2"]); + Assert.Equal("my third value", _intraActionState["MY_STATE_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Simple_SpecialCharacters() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_STATE==abc", + "MY_STATE_2=def=ghi", + "MY_STATE_3=jkl=", + }; + WriteContent(stateFile, content); + _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _intraActionState.Count); + Assert.Equal("=abc", _intraActionState["MY_STATE"]); + Assert.Equal("def=ghi", _intraActionState["MY_STATE_2"]); + Assert.Equal("jkl=", _intraActionState["MY_STATE_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Heredoc() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_STATE< + { + "MY_STATE< + { + string.Empty, + "MY_STATE< + { + "MY_STATE<<=EOF", + "hello", + "one", + "=EOF", + "MY_STATE_2<< + { + "MY_STATE<(() => _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("Matching delimiter not found", ex.Message); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Heredoc_MissingNewLineMultipleLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_STATE<(() => _saveStateFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("EOF marker missing new line", ex.Message); + } + } + +#if OS_WINDOWS + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SaveStateFileCommand_Heredoc_PreservesNewline() + { + using (var hostContext = Setup()) + { + var newline = "\n"; + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_STATE< content, + string newline = null) + { + if (string.IsNullOrEmpty(newline)) + { + newline = Environment.NewLine; + } + + var encoding = new UTF8Encoding(true); // Emit BOM + var contentStr = string.Join(newline, content); + File.WriteAllText(path, contentStr, encoding); + } + + private TestHostContext Setup([CallerMemberName] string name = "") + { + _issues = new List>(); + _intraActionState = new Dictionary(); + + var hostContext = new TestHostContext(this, name); + + // Trace + _trace = hostContext.GetTrace(); + + // Directory for test data + var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); + ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); + Directory.CreateDirectory(workDirectory); + _rootDirectory = Path.Combine(workDirectory, nameof(SaveStateFileCommandL0)); + Directory.CreateDirectory(_rootDirectory); + + // Execution context + _executionContext = new Mock(); + _executionContext.Setup(x => x.Global) + .Returns(new GlobalContext + { + EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), + WriteDebug = true, + }); + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, string logMessage) => + { + _issues.Add(new Tuple(issue, logMessage)); + var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; + _trace.Info($"Issue '{issue.Type}': {message}"); + }); + _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + _trace.Info($"{tag}{message}"); + }); + _executionContext.Setup(x => x.IntraActionState) + .Returns(_intraActionState); + + // SaveStateFileCommand + _saveStateFileCommand = new SaveStateFileCommand(); + _saveStateFileCommand.Initialize(hostContext); + + return hostContext; + } + } +} diff --git a/src/Test/L0/Worker/SetOutputFileCommandL0.cs b/src/Test/L0/Worker/SetOutputFileCommandL0.cs new file mode 100644 index 00000000000..8af9695db46 --- /dev/null +++ b/src/Test/L0/Worker/SetOutputFileCommandL0.cs @@ -0,0 +1,444 @@ +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Runtime.CompilerServices; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using GitHub.Runner.Worker.Handlers; +using Moq; +using Xunit; +using DTWebApi = GitHub.DistributedTask.WebApi; + +namespace GitHub.Runner.Common.Tests.Worker +{ + public sealed class SetOutputFileCommandL0 + { + private Mock _executionContext; + private List> _issues; + private Dictionary _outputs; + private string _rootDirectory; + private SetOutputFileCommand _setOutputFileCommand; + private ITraceWriter _trace; + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_DirectoryNotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "directory-not-found", "env"); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _outputs.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_NotFound() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "file-not-found"); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _outputs.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_EmptyFile() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "empty-file"); + var content = new List(); + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(0, _outputs.Count); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_OUTPUT=MY VALUE", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _outputs.Count); + Assert.Equal("MY VALUE", _outputs["MY_OUTPUT"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_SkipEmptyLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + string.Empty, + "MY_OUTPUT=my value", + string.Empty, + "MY_OUTPUT_2=my second value", + string.Empty, + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(2, _outputs.Count); + Assert.Equal("my value", _outputs["MY_OUTPUT"]); + Assert.Equal("my second value", _outputs["MY_OUTPUT_2"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_EmptyValue() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple-empty-value"); + var content = new List + { + "MY_OUTPUT=", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(1, _outputs.Count); + Assert.Equal(string.Empty, _outputs["MY_OUTPUT"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_MultipleValues() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_OUTPUT=my value", + "MY_OUTPUT_2=", + "MY_OUTPUT_3=my third value", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _outputs.Count); + Assert.Equal("my value", _outputs["MY_OUTPUT"]); + Assert.Equal(string.Empty, _outputs["MY_OUTPUT_2"]); + Assert.Equal("my third value", _outputs["MY_OUTPUT_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Simple_SpecialCharacters() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "simple"); + var content = new List + { + "MY_OUTPUT==abc", + "MY_OUTPUT_2=def=ghi", + "MY_OUTPUT_3=jkl=", + }; + WriteContent(stateFile, content); + _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null); + Assert.Equal(0, _issues.Count); + Assert.Equal(3, _outputs.Count); + Assert.Equal("=abc", _outputs["MY_OUTPUT"]); + Assert.Equal("def=ghi", _outputs["MY_OUTPUT_2"]); + Assert.Equal("jkl=", _outputs["MY_OUTPUT_3"]); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Heredoc() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_OUTPUT< + { + "MY_OUTPUT< + { + string.Empty, + "MY_OUTPUT< + { + "MY_OUTPUT<<=EOF", + "hello", + "one", + "=EOF", + "MY_OUTPUT_2<< + { + "MY_OUTPUT<(() => _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("Matching delimiter not found", ex.Message); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Heredoc_MissingNewLineMultipleLines() + { + using (var hostContext = Setup()) + { + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_OUTPUT<(() => _setOutputFileCommand.ProcessCommand(_executionContext.Object, stateFile, null)); + Assert.Contains("EOF marker missing new line", ex.Message); + } + } + +#if OS_WINDOWS + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void SetOutputFileCommand_Heredoc_PreservesNewline() + { + using (var hostContext = Setup()) + { + var newline = "\n"; + var stateFile = Path.Combine(_rootDirectory, "heredoc"); + var content = new List + { + "MY_OUTPUT< content, + string newline = null) + { + if (string.IsNullOrEmpty(newline)) + { + newline = Environment.NewLine; + } + + var encoding = new UTF8Encoding(true); // Emit BOM + var contentStr = string.Join(newline, content); + File.WriteAllText(path, contentStr, encoding); + } + + private TestHostContext Setup([CallerMemberName] string name = "") + { + _issues = new List>(); + _outputs = new Dictionary(); + + var hostContext = new TestHostContext(this, name); + + // Trace + _trace = hostContext.GetTrace(); + + // Directory for test data + var workDirectory = hostContext.GetDirectory(WellKnownDirectory.Work); + ArgUtil.NotNullOrEmpty(workDirectory, nameof(workDirectory)); + Directory.CreateDirectory(workDirectory); + _rootDirectory = Path.Combine(workDirectory, nameof(SetOutputFileCommandL0)); + Directory.CreateDirectory(_rootDirectory); + + // Execution context + _executionContext = new Mock(); + _executionContext.Setup(x => x.Global) + .Returns(new GlobalContext + { + EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), + WriteDebug = true, + }); + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, string logMessage) => + { + _issues.Add(new Tuple(issue, logMessage)); + var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; + _trace.Info($"Issue '{issue.Type}': {message}"); + }); + _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) + .Callback((string tag, string message) => + { + _trace.Info($"{tag}{message}"); + }); + + var reference = string.Empty; + _executionContext.Setup(x => x.SetOutput(It.IsAny(), It.IsAny(), out reference)) + .Callback((string name, string value, out string reference) => + { + reference = value; + _outputs[name] = value; + }); + + // SetOutputFileCommand + _setOutputFileCommand = new SetOutputFileCommand(); + _setOutputFileCommand.Initialize(hostContext); + + return hostContext; + } + } +} From 36c5a27d34198cb512fa101dbc65b34d55321293 Mon Sep 17 00:00:00 2001 From: Thomas Boop <52323235+thboop@users.noreply.github.com> Date: Mon, 26 Sep 2022 09:20:43 -0400 Subject: [PATCH 021/145] POC: Windows arm64 runner build (#2022) Prerelease for windows-arm64 runner build --- .github/workflows/build.yml | 10 ++- .github/workflows/release.yml | 77 +++++++++++++++++-- docs/contribute.md | 2 + releaseNote.md | 20 +++++ src/Directory.Build.props | 3 + src/Misc/contentHash/dotnetRuntime/win-arm64 | 1 + src/Misc/contentHash/externals/win-arm64 | 1 + src/Misc/externals.sh | 11 +++ src/Misc/runnerdotnetruntimeassets | 4 +- src/Runner.Common/Constants.cs | 15 +++- src/Runner.Common/Runner.Common.csproj | 2 +- src/Runner.Common/Util/NodeUtil.cs | 2 +- src/Runner.Listener/Program.cs | 12 +++ src/Runner.Listener/Runner.Listener.csproj | 2 +- .../Runner.PluginHost.csproj | 2 +- src/Runner.Plugins/Runner.Plugins.csproj | 2 +- src/Runner.Sdk/Runner.Sdk.csproj | 4 +- src/Runner.Service/Windows/App.config | 4 +- src/Runner.Service/Windows/AppARM.config | 6 ++ .../Windows/RunnerService.csproj | 14 +++- .../Handlers/NodeScriptActionHandler.cs | 10 ++- src/Runner.Worker/Runner.Worker.csproj | 2 +- src/Sdk/Sdk.csproj | 2 +- src/Test/L0/ConstantGenerationL0.cs | 1 + src/Test/L0/Listener/SelfUpdaterL0.cs | 2 +- src/Test/Test.csproj | 4 +- src/dev.sh | 35 +++++---- src/dir.proj | 12 +-- 28 files changed, 210 insertions(+), 52 deletions(-) create mode 100644 src/Misc/contentHash/dotnetRuntime/win-arm64 create mode 100644 src/Misc/contentHash/externals/win-arm64 create mode 100644 src/Runner.Service/Windows/AppARM.config diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 59d693aa68e..fd13c5b1260 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -18,7 +18,7 @@ jobs: build: strategy: matrix: - runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, osx-x64, osx-arm64 ] + runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, win-arm64, osx-x64, osx-arm64 ] include: - runtime: linux-x64 os: ubuntu-latest @@ -44,6 +44,10 @@ jobs: os: windows-2019 devScript: ./dev + - runtime: win-arm64 + os: windows-latest + devScript: ./dev + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -82,7 +86,7 @@ jobs: run: | ${{ matrix.devScript }} test working-directory: src - if: matrix.runtime != 'linux-arm64' && matrix.runtime != 'linux-arm' && matrix.runtime != 'osx-arm64' + if: matrix.runtime != 'linux-arm64' && matrix.runtime != 'linux-arm' && matrix.runtime != 'osx-arm64' && matrix.runtime != 'win-arm64' # Create runner package tar.gz/zip - name: Package Release @@ -97,7 +101,7 @@ jobs: uses: actions/upload-artifact@v2 with: name: runner-package-${{ matrix.runtime }} - path: | + path: | _package _package_trims/trim_externals _package_trims/trim_runtime diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index a3ec84623ce..20927e2176f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -50,29 +50,33 @@ jobs: linux-arm64-sha: ${{ steps.sha.outputs.linux-arm64-sha256 }} linux-arm-sha: ${{ steps.sha.outputs.linux-arm-sha256 }} win-x64-sha: ${{ steps.sha.outputs.win-x64-sha256 }} + win-arm64-sha: ${{ steps.sha.outputs.win-arm64-sha256 }} osx-x64-sha: ${{ steps.sha.outputs.osx-x64-sha256 }} osx-arm64-sha: ${{ steps.sha.outputs.osx-arm64-sha256 }} linux-x64-sha-noexternals: ${{ steps.sha_noexternals.outputs.linux-x64-sha256 }} linux-arm64-sha-noexternals: ${{ steps.sha_noexternals.outputs.linux-arm64-sha256 }} linux-arm-sha-noexternals: ${{ steps.sha_noexternals.outputs.linux-arm-sha256 }} win-x64-sha-noexternals: ${{ steps.sha_noexternals.outputs.win-x64-sha256 }} + win-arm64-sha-noexternals: ${{ steps.sha_noexternals.outputs.win-arm64-sha256 }} osx-x64-sha-noexternals: ${{ steps.sha_noexternals.outputs.osx-x64-sha256 }} osx-arm64-sha-noexternals: ${{ steps.sha_noexternals.outputs.osx-arm64-sha256 }} linux-x64-sha-noruntime: ${{ steps.sha_noruntime.outputs.linux-x64-sha256 }} linux-arm64-sha-noruntime: ${{ steps.sha_noruntime.outputs.linux-arm64-sha256 }} linux-arm-sha-noruntime: ${{ steps.sha_noruntime.outputs.linux-arm-sha256 }} win-x64-sha-noruntime: ${{ steps.sha_noruntime.outputs.win-x64-sha256 }} + win-arm64-sha-noruntime: ${{ steps.sha_noruntime.outputs.win-arm64-sha256 }} osx-x64-sha-noruntime: ${{ steps.sha_noruntime.outputs.osx-x64-sha256 }} osx-arm64-sha-noruntime: ${{ steps.sha_noruntime.outputs.osx-arm64-sha256 }} linux-x64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.linux-x64-sha256 }} linux-arm64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.linux-arm64-sha256 }} linux-arm-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.linux-arm-sha256 }} win-x64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.win-x64-sha256 }} + win-arm64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.win-arm64-sha256 }} osx-x64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.osx-x64-sha256 }} osx-arm64-sha-noruntime-noexternals: ${{ steps.sha_noruntime_noexternals.outputs.osx-arm64-sha256 }} strategy: matrix: - runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, osx-x64, osx-arm64 ] + runtime: [ linux-x64, linux-arm64, linux-arm, win-x64, osx-x64, osx-arm64, win-arm64 ] include: - runtime: linux-x64 os: ubuntu-latest @@ -89,7 +93,7 @@ jobs: - runtime: osx-x64 os: macOS-latest devScript: ./dev.sh - + - runtime: osx-arm64 os: macOS-latest devScript: ./dev.sh @@ -98,6 +102,10 @@ jobs: os: windows-2019 devScript: ./dev + - runtime: win-arm64 + os: windows-latest + devScript: ./dev + runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 @@ -158,9 +166,9 @@ jobs: id: sha_noruntime_noexternals name: Compute SHA256 working-directory: _package_trims/trim_runtime_externals - + - name: Create trimmedpackages.json for ${{ matrix.runtime }} - if: matrix.runtime == 'win-x64' + if: matrix.runtime == 'win-x64' || matrix.runtime == 'win-arm64' uses: actions/github-script@0.3.0 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -180,7 +188,7 @@ jobs: fs.writeFileSync('${{ matrix.runtime }}-trimmedpackages.json', trimmedPackages) - name: Create trimmedpackages.json for ${{ matrix.runtime }} - if: matrix.runtime != 'win-x64' + if: matrix.runtime != 'win-x64' && matrix.runtime != 'win-arm64' uses: actions/github-script@0.3.0 with: github-token: ${{secrets.GITHUB_TOKEN}} @@ -239,24 +247,28 @@ jobs: const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '') var releaseNote = fs.readFileSync('${{ github.workspace }}/releaseNote.md', 'utf8').replace(//g, runnerVersion) releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha-noexternals}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha-noruntime}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha-noruntime-noexternals}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noruntime-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha-noruntime-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha-noruntime-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha-noruntime-noexternals}}') @@ -271,6 +283,7 @@ jobs: run: | ls -l echo "${{needs.build.outputs.win-x64-sha}} actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip" | shasum -a 256 -c + echo "${{needs.build.outputs.win-arm64-sha}} actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip" | shasum -a 256 -c echo "${{needs.build.outputs.osx-x64-sha}} actions-runner-osx-x64-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c echo "${{needs.build.outputs.osx-arm64-sha}} actions-runner-osx-arm64-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c echo "${{needs.build.outputs.linux-x64-sha}} actions-runner-linux-x64-${{ steps.releaseNote.outputs.version }}.tar.gz" | shasum -a 256 -c @@ -300,6 +313,16 @@ jobs: asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}.zip asset_content_type: application/octet-stream + - name: Upload Release Asset (win-arm64) + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.createRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/_package/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip + asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}.zip + asset_content_type: application/octet-stream + - name: Upload Release Asset (linux-x64) uses: actions/upload-release-asset@v1.0.1 env: @@ -361,6 +384,17 @@ jobs: asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noexternals.zip asset_content_type: application/octet-stream + # Upload release assets (trim externals) + - name: Upload Release Asset (win-arm64-noexternals) + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.createRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/_package_trims/trim_externals/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.zip + asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noexternals.zip + asset_content_type: application/octet-stream + - name: Upload Release Asset (linux-x64-noexternals) uses: actions/upload-release-asset@v1.0.1 env: @@ -422,6 +456,17 @@ jobs: asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noruntime.zip asset_content_type: application/octet-stream + # Upload release assets (trim runtime) + - name: Upload Release Asset (win-arm64-noruntime) + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.createRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/_package_trims/trim_runtime/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.zip + asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime.zip + asset_content_type: application/octet-stream + - name: Upload Release Asset (linux-x64-noruntime) uses: actions/upload-release-asset@v1.0.1 env: @@ -483,6 +528,17 @@ jobs: asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip asset_content_type: application/octet-stream + # Upload release assets (trim runtime and externals) + - name: Upload Release Asset (win-arm64-noruntime-noexternals) + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.createRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/_package_trims/trim_runtime_externals/actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip + asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-noruntime-noexternals.zip + asset_content_type: application/octet-stream + - name: Upload Release Asset (linux-x64-noruntime-noexternals) uses: actions/upload-release-asset@v1.0.1 env: @@ -544,6 +600,17 @@ jobs: asset_name: actions-runner-win-x64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json asset_content_type: application/octet-stream + # Upload release assets (trimmedpackages.json) + - name: Upload Release Asset (win-arm64-trimmedpackages.json) + uses: actions/upload-release-asset@v1.0.1 + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + upload_url: ${{ steps.createRelease.outputs.upload_url }} + asset_path: ${{ github.workspace }}/win-arm64-trimmedpackages.json + asset_name: actions-runner-win-arm64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json + asset_content_type: application/octet-stream + - name: Upload Release Asset (linux-x64-trimmedpackages.json) uses: actions/upload-release-asset@v1.0.1 env: diff --git a/docs/contribute.md b/docs/contribute.md index fab3563afc7..bb046f4cb9a 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -27,6 +27,8 @@ An ADR is an Architectural Decision Record. This allows consensus on the direct ![Win](res/win_sm.png) Visual Studio 2017 or newer [Install here](https://visualstudio.microsoft.com) (needed for dev sh script) +![Win-arm](res/win_sm.png) Visual Studio 2022 17.3 Preview or later. [Install here](https://docs.microsoft.com/en-us/visualstudio/releases/2022/release-notes-preview) + ## Quickstart: Run a job from a real repository If you just want to get from building the sourcecode to using it to execute an action, you will need: diff --git a/releaseNote.md b/releaseNote.md index b70766e7098..9392b24cc30 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -16,6 +16,22 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem ; [System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-x64-.zip", "$PWD") ``` +## [Pre-release] Windows arm64 +**Warning:** Windows arm64 runners are currently in preview status and use [unofficial versions of nodejs](https://unofficial-builds.nodejs.org/). They are not intended for production workflows. + +We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. + +The following snipped needs to be run on `powershell`: +``` powershell +# Create a folder under the drive root +mkdir \actions-runner ; cd \actions-runner +# Download the latest runner package +Invoke-WebRequest -Uri https://github.com/actions/runner/releases/download/v/actions-runner-win-arm64-.zip -OutFile actions-runner-win-arm64-.zip +# Extract the installer +Add-Type -AssemblyName System.IO.Compression.FileSystem ; +[System.IO.Compression.ZipFile]::ExtractToDirectory("$PWD\actions-runner-win-arm64-.zip", "$PWD") +``` + ## OSX x64 ``` bash @@ -79,6 +95,7 @@ For additional details about configuring, running, or shutting down the runner p The SHA-256 checksums for the packages included in this build are shown below: - actions-runner-win-x64-.zip +- actions-runner-win-arm64-.zip - actions-runner-osx-x64-.tar.gz - actions-runner-osx-arm64-.tar.gz - actions-runner-linux-x64-.tar.gz @@ -86,6 +103,7 @@ The SHA-256 checksums for the packages included in this build are shown below: - actions-runner-linux-arm-.tar.gz - actions-runner-win-x64--noexternals.zip +- actions-runner-win-arm64--noexternals.zip - actions-runner-osx-x64--noexternals.tar.gz - actions-runner-osx-arm64--noexternals.tar.gz - actions-runner-linux-x64--noexternals.tar.gz @@ -93,6 +111,7 @@ The SHA-256 checksums for the packages included in this build are shown below: - actions-runner-linux-arm--noexternals.tar.gz - actions-runner-win-x64--noruntime.zip +- actions-runner-win-arm64--noruntime.zip - actions-runner-osx-x64--noruntime.tar.gz - actions-runner-osx-arm64--noruntime.tar.gz - actions-runner-linux-x64--noruntime.tar.gz @@ -100,6 +119,7 @@ The SHA-256 checksums for the packages included in this build are shown below: - actions-runner-linux-arm--noruntime.tar.gz - actions-runner-win-x64--noruntime-noexternals.zip +- actions-runner-win-arm64--noruntime-noexternals.zip - actions-runner-osx-x64--noruntime-noexternals.tar.gz - actions-runner-osx-arm64--noruntime-noexternals.tar.gz - actions-runner-linux-x64--noruntime-noexternals.tar.gz diff --git a/src/Directory.Build.props b/src/Directory.Build.props index 1931b7f8f08..9db5faca37a 100644 --- a/src/Directory.Build.props +++ b/src/Directory.Build.props @@ -24,6 +24,9 @@ $(DefineConstants);X86 + + $(DefineConstants);ARM64 + $(DefineConstants);X64 diff --git a/src/Misc/contentHash/dotnetRuntime/win-arm64 b/src/Misc/contentHash/dotnetRuntime/win-arm64 new file mode 100644 index 00000000000..8d279b0417b --- /dev/null +++ b/src/Misc/contentHash/dotnetRuntime/win-arm64 @@ -0,0 +1 @@ +39d0683f0f115a211cb10c473e9574c16549a19d4e9a6c637ded3d7022bf809f diff --git a/src/Misc/contentHash/externals/win-arm64 b/src/Misc/contentHash/externals/win-arm64 new file mode 100644 index 00000000000..bd10bc5ea31 --- /dev/null +++ b/src/Misc/contentHash/externals/win-arm64 @@ -0,0 +1 @@ +e5dace2d41cc0682d096dcce4970079ad48ec7107e46195970eecfdb3df2acef diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index b11b225f751..123c46ce5bd 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -3,6 +3,7 @@ PACKAGERUNTIME=$1 PRECACHE=$2 NODE_URL=https://nodejs.org/dist +UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release NODE12_VERSION="12.22.7" NODE16_VERSION="16.13.0" @@ -134,6 +135,16 @@ if [[ "$PACKAGERUNTIME" == "win-x64" || "$PACKAGERUNTIME" == "win-x86" ]]; then fi fi +# Download the external tools only for Windows. +if [[ "$PACKAGERUNTIME" == "win-arm64" ]]; then + # todo: replace these with official release when available + acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.exe" node16/bin + acquireExternalTool "$UNOFFICIAL_NODE_URL/v${NODE16_VERSION}/$PACKAGERUNTIME/node.lib" node16/bin + if [[ "$PRECACHE" != "" ]]; then + acquireExternalTool "https://github.com/microsoft/vswhere/releases/download/2.6.7/vswhere.exe" vswhere + fi +fi + # Download the external tools only for OSX. if [[ "$PACKAGERUNTIME" == "osx-x64" ]]; then acquireExternalTool "$NODE_URL/v${NODE12_VERSION}/node-v${NODE12_VERSION}-darwin-x64.tar.gz" node12 fix_nested_dir diff --git a/src/Misc/runnerdotnetruntimeassets b/src/Misc/runnerdotnetruntimeassets index 1cbb627713c..fa2d9e04b4c 100644 --- a/src/Misc/runnerdotnetruntimeassets +++ b/src/Misc/runnerdotnetruntimeassets @@ -66,12 +66,14 @@ libmscordbi.dylib libmscordbi.so Microsoft.CSharp.dll Microsoft.DiaSymReader.Native.amd64.dll +Microsoft.DiaSymReader.Native.arm64.dll Microsoft.VisualBasic.Core.dll Microsoft.VisualBasic.dll Microsoft.Win32.Primitives.dll Microsoft.Win32.Registry.dll mscordaccore.dll mscordaccore_amd64_amd64_6.0.522.21309.dll +mscordaccore_arm64_arm64_6.0.522.21309.dll mscordbi.dll mscorlib.dll mscorrc.debug.dll @@ -261,4 +263,4 @@ System.Xml.XmlSerializer.dll System.Xml.XPath.dll System.Xml.XPath.XDocument.dll ucrtbase.dll -WindowsBase.dll \ No newline at end of file +WindowsBase.dll diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 0e756ae552d..25ccc96132f 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -77,7 +77,7 @@ public static class Runner public static readonly Architecture PlatformArchitecture = Architecture.X64; #elif ARM public static readonly Architecture PlatformArchitecture = Architecture.Arm; -#elif ARM64 +#elif ARM64 public static readonly Architecture PlatformArchitecture = Architecture.Arm64; #endif @@ -152,7 +152,7 @@ public static class Features public static readonly string DiskSpaceWarning = "runner.diskspace.warning"; public static readonly string Node12Warning = "DistributedTask.AddWarningToNode12Action"; public static readonly string UseContainerPathForTemplate = "DistributedTask.UseContainerPathForTemplate"; - public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks"; + public static readonly string AllowRunnerContainerHooks = "DistributedTask.AllowRunnerContainerHooks"; } public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; @@ -241,8 +241,8 @@ public static class Agent public static readonly string ToolsDirectory = "agent.ToolsDirectory"; // Set this env var to "node12" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions. - public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION"; - public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION"; + public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION"; + public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION"; } public static class System @@ -255,5 +255,12 @@ public static class System public static readonly string PhaseDisplayName = "system.phaseDisplayName"; } } + + public static class OperatingSystem + { + public static readonly int Windows11BuildVersion = 22000; + // Both windows 10 and windows 11 share the same Major Version 10, need to use the build version to differentiate + public static readonly int Windows11MajorVersion = 10; + } } } diff --git a/src/Runner.Common/Runner.Common.csproj b/src/Runner.Common/Runner.Common.csproj index bfbdc5f2e16..64bf4cda35a 100644 --- a/src/Runner.Common/Runner.Common.csproj +++ b/src/Runner.Common/Runner.Common.csproj @@ -3,7 +3,7 @@ net6.0 Library - win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64 + win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true NU1701;NU1603 $(Version) diff --git a/src/Runner.Common/Util/NodeUtil.cs b/src/Runner.Common/Util/NodeUtil.cs index 8fbf53bdf52..a9cbfc6df21 100644 --- a/src/Runner.Common/Util/NodeUtil.cs +++ b/src/Runner.Common/Util/NodeUtil.cs @@ -7,7 +7,7 @@ public static class NodeUtil { private const string _defaultNodeVersion = "node16"; -#if OS_OSX && ARM64 +#if (OS_OSX || OS_WINDOWS) && ARM64 public static readonly ReadOnlyCollection BuiltInNodeVersions = new(new[] { "node16" }); #else public static readonly ReadOnlyCollection BuiltInNodeVersions = new(new[] { "node12", "node16" }); diff --git a/src/Runner.Listener/Program.cs b/src/Runner.Listener/Program.cs index 4d52bed8f68..1482600ce95 100644 --- a/src/Runner.Listener/Program.cs +++ b/src/Runner.Listener/Program.cs @@ -58,6 +58,18 @@ private async static Task MainAsync(IHostContext context, string[] args) terminal.WriteLine("This runner version is built for Windows. Please install a correct build for your OS."); return Constants.Runner.ReturnCode.TerminatedError; } + #if ARM64 + // A little hacky, but windows gives no way to differentiate between windows 10 and 11. + // By default only 11 supports native x64 app emulation on arm, so we only want to support windows 11 + // https://docs.microsoft.com/en-us/windows/arm/overview#build-windows-apps-that-run-on-arm + // Windows 10 and 11 share a MajorVersion, so we also check the build version. Minor for both is 0, so doing < 0 doesn't really make a lot of sense. + if (Environment.OSVersion.Version.Major < Constants.OperatingSystem.Windows11MajorVersion || + Environment.OSVersion.Version.Build < Constants.OperatingSystem.Windows11BuildVersion) + { + terminal.WriteLine("Win-arm64 runners require windows 11 or later. Please upgrade your operating system."); + return Constants.Runner.ReturnCode.TerminatedError; + } + #endif break; default: terminal.WriteLine($"Running the runner on this platform is not supported. The current platform is {RuntimeInformation.OSDescription} and it was built for {Constants.Runner.Platform.ToString()}."); diff --git a/src/Runner.Listener/Runner.Listener.csproj b/src/Runner.Listener/Runner.Listener.csproj index 4298c753079..577847891be 100644 --- a/src/Runner.Listener/Runner.Listener.csproj +++ b/src/Runner.Listener/Runner.Listener.csproj @@ -3,7 +3,7 @@ net6.0 Exe - win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64 + win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true NU1701;NU1603 $(Version) diff --git a/src/Runner.PluginHost/Runner.PluginHost.csproj b/src/Runner.PluginHost/Runner.PluginHost.csproj index 7937a25164d..df30f3450a2 100644 --- a/src/Runner.PluginHost/Runner.PluginHost.csproj +++ b/src/Runner.PluginHost/Runner.PluginHost.csproj @@ -3,7 +3,7 @@ net6.0 Exe - win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64 + win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true NU1701;NU1603 $(Version) diff --git a/src/Runner.Plugins/Runner.Plugins.csproj b/src/Runner.Plugins/Runner.Plugins.csproj index 8314aea5187..39245a3f7a7 100644 --- a/src/Runner.Plugins/Runner.Plugins.csproj +++ b/src/Runner.Plugins/Runner.Plugins.csproj @@ -3,7 +3,7 @@ net6.0 Library - win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64 + win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true NU1701;NU1603 $(Version) diff --git a/src/Runner.Sdk/Runner.Sdk.csproj b/src/Runner.Sdk/Runner.Sdk.csproj index 63d6c0e186d..202e8669a64 100644 --- a/src/Runner.Sdk/Runner.Sdk.csproj +++ b/src/Runner.Sdk/Runner.Sdk.csproj @@ -3,12 +3,12 @@ net6.0 Library - win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64 + win-x64;win-x86;linux-x64;linux-arm64;linux-arm;osx-x64;osx-arm64;win-arm64 true NU1701;NU1603 $(Version) - + diff --git a/src/Runner.Service/Windows/App.config b/src/Runner.Service/Windows/App.config index 8e15646352e..486e3ea0979 100644 --- a/src/Runner.Service/Windows/App.config +++ b/src/Runner.Service/Windows/App.config @@ -1,6 +1,6 @@  - + - \ No newline at end of file + diff --git a/src/Runner.Service/Windows/AppARM.config b/src/Runner.Service/Windows/AppARM.config new file mode 100644 index 00000000000..ca9f6a32e67 --- /dev/null +++ b/src/Runner.Service/Windows/AppARM.config @@ -0,0 +1,6 @@ + + + + + + diff --git a/src/Runner.Service/Windows/RunnerService.csproj b/src/Runner.Service/Windows/RunnerService.csproj index 73fe4ecc655..238e67aac6e 100644 --- a/src/Runner.Service/Windows/RunnerService.csproj +++ b/src/Runner.Service/Windows/RunnerService.csproj @@ -11,10 +11,15 @@ RunnerService false false - v4.5 512 true + + v4.8 + + + v4.5 + AnyCPU true @@ -61,7 +66,10 @@ Resource.resx - + + + + @@ -71,7 +79,7 @@ - ## Misc -- Bumped `actions/core` dependency (#2123) +- Added a feature flag to start warning on `save-state` and `set-output` deprecation (#2164) +- Prepare supporting `vars` in workflow templates (#2096) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index 8b05c7a6bbc..4114f64cd7b 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.297.0 +2.298.0 From 5156aab25b8f4dfd9b266a1da326863d6bc985e2 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Tue, 4 Oct 2022 17:05:18 +0200 Subject: [PATCH 028/145] Fix incorrect template vars to show SHA for WIN-ARM64 (#2171) --- .github/workflows/release.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 20927e2176f..abad70edaa7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -247,28 +247,28 @@ jobs: const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '') var releaseNote = fs.readFileSync('${{ github.workspace }}/releaseNote.md', 'utf8').replace(//g, runnerVersion) releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha}}') - releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm64-sha}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha-noexternals}}') - releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noexternals}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm64-sha-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha-noruntime}}') - releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noruntime}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-arm64-sha-noruntime}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-x64-sha-noruntime-noexternals}}') - releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noruntime-noexternals}}') + releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.win-arm64-sha-noruntime-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-x64-sha-noruntime-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.osx-arm64-sha-noruntime-noexternals}}') releaseNote = releaseNote.replace(//g, '${{needs.build.outputs.linux-x64-sha-noruntime-noexternals}}') From aa5f3d9d7c29b669cc5b07aa18c07c6a0eb79bdf Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Tue, 4 Oct 2022 17:20:35 +0200 Subject: [PATCH 029/145] Backport 2.298.1 (#2175) * Update releaseNote.md * Update runnerversion --- releaseNote.md | 3 ++- src/runnerversion | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 6e9ca404816..5825dee692c 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,7 +1,8 @@ ## Features - Service containers startup error logs are now included in workflow's logs (#2110) - +## Bugs +- Fixed missing SHA for Windows arm64 release archive (#2171) ## Misc - Added a feature flag to start warning on `save-state` and `set-output` deprecation (#2164) diff --git a/src/runnerversion b/src/runnerversion index 4114f64cd7b..6abfac6bb9c 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.298.0 +2.298.1 From 4cecc9fbcb715d25a398f8c82c75e281775d4fd1 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Wed, 5 Oct 2022 17:00:22 +0200 Subject: [PATCH 030/145] Revert "Avastancu/joannaakl/service container error log (#2110)" (#2179) This reverts commit 949269104d86b6d70d1fe07d34db9b3bedd57b0f. --- src/Runner.Worker/Container/ContainerInfo.cs | 2 - .../ContainerOperationProvider.cs | 59 +++------- .../L0/Worker/ContainerOperationProviderL0.cs | 108 ------------------ 3 files changed, 19 insertions(+), 150 deletions(-) delete mode 100644 src/Test/L0/Worker/ContainerOperationProviderL0.cs diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index 32e55eb3c40..9c114939e2d 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -92,8 +92,6 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, public bool IsJobContainer { get; set; } public bool IsAlpine { get; set; } - public bool FailedInitialization { get; set; } - public IDictionary ContainerEnvironmentVariables { get diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 2fc59730888..73472795c5e 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -98,41 +98,12 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec await StartContainerAsync(executionContext, container); } - await RunContainersHealthcheck(executionContext, containers); - } - - public async Task RunContainersHealthcheck(IExecutionContext executionContext, List containers) - { executionContext.Output("##[group]Waiting for all services to be ready"); - - var unhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { - var healthcheck = await ContainerHealthcheck(executionContext, container); - - if (!string.Equals(healthcheck, "healthy", StringComparison.OrdinalIgnoreCase)) - { - unhealthyContainers.Add(container); - } - else - { - executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); - } + await ContainerHealthcheck(executionContext, container); } executionContext.Output("##[endgroup]"); - - if (unhealthyContainers.Count > 0) - { - foreach (var container in unhealthyContainers) - { - executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); - await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); - executionContext.Error($"Failed to initialize container {container.ContainerImage}"); - container.FailedInitialization = true; - executionContext.Output("##[endgroup]"); - } - throw new InvalidOperationException("One or more containers failed to start."); - } } public async Task StopContainersAsync(IExecutionContext executionContext, object data) @@ -328,15 +299,16 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai if (!string.IsNullOrEmpty(container.ContainerId)) { - if (!container.IsJobContainer && !container.FailedInitialization) + if (!container.IsJobContainer) { - executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); + // Print logs for service container jobs (not the "action" job itself b/c that's already logged). + executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); - int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); - if (logsExitCode != 0) - { - executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); - } + int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); + if (logsExitCode != 0) + { + executionContext.Warning($"Docker logs fail with exit code {logsExitCode}"); + } } executionContext.Output($"Stop and remove container: {container.ContainerDisplayName}"); @@ -423,14 +395,14 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) + private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) { // Container has no HEALTHCHECK - return String.Empty; + return; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) @@ -441,7 +413,14 @@ private async Task ContainerHealthcheck(IExecutionContext executionConte serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); retryCount++; } - return serviceHealth; + if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) + { + executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); + } + else + { + throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); + } } private async Task ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs deleted file mode 100644 index bfc09f343ac..00000000000 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ /dev/null @@ -1,108 +0,0 @@ -using GitHub.Runner.Worker; -using GitHub.Runner.Worker.Container; -using Xunit; -using Moq; -using GitHub.Runner.Worker.Container.ContainerHooks; -using System.Threading.Tasks; -using System.Collections.Generic; -using System.Runtime.CompilerServices; -using GitHub.DistributedTask.WebApi; -using System; - -namespace GitHub.Runner.Common.Tests.Worker -{ - - public sealed class ContainerOperationProviderL0 - { - - private TestHostContext _hc; - private Mock _ec; - private Mock _dockerManager; - private Mock _containerHookManager; - private ContainerOperationProvider containerOperationProvider; - private Mock serverQueue; - private Mock pagingLogger; - private List healthyDockerStatus = new List { "healthy" }; - private List unhealthyDockerStatus = new List { "unhealthy" }; - private List dockerLogs = new List { "log1", "log2", "log3" }; - - List containers = new List(); - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertFailedTask() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); - - //Act - try - { - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - } - catch (InvalidOperationException) - { - - //Assert - Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); - } - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); - - //Act and Assert - await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); - - } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Worker")] - public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask() - { - //Arrange - Setup(); - _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); - - //Act - await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); - - //Assert - Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); - - } - - private void Setup([CallerMemberName] string testName = "") - { - containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); - _hc = new TestHostContext(this, testName); - _ec = new Mock(); - serverQueue = new Mock(); - pagingLogger = new Mock(); - - _dockerManager = new Mock(); - _containerHookManager = new Mock(); - containerOperationProvider = new ContainerOperationProvider(); - - _hc.SetSingleton(_dockerManager.Object); - _hc.SetSingleton(serverQueue.Object); - _hc.SetSingleton(pagingLogger.Object); - - _hc.SetSingleton(_dockerManager.Object); - _hc.SetSingleton(_containerHookManager.Object); - - _ec.Setup(x => x.Global).Returns(new GlobalContext()); - - containerOperationProvider.Initialize(_hc); - } - } -} \ No newline at end of file From d719a7bb9c4f8728847a51164c4dab8555af6b02 Mon Sep 17 00:00:00 2001 From: Robin Neatherway Date: Thu, 6 Oct 2022 13:20:30 +0100 Subject: [PATCH 031/145] Correct grammar in archive extraction error message (#2184) --- src/Runner.Listener/SelfUpdater.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Listener/SelfUpdater.cs b/src/Runner.Listener/SelfUpdater.cs index 3c8242402d5..07db370f634 100644 --- a/src/Runner.Listener/SelfUpdater.cs +++ b/src/Runner.Listener/SelfUpdater.cs @@ -596,7 +596,7 @@ private async Task ExtractRunnerPackage(string archiveFile, string extractDirect int exitCode = await processInvoker.ExecuteAsync(extractDirectory, tar, $"-xzf \"{archiveFile}\"", null, token); if (exitCode != 0) { - throw new NotSupportedException($"Can't use 'tar -xzf' extract archive file: {archiveFile}. return code: {exitCode}."); + throw new NotSupportedException($"Can't use 'tar -xzf' to extract archive file: {archiveFile}. return code: {exitCode}."); } } } From d977f4c2824cf5f7f6ed90df9b2b5448a3cc4ac0 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 6 Oct 2022 10:28:32 -0400 Subject: [PATCH 032/145] changes to support specific run service URL (#2158) * changes to support run service url * feedback --- src/Runner.Common/ActionsRunServer.cs | 51 +++ src/Runner.Common/RunServer.cs | 78 +--- src/Runner.Common/RunnerServer.cs | 25 -- src/Runner.Common/RunnerService.cs | 62 ++- src/Runner.Listener/Runner.cs | 18 +- src/Runner.Listener/RunnerJobRequestRef.cs | 4 +- src/Runner.Sdk/Util/VssUtil.cs | 47 ++- .../Authentication/VssCredentialsExtension.cs | 20 + .../Common/RawClientHttpRequestSettings.cs | 194 ++++++++++ src/Sdk/Common/Common/RawHttpHeaders.cs | 12 + .../Common/Common/RawHttpMessageHandler.cs | 349 +++++++++++++++++ .../DTWebApi/WebApi/RunServiceHttpClient.cs | 75 ++++ .../WebApi/WebApi/OAuth/VssOAuthCredential.cs | 10 +- src/Sdk/WebApi/WebApi/RawConnection.cs | 207 ++++++++++ src/Sdk/WebApi/WebApi/RawHttpClientBase.cs | 352 ++++++++++++++++++ 15 files changed, 1408 insertions(+), 96 deletions(-) create mode 100644 src/Runner.Common/ActionsRunServer.cs create mode 100644 src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs create mode 100644 src/Sdk/Common/Common/RawClientHttpRequestSettings.cs create mode 100644 src/Sdk/Common/Common/RawHttpHeaders.cs create mode 100644 src/Sdk/Common/Common/RawHttpMessageHandler.cs create mode 100644 src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs create mode 100644 src/Sdk/WebApi/WebApi/RawConnection.cs create mode 100644 src/Sdk/WebApi/WebApi/RawHttpClientBase.cs diff --git a/src/Runner.Common/ActionsRunServer.cs b/src/Runner.Common/ActionsRunServer.cs new file mode 100644 index 00000000000..704a690e3ca --- /dev/null +++ b/src/Runner.Common/ActionsRunServer.cs @@ -0,0 +1,51 @@ +using System; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.Pipelines; +using GitHub.DistributedTask.WebApi; +using GitHub.Services.Common; +using GitHub.Services.WebApi; + +namespace GitHub.Runner.Common +{ + [ServiceLocator(Default = typeof(ActionsRunServer))] + public interface IActionsRunServer : IRunnerService + { + Task ConnectAsync(Uri serverUrl, VssCredentials credentials); + + Task GetJobMessageAsync(string id, CancellationToken token); + } + + public sealed class ActionsRunServer : RunnerService, IActionsRunServer + { + private bool _hasConnection; + private VssConnection _connection; + private TaskAgentHttpClient _taskAgentClient; + + public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials) + { + _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100)); + _taskAgentClient = _connection.GetClient(); + _hasConnection = true; + } + + private void CheckConnection() + { + if (!_hasConnection) + { + throw new InvalidOperationException($"SetConnection"); + } + } + + public Task GetJobMessageAsync(string id, CancellationToken cancellationToken) + { + CheckConnection(); + var jobMessage = RetryRequest(async () => + { + return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken); + }, cancellationToken); + + return jobMessage; + } + } +} diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 981a61b7f0c..c4ccb649441 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.Pipelines; @@ -6,6 +6,7 @@ using GitHub.Runner.Sdk; using GitHub.Services.Common; using GitHub.Services.WebApi; +using Sdk.WebApi.WebApi.RawClient; namespace GitHub.Runner.Common { @@ -20,40 +21,17 @@ public interface IRunServer : IRunnerService public sealed class RunServer : RunnerService, IRunServer { private bool _hasConnection; - private VssConnection _connection; - private TaskAgentHttpClient _taskAgentClient; + private Uri requestUri; + private RawConnection _connection; + private RunServiceHttpClient _runServiceHttpClient; - public async Task ConnectAsync(Uri serverUrl, VssCredentials credentials) + public async Task ConnectAsync(Uri serverUri, VssCredentials credentials) { - _connection = await EstablishVssConnection(serverUrl, credentials, TimeSpan.FromSeconds(100)); - _taskAgentClient = _connection.GetClient(); - _hasConnection = true; - } - - private async Task EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout) - { - Trace.Info($"EstablishVssConnection"); - Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout."); - int attemptCount = 5; - while (attemptCount-- > 0) - { - var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout); - try - { - await connection.ConnectAsync(); - return connection; - } - catch (Exception ex) when (attemptCount > 0) - { - Trace.Info($"Catch exception during connect. {attemptCount} attempt left."); - Trace.Error(ex); - - await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None); - } - } + requestUri = serverUri; - // should never reach here. - throw new InvalidOperationException(nameof(EstablishVssConnection)); + _connection = VssUtil.CreateRawConnection(new Uri(serverUri.Authority), credentials); + _runServiceHttpClient = await _connection.GetClientAsync(); + _hasConnection = true; } private void CheckConnection() @@ -67,37 +45,15 @@ private void CheckConnection() public Task GetJobMessageAsync(string id, CancellationToken cancellationToken) { CheckConnection(); - var jobMessage = RetryRequest(async () => - { - return await _taskAgentClient.GetJobMessageAsync(id, cancellationToken); - }, cancellationToken); - return jobMessage; - } - - private async Task RetryRequest(Func> func, - CancellationToken cancellationToken, - int maxRetryAttemptsCount = 5 - ) - { - var retryCount = 0; - while (true) + var jobMessage = RetryRequest( + async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken); + if (jobMessage == null) { - retryCount++; - cancellationToken.ThrowIfCancellationRequested(); - try - { - return await func(); - } - // TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122 - catch (Exception ex) when (retryCount < maxRetryAttemptsCount) - { - Trace.Error("Catch exception during get full job message"); - Trace.Error(ex); - var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); - Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left."); - await Task.Delay(backOff, cancellationToken); - } + throw new TaskOrchestrationJobNotFoundException(id); } + + return jobMessage; } + } } diff --git a/src/Runner.Common/RunnerServer.cs b/src/Runner.Common/RunnerServer.cs index 6f335a4aa3b..531375b948f 100644 --- a/src/Runner.Common/RunnerServer.cs +++ b/src/Runner.Common/RunnerServer.cs @@ -179,31 +179,6 @@ public void SetConnectionTimeout(RunnerConnectionType connectionType, TimeSpan t } } - private async Task EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout) - { - Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout."); - int attemptCount = 5; - while (attemptCount-- > 0) - { - var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout); - try - { - await connection.ConnectAsync(); - return connection; - } - catch (Exception ex) when (attemptCount > 0) - { - Trace.Info($"Catch exception during connect. {attemptCount} attempt left."); - Trace.Error(ex); - - await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None); - } - } - - // should never reach here. - throw new InvalidOperationException(nameof(EstablishVssConnection)); - } - private void CheckConnection(RunnerConnectionType connectionType) { switch (connectionType) diff --git a/src/Runner.Common/RunnerService.cs b/src/Runner.Common/RunnerService.cs index 7d4f52ea0a0..403764b036c 100644 --- a/src/Runner.Common/RunnerService.cs +++ b/src/Runner.Common/RunnerService.cs @@ -1,4 +1,10 @@ using System; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; +using GitHub.Services.WebApi; +using Sdk.WebApi.WebApi.RawClient; namespace GitHub.Runner.Common { @@ -21,9 +27,9 @@ public abstract class RunnerService protected IHostContext HostContext { get; private set; } protected Tracing Trace { get; private set; } - public string TraceName + public string TraceName { - get + get { return GetType().Name; } @@ -35,5 +41,57 @@ public virtual void Initialize(IHostContext hostContext) Trace = HostContext.GetTrace(TraceName); Trace.Entering(); } + + protected async Task EstablishVssConnection(Uri serverUrl, VssCredentials credentials, TimeSpan timeout) + { + Trace.Info($"EstablishVssConnection"); + Trace.Info($"Establish connection with {timeout.TotalSeconds} seconds timeout."); + int attemptCount = 5; + while (attemptCount-- > 0) + { + var connection = VssUtil.CreateConnection(serverUrl, credentials, timeout: timeout); + try + { + await connection.ConnectAsync(); + return connection; + } + catch (Exception ex) when (attemptCount > 0) + { + Trace.Info($"Catch exception during connect. {attemptCount} attempt left."); + Trace.Error(ex); + + await HostContext.Delay(TimeSpan.FromMilliseconds(100), CancellationToken.None); + } + } + + // should never reach here. + throw new InvalidOperationException(nameof(EstablishVssConnection)); + } + + protected async Task RetryRequest(Func> func, + CancellationToken cancellationToken, + int maxRetryAttemptsCount = 5 + ) + { + var retryCount = 0; + while (true) + { + retryCount++; + cancellationToken.ThrowIfCancellationRequested(); + try + { + return await func(); + } + // TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122 + catch (Exception ex) when (retryCount < maxRetryAttemptsCount) + { + Trace.Error("Catch exception during get full job message"); + Trace.Error(ex); + var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); + Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left."); + await Task.Delay(backOff, cancellationToken); + } + } + } } } diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 13cc7fe97aa..20235d51d78 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -496,16 +496,26 @@ private async Task RunAsync(RunnerSettings settings, bool runOnce = false) else { var messageRef = StringUtil.ConvertFromJson(message.Body); + Pipelines.AgentJobRequestMessage jobRequestMessage = null; // Create connection var credMgr = HostContext.GetService(); var creds = credMgr.LoadCredentials(); - var runServer = HostContext.CreateService(); - await runServer.ConnectAsync(new Uri(settings.ServerUrl), creds); - var jobMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token); + if (string.IsNullOrEmpty(messageRef.RunServiceUrl)) + { + var actionsRunServer = HostContext.CreateService(); + await actionsRunServer.ConnectAsync(new Uri(settings.ServerUrl), creds); + jobRequestMessage = await actionsRunServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token); + } + else + { + var runServer = HostContext.CreateService(); + await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds); + jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token); + } - jobDispatcher.Run(jobMessage, runOnce); + jobDispatcher.Run(jobRequestMessage, runOnce); if (runOnce) { Trace.Info("One time used runner received job message."); diff --git a/src/Runner.Listener/RunnerJobRequestRef.cs b/src/Runner.Listener/RunnerJobRequestRef.cs index fad71d121f3..70ed34146f2 100644 --- a/src/Runner.Listener/RunnerJobRequestRef.cs +++ b/src/Runner.Listener/RunnerJobRequestRef.cs @@ -9,5 +9,7 @@ public sealed class RunnerJobRequestRef public string Id { get; set; } [DataMember(Name = "runner_request_id")] public string RunnerRequestId { get; set; } + [DataMember(Name = "run_service_url")] + public string RunServiceUrl { get; set; } } -} \ No newline at end of file +} diff --git a/src/Runner.Sdk/Util/VssUtil.cs b/src/Runner.Sdk/Util/VssUtil.cs index 70cdf769c48..2194e28ff03 100644 --- a/src/Runner.Sdk/Util/VssUtil.cs +++ b/src/Runner.Sdk/Util/VssUtil.cs @@ -9,6 +9,7 @@ using System.Net.Http.Headers; using System.Runtime.InteropServices; using System.Net; +using Sdk.WebApi.WebApi.RawClient; namespace GitHub.Runner.Sdk { @@ -34,7 +35,11 @@ public static void InitializeVssClientSettings(List addi } } - public static VssConnection CreateConnection(Uri serverUri, VssCredentials credentials, IEnumerable additionalDelegatingHandler = null, TimeSpan? timeout = null) + public static VssConnection CreateConnection( + Uri serverUri, + VssCredentials credentials, + IEnumerable additionalDelegatingHandler = null, + TimeSpan? timeout = null) { VssClientHttpRequestSettings settings = VssClientHttpRequestSettings.Default.Clone(); @@ -75,6 +80,46 @@ public static VssConnection CreateConnection(Uri serverUri, VssCredentials crede return connection; } + public static RawConnection CreateRawConnection( + Uri serverUri, + VssCredentials credentials, + IEnumerable additionalDelegatingHandler = null, + TimeSpan? timeout = null) + { + RawClientHttpRequestSettings settings = RawClientHttpRequestSettings.Default.Clone(); + + int maxRetryRequest; + if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_HTTP_RETRY") ?? string.Empty, out maxRetryRequest)) + { + maxRetryRequest = 3; + } + + // make sure MaxRetryRequest in range [3, 10] + settings.MaxRetryRequest = Math.Min(Math.Max(maxRetryRequest, 3), 10); + + if (!int.TryParse(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_HTTP_TIMEOUT") ?? string.Empty, out int httpRequestTimeoutSeconds)) + { + settings.SendTimeout = timeout ?? TimeSpan.FromSeconds(100); + } + else + { + // prefer environment variable + settings.SendTimeout = TimeSpan.FromSeconds(Math.Min(Math.Max(httpRequestTimeoutSeconds, 100), 1200)); + } + + // Remove Invariant from the list of accepted languages. + // + // The constructor of VssHttpRequestSettings (base class of VssClientHttpRequestSettings) adds the current + // UI culture to the list of accepted languages. The UI culture will be Invariant on OSX/Linux when the + // LANG environment variable is not set when the program starts. If Invariant is in the list of accepted + // languages, then "System.ArgumentException: The value cannot be null or empty." will be thrown when the + // settings are applied to an HttpRequestMessage. + settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture); + + RawConnection connection = new RawConnection(serverUri, new RawHttpMessageHandler(credentials.ToOAuthCredentials(), settings), additionalDelegatingHandler); + return connection; + } + public static VssCredentials GetVssCredential(ServiceEndpoint serviceEndpoint) { ArgUtil.NotNull(serviceEndpoint, nameof(serviceEndpoint)); diff --git a/src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs b/src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs new file mode 100644 index 00000000000..c91fdb70a2e --- /dev/null +++ b/src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs @@ -0,0 +1,20 @@ +using GitHub.Services.OAuth; + +namespace GitHub.Services.Common +{ + public static class VssCredentialsExtension + { + public static VssOAuthCredential ToOAuthCredentials( + this VssCredentials credentials) + { + if (credentials.Federated.CredentialType == VssCredentialsType.OAuth) + { + return credentials.Federated as VssOAuthCredential; + } + else + { + return null; + } + } + } +} diff --git a/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs b/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs new file mode 100644 index 00000000000..60fbbe093fa --- /dev/null +++ b/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs @@ -0,0 +1,194 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Globalization; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Net.Security; +using System.Security.Cryptography.X509Certificates; +using GitHub.Services.WebApi.Utilities.Internal; + +namespace GitHub.Services.Common +{ + public class RawClientHttpRequestSettings + { + /// + /// Timespan to wait before timing out a request. Defaults to 100 seconds + /// + public TimeSpan SendTimeout + { + get; + set; + } + + /// + /// User-Agent header passed along in the request, + /// For multiple values, the order in the list is the order + /// in which they will appear in the header + /// + public List UserAgent + { + get; + set; + } + + /// + /// The name of the culture is passed in the Accept-Language header + /// + public ICollection AcceptLanguages + { + get + { + return m_acceptLanguages; + } + } + + /// + /// A unique identifier for the user session + /// + public Guid SessionId + { + get; + set; + } + + /// + /// Optional implementation used to validate server certificate validation + /// + public Func ServerCertificateValidationCallback + { + get; + set; + } + + /// + /// Number of times to retry a request that has an ambient failure + /// + /// + /// This property is only used by RawConnection, so only relevant on the client + /// + [DefaultValue(c_defaultMaxRetry)] + public Int32 MaxRetryRequest + { + get; + set; + } + + /// + /// Gets the property name used to reference this object. + /// + public const String PropertyName = "Actions.RequestSettings"; + + public static RawClientHttpRequestSettings Default => s_defaultSettings.Value; + + protected RawClientHttpRequestSettings(RawClientHttpRequestSettings copy) + { + this.SendTimeout = copy.SendTimeout; + this.m_acceptLanguages = new List(copy.AcceptLanguages); + this.SessionId = copy.SessionId; + this.UserAgent = new List(copy.UserAgent); + this.ServerCertificateValidationCallback = copy.ServerCertificateValidationCallback; + this.MaxRetryRequest = copy.MaxRetryRequest; + } + + public RawClientHttpRequestSettings Clone() + { + return new RawClientHttpRequestSettings(this); + } + + public RawClientHttpRequestSettings() + : this(Guid.NewGuid()) + { + } + + public RawClientHttpRequestSettings(Guid sessionId) + { + this.SendTimeout = s_defaultTimeout; + if (!String.IsNullOrEmpty(CultureInfo.CurrentUICulture.Name)) // InvariantCulture for example has an empty name. + { + this.AcceptLanguages.Add(CultureInfo.CurrentUICulture); + } + this.SessionId = sessionId; + this.ServerCertificateValidationCallback = null; + + // If different, we'll also add CurrentCulture to the request headers, + // but UICulture was added first, so it gets first preference + if (!CultureInfo.CurrentCulture.Equals(CultureInfo.CurrentUICulture) && !String.IsNullOrEmpty(CultureInfo.CurrentCulture.Name)) + { + this.AcceptLanguages.Add(CultureInfo.CurrentCulture); + } + + this.MaxRetryRequest = c_defaultMaxRetry; + +#if DEBUG + string customClientRequestTimeout = Environment.GetEnvironmentVariable("VSS_Client_Request_Timeout"); + if (!string.IsNullOrEmpty(customClientRequestTimeout) && int.TryParse(customClientRequestTimeout, out int customTimeout)) + { + // avoid disrupting a debug session due to the request timing out by setting a custom timeout. + this.SendTimeout = TimeSpan.FromSeconds(customTimeout); + } +#endif + } + + protected internal virtual Boolean ApplyTo(HttpRequestMessage request) + { + // Make sure we only apply the settings to the request once + if (request.Options.TryGetValue(PropertyName, out _)) + { + return false; + } + + request.Options.Set(new HttpRequestOptionsKey(PropertyName), this); + + if (this.AcceptLanguages != null && this.AcceptLanguages.Count > 0) + { + // An empty or null CultureInfo name will cause an ArgumentNullException in the + // StringWithQualityHeaderValue constructor. CultureInfo.InvariantCulture is an example of + // a CultureInfo that has an empty name. + foreach (CultureInfo culture in this.AcceptLanguages.Where(a => !String.IsNullOrEmpty(a.Name))) + { + request.Headers.AcceptLanguage.Add(new StringWithQualityHeaderValue(culture.Name)); + } + } + + if (this.UserAgent != null) + { + foreach (var headerVal in this.UserAgent) + { + if (!request.Headers.UserAgent.Contains(headerVal)) + { + request.Headers.UserAgent.Add(headerVal); + } + } + } + + if (!request.Headers.Contains(Internal.RawHttpHeaders.SessionHeader)) + { + request.Headers.Add(Internal.RawHttpHeaders.SessionHeader, this.SessionId.ToString("D")); + } + + return true; + } + + /// + /// Creates an instance of the default request settings. + /// + /// The default request settings + private static RawClientHttpRequestSettings ConstructDefaultSettings() + { + // Set up reasonable defaults in case the registry keys are not present + var settings = new RawClientHttpRequestSettings(); + settings.UserAgent = UserAgentUtility.GetDefaultRestUserAgent(); + + return settings; + } + + private static Lazy s_defaultSettings + = new Lazy(ConstructDefaultSettings); + + private const Int32 c_defaultMaxRetry = 3; + private static readonly TimeSpan s_defaultTimeout = TimeSpan.FromSeconds(100); //default WebAPI timeout + private ICollection m_acceptLanguages = new List(); + } +} diff --git a/src/Sdk/Common/Common/RawHttpHeaders.cs b/src/Sdk/Common/Common/RawHttpHeaders.cs new file mode 100644 index 00000000000..49dae2d6dd8 --- /dev/null +++ b/src/Sdk/Common/Common/RawHttpHeaders.cs @@ -0,0 +1,12 @@ +using System; +using System.ComponentModel; +using System.Diagnostics.CodeAnalysis; + +namespace GitHub.Services.Common.Internal +{ + [EditorBrowsable(EditorBrowsableState.Never)] + public static class RawHttpHeaders + { + public const String SessionHeader = "X-Runner-Session"; + } +} diff --git a/src/Sdk/Common/Common/RawHttpMessageHandler.cs b/src/Sdk/Common/Common/RawHttpMessageHandler.cs new file mode 100644 index 00000000000..286fac8403e --- /dev/null +++ b/src/Sdk/Common/Common/RawHttpMessageHandler.cs @@ -0,0 +1,349 @@ +using System; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Common.Diagnostics; +using GitHub.Services.Common.Internal; +using GitHub.Services.OAuth; + +namespace GitHub.Services.Common +{ + public class RawHttpMessageHandler: HttpMessageHandler + { + public RawHttpMessageHandler( + VssOAuthCredential credentials) + : this(credentials, new RawClientHttpRequestSettings()) + { + } + + public RawHttpMessageHandler( + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings) + : this(credentials, settings, new HttpClientHandler()) + { + } + + public RawHttpMessageHandler( + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings, + HttpMessageHandler innerHandler) + { + this.Credentials = credentials; + this.Settings = settings; + m_messageInvoker = new HttpMessageInvoker(innerHandler); + m_credentialWrapper = new CredentialWrapper(); + + // If we were given a pipeline make sure we find the inner-most handler to apply our settings as this + // will be the actual outgoing transport. + { + HttpMessageHandler transportHandler = innerHandler; + DelegatingHandler delegatingHandler = transportHandler as DelegatingHandler; + while (delegatingHandler != null) + { + transportHandler = delegatingHandler.InnerHandler; + delegatingHandler = transportHandler as DelegatingHandler; + } + + m_transportHandler = transportHandler; + } + + ApplySettings(m_transportHandler, m_credentialWrapper, this.Settings); + + m_thisLock = new Object(); + } + + /// + /// Gets the credentials associated with this handler. + /// + public VssOAuthCredential Credentials + { + get; + private set; + } + + /// + /// Gets the settings associated with this handler. + /// + public RawClientHttpRequestSettings Settings + { + get; + private set; + } + + // setting this to WebRequest.DefaultWebProxy in NETSTANDARD is causing a System.PlatformNotSupportedException + //.in System.Net.SystemWebProxy.IsBypassed. Comment in IsBypassed method indicates ".NET Core and .NET Native + // code will handle this exception and call into WinInet/WinHttp as appropriate to use the system proxy." + // This needs to be investigated further. + private static IWebProxy s_defaultWebProxy = null; + + /// + /// Allows you to set a proxy to be used by all RawHttpMessageHandler requests without affecting the global WebRequest.DefaultWebProxy. If not set it returns the WebRequest.DefaultWebProxy. + /// + public static IWebProxy DefaultWebProxy + { + get + { + var toReturn = WebProxyWrapper.Wrap(s_defaultWebProxy); + + if (null != toReturn && + toReturn.Credentials == null) + { + toReturn.Credentials = CredentialCache.DefaultCredentials; + } + + return toReturn; + } + set + { + s_defaultWebProxy = value; + } + } + + protected override async Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + VssTraceActivity traceActivity = VssTraceActivity.Current; + + lock (m_thisLock) + { + // Ensure that we attempt to use the most appropriate authentication mechanism by default. + if (m_tokenProvider == null) + { + m_tokenProvider = this.Credentials.GetTokenProvider(request.RequestUri); + } + } + + CancellationTokenSource tokenSource = null; + HttpResponseMessage response = null; + Boolean succeeded = false; + HttpResponseMessageWrapper responseWrapper; + + Int32 retries = m_maxAuthRetries; + try + { + tokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); + if (this.Settings.SendTimeout > TimeSpan.Zero) + { + tokenSource.CancelAfter(this.Settings.SendTimeout); + } + + do + { + if (response != null) + { + response.Dispose(); + } + + // Let's start with sending a token + IssuedToken token = await m_tokenProvider.GetTokenAsync(null, tokenSource.Token).ConfigureAwait(false); + ApplyToken(request, token); + + // ConfigureAwait(false) enables the continuation to be run outside any captured + // SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking... + response = await m_messageInvoker.SendAsync(request, tokenSource.Token).ConfigureAwait(false); + + responseWrapper = new HttpResponseMessageWrapper(response); + + var isUnAuthorized = responseWrapper.StatusCode == HttpStatusCode.Unauthorized; + if (!isUnAuthorized) + { + // Validate the token after it has been successfully authenticated with the server. + m_tokenProvider?.ValidateToken(token, responseWrapper); + succeeded = true; + break; + } + else + { + m_tokenProvider?.InvalidateToken(token); + + if (retries == 0 || retries < m_maxAuthRetries) + { + break; + } + + token = await m_tokenProvider.GetTokenAsync(token, tokenSource.Token).ConfigureAwait(false); + + retries--; + } + } + while (retries >= 0); + + // We're out of retries and the response was an auth challenge -- then the request was unauthorized + // and we will throw a strongly-typed exception with a friendly error message. + if (!succeeded && response != null && responseWrapper.StatusCode == HttpStatusCode.Unauthorized) + { + // Make sure we do not leak the response object when raising an exception + if (response != null) + { + response.Dispose(); + } + + var message = CommonResources.VssUnauthorized(request.RequestUri.GetLeftPart(UriPartial.Authority)); + VssHttpEventSource.Log.HttpRequestUnauthorized(traceActivity, request, message); + VssUnauthorizedException unauthorizedException = new VssUnauthorizedException(message); + throw unauthorizedException; + } + + return response; + } + catch (OperationCanceledException ex) + { + if (cancellationToken.IsCancellationRequested) + { + VssHttpEventSource.Log.HttpRequestCancelled(traceActivity, request); + throw; + } + else + { + VssHttpEventSource.Log.HttpRequestTimedOut(traceActivity, request, this.Settings.SendTimeout); + throw new TimeoutException(CommonResources.HttpRequestTimeout(this.Settings.SendTimeout), ex); + } + } + finally + { + // We always dispose of the token source since otherwise we leak resources if there is a timer pending + if (tokenSource != null) + { + tokenSource.Dispose(); + } + } + } + + private void ApplyToken( + HttpRequestMessage request, + IssuedToken token) + { + switch (token) + { + case null: + return; + case ICredentials credentialsToken: + m_credentialWrapper.InnerCredentials = credentialsToken; + break; + default: + token.ApplyTo(new HttpRequestMessageWrapper(request)); + break; + } + } + + private static void ApplySettings( + HttpMessageHandler handler, + ICredentials defaultCredentials, + RawClientHttpRequestSettings settings) + { + HttpClientHandler httpClientHandler = handler as HttpClientHandler; + if (httpClientHandler != null) + { + httpClientHandler.ClientCertificateOptions = ClientCertificateOption.Manual; + //Setting httpClientHandler.UseDefaultCredentials to false in .Net Core, clears httpClientHandler.Credentials if + //credentials is already set to defaultcredentials. Therefore httpClientHandler.Credentials must be + //set after httpClientHandler.UseDefaultCredentials. + httpClientHandler.UseDefaultCredentials = false; + httpClientHandler.Credentials = defaultCredentials; + httpClientHandler.PreAuthenticate = false; + httpClientHandler.Proxy = DefaultWebProxy; + httpClientHandler.UseCookies = false; + httpClientHandler.UseProxy = true; + } + } + + private readonly HttpMessageHandler m_transportHandler; + private HttpMessageInvoker m_messageInvoker; + private CredentialWrapper m_credentialWrapper; + private object m_thisLock; + private const Int32 m_maxAuthRetries = 3; + private VssOAuthTokenProvider m_tokenProvider; + + //.Net Core does not attempt NTLM schema on Linux, unless ICredentials is a CredentialCache instance + //This workaround may not be needed after this corefx fix is consumed: https://github.com/dotnet/corefx/pull/7923 + private sealed class CredentialWrapper : CredentialCache, ICredentials + { + public ICredentials InnerCredentials + { + get; + set; + } + + NetworkCredential ICredentials.GetCredential( + Uri uri, + String authType) + { + return InnerCredentials != null ? InnerCredentials.GetCredential(uri, authType) : null; + } + } + + private sealed class WebProxyWrapper : IWebProxy + { + private WebProxyWrapper(IWebProxy toWrap) + { + m_wrapped = toWrap; + m_credentials = null; + } + + public static WebProxyWrapper Wrap(IWebProxy toWrap) + { + if (null == toWrap) + { + return null; + } + + return new WebProxyWrapper(toWrap); + } + + public ICredentials Credentials + { + get + { + ICredentials credentials = m_credentials; + + if (null == credentials) + { + // This means to fall back to the Credentials from the wrapped + // IWebProxy. + credentials = m_wrapped.Credentials; + } + else if (Object.ReferenceEquals(credentials, m_nullCredentials)) + { + // This sentinel value means we have explicitly had our credentials + // set to null. + credentials = null; + } + + return credentials; + } + + set + { + if (null == value) + { + // Use this as a sentinel value to distinguish the case when someone has + // explicitly set our credentials to null. We don't want to fall back to + // m_wrapped.Credentials when we have credentials that are explicitly null. + m_credentials = m_nullCredentials; + } + else + { + m_credentials = value; + } + } + } + + public Uri GetProxy(Uri destination) + { + return m_wrapped.GetProxy(destination); + } + + public bool IsBypassed(Uri host) + { + return m_wrapped.IsBypassed(host); + } + + private readonly IWebProxy m_wrapped; + private ICredentials m_credentials; + + private static readonly ICredentials m_nullCredentials = new CredentialWrapper(); + } + } +} diff --git a/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs b/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs new file mode 100644 index 00000000000..64c1784e6d1 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs @@ -0,0 +1,75 @@ +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Common; +using GitHub.Services.OAuth; +using GitHub.Services.WebApi; +using Sdk.WebApi.WebApi; + +namespace GitHub.DistributedTask.WebApi +{ + [ResourceArea(TaskResourceIds.AreaId)] + public class RunServiceHttpClient : RawHttpClientBase + { + public RunServiceHttpClient( + Uri baseUrl, + VssOAuthCredential credentials) + : base(baseUrl, credentials) + { + } + + public RunServiceHttpClient( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings) + : base(baseUrl, credentials, settings) + { + } + + public RunServiceHttpClient( + Uri baseUrl, + VssOAuthCredential credentials, + params DelegatingHandler[] handlers) + : base(baseUrl, credentials, handlers) + { + } + + public RunServiceHttpClient( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings, + params DelegatingHandler[] handlers) + : base(baseUrl, credentials, settings, handlers) + { + } + + public RunServiceHttpClient( + Uri baseUrl, + HttpMessageHandler pipeline, + Boolean disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + } + + public Task GetJobMessageAsync( + Uri requestUri, + string messageId, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + var payload = new { + StreamID = messageId + }; + + var payloadJson = JsonUtility.ToString(payload); + var requestContent = new StringContent(payloadJson, System.Text.Encoding.UTF8, "application/json"); + return SendAsync( + httpMethod, + additionalHeaders: null, + requestUri: requestUri, + content: requestContent, + cancellationToken: cancellationToken); + } + } +} diff --git a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthCredential.cs b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthCredential.cs index b5c4ea5c130..78cf4fc9fa4 100644 --- a/src/Sdk/WebApi/WebApi/OAuth/VssOAuthCredential.cs +++ b/src/Sdk/WebApi/WebApi/OAuth/VssOAuthCredential.cs @@ -12,7 +12,7 @@ namespace GitHub.Services.OAuth public class VssOAuthCredential : FederatedCredential { /// - /// Initializes a new VssOAuthCredential instance with the specified authorization grant and client + /// Initializes a new VssOAuthCredential instance with the specified authorization grant and client /// credentials. /// /// The location of the token endpoint for the target authorization server @@ -117,8 +117,14 @@ public override Boolean IsAuthenticationChallenge(IHttpResponse webResponse) return false; } + public VssOAuthTokenProvider GetTokenProvider( + Uri serviceUrl) + { + return new VssOAuthTokenProvider(this, serviceUrl); + } + protected override IssuedTokenProvider OnCreateTokenProvider( - Uri serverUrl, + Uri serverUrl, IHttpResponse response) { return new VssOAuthTokenProvider(this, serverUrl); diff --git a/src/Sdk/WebApi/WebApi/RawConnection.cs b/src/Sdk/WebApi/WebApi/RawConnection.cs new file mode 100644 index 00000000000..a486d5245f3 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/RawConnection.cs @@ -0,0 +1,207 @@ +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.Linq; +using System.Net.Http; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Common; +using GitHub.Services.OAuth; +using GitHub.Services.WebApi; +using GitHub.Services.WebApi.Utilities; + +namespace Sdk.WebApi.WebApi.RawClient +{ + public class RawConnection : IDisposable + { + public RawConnection( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings) + : this(baseUrl, new RawHttpMessageHandler(credentials, settings), null) + { + } + + public RawConnection( + Uri baseUrl, + RawHttpMessageHandler innerHandler, + IEnumerable delegatingHandlers) + { + ArgumentUtility.CheckForNull(baseUrl, "baseUrl"); + ArgumentUtility.CheckForNull(innerHandler, "innerHandler"); + + // Permit delegatingHandlers to be null + m_delegatingHandlers = delegatingHandlers = delegatingHandlers ?? Enumerable.Empty(); + + m_baseUrl = baseUrl; + m_innerHandler = innerHandler; + + if (this.Settings.MaxRetryRequest > 0) + { + delegatingHandlers = delegatingHandlers.Concat(new DelegatingHandler[] { new VssHttpRetryMessageHandler(this.Settings.MaxRetryRequest) }); + } + + // Create and persist the pipeline. + if (delegatingHandlers.Any()) + { + m_pipeline = HttpClientFactory.CreatePipeline(m_innerHandler, delegatingHandlers); + } + else + { + m_pipeline = m_innerHandler; + } + } + + /// + /// + /// + public RawClientHttpRequestSettings Settings + { + get + { + return (RawClientHttpRequestSettings)m_innerHandler.Settings; + } + } + + public async Task GetClientAsync(CancellationToken cancellationToken = default(CancellationToken)) where T : RawHttpClientBase + { + CheckForDisposed(); + Type clientType = typeof(T); + + return (T)await GetClientServiceImplAsync(typeof(T), cancellationToken).ConfigureAwait(false); + } + + private async Task GetClientServiceImplAsync( + Type requestedType, + CancellationToken cancellationToken = default(CancellationToken)) + { + CheckForDisposed(); + Object requestedObject = null; + + // Get the actual type to lookup or instantiate, which will either be requestedType itself + // or an extensible type if one was registered + Type managedType = GetExtensibleType(requestedType); + + if (!m_cachedTypes.TryGetValue(managedType, out requestedObject)) + { + AsyncLock typeLock = m_loadingTypes.GetOrAdd(managedType, (t) => new AsyncLock()); + + // This ensures only a single thread at a time will be performing the work to initialize this particular type + // The other threads will go async awaiting the lock task. This is still an improvement over the old synchronous locking, + // as this thread won't be blocked (like a Monitor.Enter), but can return a task to the caller so that the thread + // can continue to be used to do useful work while the result is being worked on. + // We are trusting that getInstanceAsync does not have any code paths that lead back here (for the same type), otherwise we can deadlock on ourselves. + // The old code also extended the same trust which (if violated) would've resulted in a StackOverflowException, + // but with async tasks it will lead to a deadlock. + using (await typeLock.LockAsync(cancellationToken).ConfigureAwait(false)) + { + if (!m_cachedTypes.TryGetValue(managedType, out requestedObject)) + { + requestedObject = (RawHttpClientBase)Activator.CreateInstance(managedType, m_baseUrl, m_pipeline, false /* disposeHandler */); + m_cachedTypes[managedType] = requestedObject; + + AsyncLock removed; + m_loadingTypes.TryRemove(managedType, out removed); + } + } + } + + return requestedObject; + } + + /// + /// + /// + /// + /// + private Type GetExtensibleType(Type managedType) + { + if (managedType.GetTypeInfo().IsAbstract || managedType.GetTypeInfo().IsInterface) + { + Type extensibleType = null; + + // We can add extensible type registration for the client later (app.config? windows registry?). For now it is based solely on the attribute + if (!m_extensibleServiceTypes.TryGetValue(managedType.Name, out extensibleType)) + { + VssClientServiceImplementationAttribute[] attributes = (VssClientServiceImplementationAttribute[])managedType.GetTypeInfo().GetCustomAttributes(true); + if (attributes.Length > 0) + { + if (attributes[0].Type != null) + { + extensibleType = attributes[0].Type; + m_extensibleServiceTypes[managedType.Name] = extensibleType; + } + else if (!String.IsNullOrEmpty(attributes[0].TypeName)) + { + extensibleType = Type.GetType(attributes[0].TypeName); + + if (extensibleType != null) + { + m_extensibleServiceTypes[managedType.Name] = extensibleType; + } + else + { + Debug.Assert(false, "VssConnection: Could not load type from type name: " + attributes[0].TypeName); + } + } + } + } + + if (extensibleType == null) + { + throw new ExtensibleServiceTypeNotRegisteredException(managedType); + } + + if (!managedType.GetTypeInfo().IsAssignableFrom(extensibleType.GetTypeInfo())) + { + throw new ExtensibleServiceTypeNotValidException(managedType, extensibleType); + } + + return extensibleType; + } + else + { + return managedType; + } + } + + public void Dispose() + { + if (!m_isDisposed) + { + lock (m_disposeLock) + { + if (!m_isDisposed) + { + m_isDisposed = true; + foreach (var cachedType in m_cachedTypes.Values.Where(v => v is IDisposable).Select(v => v as IDisposable)) + { + cachedType.Dispose(); + } + m_cachedTypes.Clear(); + } + } + } + } + + private void CheckForDisposed() + { + if (m_isDisposed) + { + throw new ObjectDisposedException(this.GetType().Name); + } + } + + private bool m_isDisposed = false; + private object m_disposeLock = new object(); + private readonly ConcurrentDictionary m_extensibleServiceTypes = new ConcurrentDictionary(); + private readonly Uri m_baseUrl; + private readonly HttpMessageHandler m_pipeline; + private readonly IEnumerable m_delegatingHandlers; + private readonly RawHttpMessageHandler m_innerHandler; + private readonly ConcurrentDictionary m_loadingTypes = new ConcurrentDictionary(); + private readonly ConcurrentDictionary m_cachedTypes = new ConcurrentDictionary(); + } +} diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs new file mode 100644 index 00000000000..ffeecc8084e --- /dev/null +++ b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs @@ -0,0 +1,352 @@ +using System; +using System.Collections; +using System.Collections.Generic; +using System.ComponentModel; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Formatting; +using System.Net.Http.Headers; +using System.Reflection; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Common; +using GitHub.Services.Common.Diagnostics; +using GitHub.Services.OAuth; +using GitHub.Services.WebApi; +using GitHub.Services.WebApi.Utilities.Internal; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; + +namespace Sdk.WebApi.WebApi +{ + public class RawHttpClientBase: IDisposable + { + protected RawHttpClientBase( + Uri baseUrl, + VssOAuthCredential credentials) + : this(baseUrl, credentials, settings: null) + { + } + + protected RawHttpClientBase( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings) + : this(baseUrl, credentials, settings: settings, handlers: null) + { + } + + protected RawHttpClientBase( + Uri baseUrl, + VssOAuthCredential credentials, + params DelegatingHandler[] handlers) + : this(baseUrl, credentials, null, handlers) + { + } + + protected RawHttpClientBase( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings, + params DelegatingHandler[] handlers) + : this(baseUrl, BuildHandler(credentials, settings, handlers), disposeHandler: true) + { + } + + protected RawHttpClientBase( + Uri baseUrl, + HttpMessageHandler pipeline, + bool disposeHandler) + { + m_client = new HttpClient(pipeline, disposeHandler); + + // Disable their timeout since we handle it ourselves + m_client.Timeout = TimeSpan.FromMilliseconds(-1.0); + m_client.BaseAddress = baseUrl; + m_formatter = new VssJsonMediaTypeFormatter(); + } + + public void Dispose() + { + if (!m_isDisposed) + { + lock (m_disposeLock) + { + if (!m_isDisposed) + { + m_isDisposed = true; + m_client.Dispose(); + } + } + } + } + + [EditorBrowsable(EditorBrowsableState.Never)] + public static TimeSpan TestDelay { get; set; } + + protected async Task SendAsync( + HttpMethod method, + Uri requestUri, + HttpContent content = null, + IEnumerable> queryParameters = null, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + using (VssTraceActivity.GetOrCreate().EnterCorrelationScope()) + using (HttpRequestMessage requestMessage = CreateRequestMessage(method, null, requestUri, content, queryParameters)) + { + return await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false); + } + } + + protected async Task SendAsync( + HttpMethod method, + IEnumerable> additionalHeaders, + Uri requestUri, + HttpContent content = null, + IEnumerable> queryParameters = null, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + using (VssTraceActivity.GetOrCreate().EnterCorrelationScope()) + using (HttpRequestMessage requestMessage = CreateRequestMessage(method, additionalHeaders, requestUri, content, queryParameters)) + { + return await SendAsync(requestMessage, userState, cancellationToken).ConfigureAwait(false); + } + } + + protected async Task SendAsync( + HttpRequestMessage message, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + //ConfigureAwait(false) enables the continuation to be run outside + //any captured SyncronizationContext (such as ASP.NET's) which keeps things + //from deadlocking... + using (HttpResponseMessage response = await this.SendAsync(message, userState, cancellationToken).ConfigureAwait(false)) + { + return await ReadContentAsAsync(response, cancellationToken).ConfigureAwait(false); + } + } + + protected Task SendAsync( + HttpRequestMessage message, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + // the default in httpClient for HttpCompletionOption is ResponseContentRead so that is what we do here + return this.SendAsync( + message, + /*completionOption:*/ HttpCompletionOption.ResponseContentRead, + userState, + cancellationToken); + } + + protected async Task SendAsync( + HttpRequestMessage message, + HttpCompletionOption completionOption, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + CheckForDisposed(); + if (message.Headers.UserAgent != null) + { + foreach (ProductInfoHeaderValue headerValue in UserAgentUtility.GetDefaultRestUserAgent()) + { + if (!message.Headers.UserAgent.Contains(headerValue)) + { + message.Headers.UserAgent.Add(headerValue); + } + } + } + + VssTraceActivity traceActivity = VssTraceActivity.GetOrCreate(); + using (traceActivity.EnterCorrelationScope()) + { + VssHttpEventSource.Log.HttpRequestStart(traceActivity, message); + message.Trace(); + HttpResponseMessage response = await Client.SendAsync(message, completionOption, cancellationToken) + .ConfigureAwait(false); + + // Inject delay or failure for testing + if (TestDelay != TimeSpan.Zero) + { + await ProcessDelayAsync().ConfigureAwait(false); + } + + response.Trace(); + VssHttpEventSource.Log.HttpRequestStop(VssTraceActivity.Current, response); + + return response; + } + } + + protected async Task ReadContentAsAsync(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken)) + { + CheckForDisposed(); + Boolean isJson = IsJsonResponse(response); + try + { + //deal with wrapped collections in json + if (isJson && + typeof(IEnumerable).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo()) && + !typeof(Byte[]).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo()) && + !typeof(JObject).GetTypeInfo().IsAssignableFrom(typeof(T).GetTypeInfo())) + { + var wrapper = await ReadJsonContentAsync>(response, cancellationToken).ConfigureAwait(false); + return wrapper.Value; + } + else if (isJson) + { + return await ReadJsonContentAsync(response, cancellationToken).ConfigureAwait(false); + } + } + catch (JsonReaderException) + { + // We thought the content was JSON but failed to parse. + // We ignore for now + } + + return default(T); + } + + protected virtual async Task ReadJsonContentAsync(HttpResponseMessage response, CancellationToken cancellationToken = default(CancellationToken)) + { + return await response.Content.ReadAsAsync(new[] { m_formatter }, cancellationToken).ConfigureAwait(false); + } + + protected HttpRequestMessage CreateRequestMessage( + HttpMethod method, + IEnumerable> additionalHeaders, + Uri requestUri, + HttpContent content = null, + IEnumerable> queryParameters = null, + String mediaType = c_jsonMediaType) + { + CheckForDisposed(); + if (queryParameters != null && queryParameters.Any()) + { + requestUri = requestUri.AppendQuery(queryParameters); + } + + HttpRequestMessage requestMessage = new HttpRequestMessage(method, requestUri.AbsoluteUri); + + MediaTypeWithQualityHeaderValue acceptType = new MediaTypeWithQualityHeaderValue(mediaType); + requestMessage.Headers.Accept.Add(acceptType); + if (additionalHeaders != null) + { + foreach (KeyValuePair kvp in additionalHeaders) + { + requestMessage.Headers.Add(kvp.Key, kvp.Value); + } + } + + if (content != null) + { + requestMessage.Content = content; + } + + return requestMessage; + } + + /// + /// The inner client. + /// + /// + /// Note to implementers: You should not update or expose the inner client + /// unless you instantiate your own instance of this class. Getting + /// an instance of this class from method such as GetClient<T> + /// a cached and shared instance. + /// + protected HttpClient Client + { + get + { + return m_client; + } + } + + /// + /// The media type formatter. + /// + /// + /// Note to implementers: You should not update or expose the media type formatter + /// unless you instantiate your own instance of this class. Getting + /// an instance of this class from method such as GetClient<T> + /// a cached and shared instance. + /// + protected MediaTypeFormatter Formatter + { + get + { + return m_formatter; + } + } + + private static HttpMessageHandler BuildHandler(VssOAuthCredential credentials, RawClientHttpRequestSettings settings, DelegatingHandler[] handlers) + { + RawHttpMessageHandler innerHandler = new RawHttpMessageHandler(credentials, settings ?? new RawClientHttpRequestSettings()); + + if (null == handlers || + 0 == handlers.Length) + { + return innerHandler; + } + + return HttpClientFactory.CreatePipeline(innerHandler, handlers); + } + + private void CheckForDisposed() + { + if (m_isDisposed) + { + throw new ObjectDisposedException(this.GetType().Name); + } + } + + private async Task ProcessDelayAsync() + { + await Task.Delay(Math.Abs((Int32)TestDelay.TotalMilliseconds)).ConfigureAwait(false); + if (TestDelay < TimeSpan.Zero) + { + throw new Exception("User injected failure."); + } + } + + private Boolean IsJsonResponse( + HttpResponseMessage response) + { + if (HasContent(response) + && response.Content.Headers != null && response.Content.Headers.ContentType != null + && !String.IsNullOrEmpty(response.Content.Headers.ContentType.MediaType)) + { + return (0 == String.Compare("application/json", response.Content.Headers.ContentType.MediaType, StringComparison.OrdinalIgnoreCase)); + } + + return false; + } + + private Boolean HasContent(HttpResponseMessage response) + { + if (response != null && + response.StatusCode != HttpStatusCode.NoContent && + response.RequestMessage?.Method != HttpMethod.Head && + response.Content?.Headers != null && + (!response.Content.Headers.ContentLength.HasValue || + (response.Content.Headers.ContentLength.HasValue && response.Content.Headers.ContentLength != 0))) + { + return true; + } + + return false; + } + + private readonly HttpClient m_client; + private MediaTypeFormatter m_formatter; + private bool m_isDisposed = false; + private object m_disposeLock = new object(); + private const String c_jsonMediaType = "application/json"; + } +} From 2f3f6cb620b4cac942b27f91fd94319e4e280c35 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Fri, 7 Oct 2022 16:05:56 +0200 Subject: [PATCH 033/145] Backfill notes from release branch 2.298.2 (wrong branch name) (#2180) * Update releaseNote.md * Update runnerversion --- releaseNote.md | 3 ++- src/runnerversion | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 5825dee692c..10b9a6301a4 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,5 +1,6 @@ ## Features -- Service containers startup error logs are now included in workflow's logs (#2110) +- [REVERTED] Service containers startup error logs are now included in workflow's logs (#2110) + - Reverted due to https://github.com/actions/runner/issues/2173 ## Bugs - Fixed missing SHA for Windows arm64 release archive (#2171) diff --git a/src/runnerversion b/src/runnerversion index 6abfac6bb9c..6f38b5b7a45 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.298.1 +2.298.2 From 9ba99d7e348aa29d657089e50eea604c3d07af34 Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Tue, 11 Oct 2022 10:09:24 +0000 Subject: [PATCH 034/145] Avastancu/joannaakl/service container error log (#2110) (#2182) * Avastancu/joannaakl/service container error log (#2110) * adding support for a service container docker logs * Adding Unit test to ContainerOperationProvider * Adding another test to ContainerOperationProvider * placed the docker logs output in dedicated ##group section * Removed the exception thrown if the service container was not healthy * Removed duplicated logging to the executionContext * Updated the container logs sub-section message * Print service containers only if they were healthy Unhealthy service logs are printed in ContainerHealthCheckLogs called prior to this step. * Removed recently added method to inspect docker logs The method was doing the same thing as the existing DockerLogs method. * Added execution context error This will make a failed health check more visible in the UI without disrupting the execution of the program. * Removing the section 'Waiting for all services to be ready' Since nested subsections are not being displayed properly and we already need one subsection per service error. * Update src/Runner.Worker/Container/DockerCommandManager.cs Co-authored-by: Tingluo Huang * Update src/Test/L0/TestHostContext.cs Co-authored-by: Tingluo Huang * Change the logic for printing Service Containers logs Service container logs will be printed in the 'Start containers' section only if there is an error. Healthy services will have their logs printed in the 'Stop Containers' section. * Removed unused import * Added back section group. * Moved service containers error logs to separate group sections * Removed the test testing the old logic flow. * Remove unnecessary 'IsAnyUnhealthy' flag * Remove printHello() function * Add newline to TestHostContext * Remove unnecessary field 'UnhealthyContainers' * Rename boolean flag indicating service container failure * Refactor healthcheck logic to separate method to enable unit testing. * Remove the default value for bool variable * Update src/Runner.Worker/ContainerOperationProvider.cs Co-authored-by: Tingluo Huang * Update src/Runner.Worker/ContainerOperationProvider.cs Co-authored-by: Tingluo Huang * Rename Healthcheck back to ContainerHealthcheck * Make test sequential * Unextract the container error logs method * remove test asserting thrown exception * Add configure await * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang * Update src/Test/L0/Worker/ContainerOperationProviderL0.cs Co-authored-by: Tingluo Huang * Add back test asserting exception * Check service exit code if there is no healtcheck configured * Remove unnecessary healthcheck for healthy service container * Revert "Check service exit code if there is no healtcheck configured" This reverts commit fec24e834137afe2dc751a38707d0e399c10f8fb. Co-authored-by: Ava S Co-authored-by: Tingluo Huang * Do not fail service containers without the healthcheck Co-authored-by: JoannaaKL Co-authored-by: Tingluo Huang --- src/Runner.Worker/Container/ContainerInfo.cs | 2 + .../ContainerOperationProvider.cs | 47 +++++-- .../L0/Worker/ContainerOperationProviderL0.cs | 126 ++++++++++++++++++ 3 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 src/Test/L0/Worker/ContainerOperationProviderL0.cs diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index 9c114939e2d..32e55eb3c40 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -92,6 +92,8 @@ public ContainerInfo(IHostContext hostContext, Pipelines.JobContainer container, public bool IsJobContainer { get; set; } public bool IsAlpine { get; set; } + public bool FailedInitialization { get; set; } + public IDictionary ContainerEnvironmentVariables { get diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 73472795c5e..b0da204715c 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -98,12 +98,41 @@ public async Task StartContainersAsync(IExecutionContext executionContext, objec await StartContainerAsync(executionContext, container); } + await RunContainersHealthcheck(executionContext, containers); + } + + public async Task RunContainersHealthcheck(IExecutionContext executionContext, List containers) + { executionContext.Output("##[group]Waiting for all services to be ready"); + + var unhealthyContainers = new List(); foreach (var container in containers.Where(c => !c.IsJobContainer)) { - await ContainerHealthcheck(executionContext, container); + var healthy_container = await ContainerHealthcheck(executionContext, container); + + if (!healthy_container) + { + unhealthyContainers.Add(container); + } + else + { + executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); + } } executionContext.Output("##[endgroup]"); + + if (unhealthyContainers.Count > 0) + { + foreach (var container in unhealthyContainers) + { + executionContext.Output($"##[group]Service container {container.ContainerNetworkAlias} failed."); + await _dockerManager.DockerLogs(context: executionContext, containerId: container.ContainerId); + executionContext.Error($"Failed to initialize container {container.ContainerImage}"); + container.FailedInitialization = true; + executionContext.Output("##[endgroup]"); + } + throw new InvalidOperationException("One or more containers failed to start."); + } } public async Task StopContainersAsync(IExecutionContext executionContext, object data) @@ -299,9 +328,8 @@ private async Task StopContainerAsync(IExecutionContext executionContext, Contai if (!string.IsNullOrEmpty(container.ContainerId)) { - if (!container.IsJobContainer) + if (!container.IsJobContainer && !container.FailedInitialization) { - // Print logs for service container jobs (not the "action" job itself b/c that's already logged). executionContext.Output($"Print service container logs: {container.ContainerDisplayName}"); int logsExitCode = await _dockerManager.DockerLogs(executionContext, container.ContainerId); @@ -395,14 +423,14 @@ private async Task RemoveContainerNetworkAsync(IExecutionContext executionContex } } - private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) + private async Task ContainerHealthcheck(IExecutionContext executionContext, ContainerInfo container) { string healthCheck = "--format=\"{{if .Config.Healthcheck}}{{print .State.Health.Status}}{{end}}\""; string serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); if (string.IsNullOrEmpty(serviceHealth)) { // Container has no HEALTHCHECK - return; + return true; } var retryCount = 0; while (string.Equals(serviceHealth, "starting", StringComparison.OrdinalIgnoreCase)) @@ -413,14 +441,7 @@ private async Task ContainerHealthcheck(IExecutionContext executionContext, Cont serviceHealth = (await _dockerManager.DockerInspect(context: executionContext, dockerObject: container.ContainerId, options: healthCheck)).FirstOrDefault(); retryCount++; } - if (string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase)) - { - executionContext.Output($"{container.ContainerNetworkAlias} service is healthy."); - } - else - { - throw new InvalidOperationException($"Failed to initialize, {container.ContainerNetworkAlias} service is {serviceHealth}."); - } + return string.Equals(serviceHealth, "healthy", StringComparison.OrdinalIgnoreCase); } private async Task ContainerRegistryLogin(IExecutionContext executionContext, ContainerInfo container) diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs new file mode 100644 index 00000000000..e0819511bc7 --- /dev/null +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -0,0 +1,126 @@ +using GitHub.Runner.Worker; +using GitHub.Runner.Worker.Container; +using Xunit; +using Moq; +using GitHub.Runner.Worker.Container.ContainerHooks; +using System.Threading.Tasks; +using System.Collections.Generic; +using System.Runtime.CompilerServices; +using GitHub.DistributedTask.WebApi; +using System; + +namespace GitHub.Runner.Common.Tests.Worker +{ + + public sealed class ContainerOperationProviderL0 + { + + private TestHostContext _hc; + private Mock _ec; + private Mock _dockerManager; + private Mock _containerHookManager; + private ContainerOperationProvider containerOperationProvider; + private Mock serverQueue; + private Mock pagingLogger; + private List healthyDockerStatus = new List { "healthy" }; + private List emptyDockerStatus = new List { string.Empty }; + private List unhealthyDockerStatus = new List { "unhealthy" }; + private List dockerLogs = new List { "log1", "log2", "log3" }; + + List containers = new List(); + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertFailedTask() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); + + //Act + try + { + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + } + catch (InvalidOperationException) + { + + //Assert + Assert.Equal(TaskResult.Failed, _ec.Object.Result ?? TaskResult.Failed); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_UnhealthyServiceContainer_AssertExceptionThrown() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(unhealthyDockerStatus)); + + //Act and Assert + await Assert.ThrowsAsync(() => containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers)); + + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_healthyServiceContainer_AssertSucceededTask() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(healthyDockerStatus)); + + //Act + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + + //Assert + Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async void RunServiceContainersHealthcheck_healthyServiceContainerWithoutHealthcheck_AssertSucceededTask() + { + //Arrange + Setup(); + _dockerManager.Setup(x => x.DockerInspect(_ec.Object, It.IsAny(), It.IsAny())).Returns(Task.FromResult(emptyDockerStatus)); + + //Act + await containerOperationProvider.RunContainersHealthcheck(_ec.Object, containers); + + //Assert + Assert.Equal(TaskResult.Succeeded, _ec.Object.Result ?? TaskResult.Succeeded); + + } + + private void Setup([CallerMemberName] string testName = "") + { + containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); + _hc = new TestHostContext(this, testName); + _ec = new Mock(); + serverQueue = new Mock(); + pagingLogger = new Mock(); + + _dockerManager = new Mock(); + _containerHookManager = new Mock(); + containerOperationProvider = new ContainerOperationProvider(); + + _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(serverQueue.Object); + _hc.SetSingleton(pagingLogger.Object); + + _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(_containerHookManager.Object); + + _ec.Setup(x => x.Global).Returns(new GlobalContext()); + + containerOperationProvider.Initialize(_hc); + } + } +} \ No newline at end of file From e5b069bb324719bcff85387f8d54b0f23d7acb14 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Thu, 13 Oct 2022 12:40:12 +0200 Subject: [PATCH 035/145] Allow '--disableupdate' in create-latest-svc.sh (#2201) * Allow '--disableupdate' in create-latest-svc.sh * Echo '--disableupdate' when used --- scripts/create-latest-svc.sh | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/scripts/create-latest-svc.sh b/scripts/create-latest-svc.sh index 5a5c7831e42..b6eca26e7a8 100755 --- a/scripts/create-latest-svc.sh +++ b/scripts/create-latest-svc.sh @@ -13,7 +13,7 @@ set -e flags_found=false -while getopts 's:g:n:r:u:l:' opt; do +while getopts 's:g:n:r:u:l:d' opt; do flags_found=true case $opt in @@ -35,6 +35,9 @@ while getopts 's:g:n:r:u:l:' opt; do l) labels=$OPTARG ;; + d) + disableupdate='true' + ;; *) echo " Runner Service Installer @@ -49,7 +52,8 @@ Usage: -n optional name of the runner, defaults to hostname -r optional name of the runner group to add the runner to, defaults to the Default group -u optional user svc will run as, defaults to current - -l optional list of labels (split by comma) applied on the runner" + -l optional list of labels (split by comma) applied on the runner + -d optional allow runner to remain on the current version for one month after the release of a newer version" exit 0 ;; esac @@ -169,8 +173,8 @@ fi echo echo "Configuring ${runner_name} @ $runner_url" -echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup \"$runner_group\"}" -sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup "$runner_group"} +echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup \"$runner_group\"} ${disableupdate:+--disableupdate}" +sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup "$runner_group"} ${disableupdate:+--disableupdate} #--------------------------------------- # Configuring as a service From c0e9d5313f72371628d831a6336fa1ded60a6faf Mon Sep 17 00:00:00 2001 From: Ben Sherman <3365+bensherman@users.noreply.github.com> Date: Thu, 13 Oct 2022 07:56:10 -0700 Subject: [PATCH 036/145] Fix markup for support link (#2114) --- docs/checks/actions.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/checks/actions.md b/docs/checks/actions.md index 7114ee443c9..6924d1f9ec7 100644 --- a/docs/checks/actions.md +++ b/docs/checks/actions.md @@ -64,4 +64,4 @@ Make sure the runner has access to actions service for GitHub.com or GitHub Ente ## Still not working? -Contact [GitHub Support](https://support.github.com] if you have further questuons, or log an issue at https://github.com/actions/runner if you think it's a runner issue. +Contact [GitHub Support](https://support.github.com) if you have further questuons, or log an issue at https://github.com/actions/runner if you think it's a runner issue. From 1b149c28934c4ace586bdd86985069a19b7b666a Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Mon, 17 Oct 2022 14:03:24 +0200 Subject: [PATCH 037/145] Add runner devcontainer (#2187) Add runner devcontainer.json definition. Sets up dotnet, docker-in-docker, omnisharp and some vscode extensions --- .devcontainer/devcontainer.json | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000000..2981e97e275 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,24 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +{ + "name": "Actions Runner Devcontainer", + "image": "mcr.microsoft.com/devcontainers/base:focal", + "features": { + "ghcr.io/devcontainers/features/docker-in-docker:1": {}, + "ghcr.io/devcontainers/features/dotnet": { + "version": "6.0.300" + } + }, + "customizations": { + "vscode": { + "extensions": [ + "ms-azuretools.vscode-docker", + "ms-dotnettools.csharp", + "eamodio.gitlens" + ] + } + }, + // dotnet restore to install dependencies so OmniSharp works out of the box + // src/Test restores all other projects it references, src/Runner.PluginHost is not one of them + "postCreateCommand": "dotnet restore src/Test && dotnet restore src/Runner.PluginHost", + "remoteUser": "vscode" +} \ No newline at end of file From 2efddc2fec61727b86285180d195f793cf1e3c3b Mon Sep 17 00:00:00 2001 From: Cory Miller <13227161+cory-miller@users.noreply.github.com> Date: Tue, 18 Oct 2022 10:54:08 -0400 Subject: [PATCH 038/145] Fix IDE0090 (#2211) --- src/Runner.Common/ActionCommand.cs | 2 +- src/Runner.Common/ConfigurationStore.cs | 4 +- src/Runner.Common/ExtensionManager.cs | 2 +- src/Runner.Common/HostContext.cs | 12 ++--- src/Runner.Common/HostTraceListener.cs | 2 +- src/Runner.Common/JobServerQueue.cs | 30 +++++------ src/Runner.Common/ProcessChannel.cs | 2 +- src/Runner.Common/ProcessExtensions.cs | 6 +-- src/Runner.Common/Terminal.cs | 2 +- src/Runner.Common/TraceManager.cs | 2 +- src/Runner.Listener/Checks/CheckUtil.cs | 4 +- src/Runner.Listener/Checks/NodeJsCheck.cs | 2 +- src/Runner.Listener/CommandSettings.cs | 6 +-- .../Configuration/ConfigurationManager.cs | 4 +- .../Configuration/CredentialManager.cs | 2 +- .../Configuration/CredentialProvider.cs | 2 +- .../Configuration/ServiceControlManager.cs | 2 +- src/Runner.Listener/JobDispatcher.cs | 22 ++++---- src/Runner.Listener/MessageListener.cs | 4 +- src/Runner.Listener/Program.cs | 2 +- src/Runner.Listener/Runner.cs | 2 +- src/Runner.Listener/SelfUpdater.cs | 14 +++--- src/Runner.PluginHost/Program.cs | 2 +- .../Artifact/DownloadArtifact.cs | 2 +- .../Artifact/FileContainerServer.cs | 26 +++++----- .../Artifact/PublishArtifact.cs | 4 +- .../Repository/GitCliManager.cs | 18 +++---- .../Repository/v1.0/GitSourceProvider.cs | 18 +++---- .../Repository/v1.0/RepositoryPlugin.cs | 2 +- .../Repository/v1.1/GitSourceProvider.cs | 24 ++++----- src/Runner.Sdk/ActionPlugin.cs | 4 +- src/Runner.Sdk/ProcessInvoker.cs | 16 +++--- src/Runner.Sdk/RunnerWebProxy.cs | 6 +-- src/Runner.Sdk/Util/IOUtil.cs | 8 +-- src/Runner.Sdk/Util/StringUtil.cs | 2 +- src/Runner.Sdk/Util/UrlUtil.cs | 2 +- src/Runner.Sdk/Util/VssUtil.cs | 4 +- src/Runner.Worker/ActionCommandManager.cs | 8 +-- src/Runner.Worker/ActionManager.cs | 10 ++-- src/Runner.Worker/ActionManifestManager.cs | 2 +- src/Runner.Worker/ConditionTraceWriter.cs | 2 +- src/Runner.Worker/Container/ContainerInfo.cs | 2 +- .../Container/DockerCommandManager.cs | 6 +-- src/Runner.Worker/Container/DockerUtil.cs | 6 +-- .../ContainerOperationProvider.cs | 6 +-- src/Runner.Worker/ExecutionContext.cs | 10 ++-- .../Expressions/HashFilesFunction.cs | 2 +- src/Runner.Worker/GitHubContext.cs | 2 +- src/Runner.Worker/Handlers/OutputManager.cs | 6 +-- .../Handlers/ScriptHandlerHelpers.cs | 4 +- src/Runner.Worker/JobExtension.cs | 12 ++--- src/Runner.Worker/Program.cs | 2 +- src/Runner.Worker/RunnerPluginManager.cs | 4 +- src/Runner.Worker/StepsContext.cs | 4 +- src/Runner.Worker/TrackingManager.cs | 2 +- src/Runner.Worker/Variables.cs | 6 +-- src/Runner.Worker/Worker.cs | 4 +- src/Test/L0/CommandLineParserL0.cs | 12 ++--- src/Test/L0/ConstantGenerationL0.cs | 2 +- src/Test/L0/Container/ContainerInfoL0.cs | 8 +-- src/Test/L0/DotnetsdkDownloadScriptL0.cs | 4 +- src/Test/L0/ExtensionManagerL0.cs | 4 +- src/Test/L0/Listener/CommandSettingsL0.cs | 4 +- .../Configuration/ArgumentValidatorTestsL0.cs | 6 +-- .../Configuration/ConfigurationManagerL0.cs | 4 +- .../Configuration/PromptManagerTestsL0.cs | 18 +++---- .../Configuration/RunnerCredentialL0.cs | 2 +- src/Test/L0/Listener/JobDispatcherL0.cs | 50 +++++++++---------- src/Test/L0/Listener/MessageListenerL0.cs | 12 ++--- src/Test/L0/Listener/RunnerL0.cs | 4 +- src/Test/L0/Listener/SelfUpdaterL0.cs | 6 +-- src/Test/L0/PackagesTrimL0.cs | 12 ++--- src/Test/L0/PagingLoggerL0.cs | 2 +- src/Test/L0/ProcessExtensionL0.cs | 2 +- src/Test/L0/ProcessInvokerL0.cs | 26 +++++----- src/Test/L0/RunnerWebProxyL0.cs | 10 ++-- src/Test/L0/ServiceControlManagerL0.cs | 14 +++--- src/Test/L0/TestHostContext.cs | 10 ++-- src/Test/L0/Util/ArgUtilL0.cs | 24 ++++----- src/Test/L0/Util/IOUtilL0.cs | 40 +++++++-------- src/Test/L0/Util/StringUtilL0.cs | 12 ++--- src/Test/L0/Util/TaskResultUtilL0.cs | 4 +- src/Test/L0/Util/VssUtilL0.cs | 2 +- src/Test/L0/Util/WhichUtilL0.cs | 8 +-- src/Test/L0/Worker/ActionCommandL0.cs | 4 +- src/Test/L0/Worker/ActionCommandManagerL0.cs | 4 +- src/Test/L0/Worker/ActionManagerL0.cs | 26 +++++----- src/Test/L0/Worker/ActionRunnerL0.cs | 10 ++-- .../L0/Worker/ContainerOperationProviderL0.cs | 10 ++-- .../L0/Worker/CreateStepSummaryCommandL0.cs | 4 +- src/Test/L0/Worker/ExecutionContextL0.cs | 48 +++++++++--------- src/Test/L0/Worker/HandlerFactoryL0.cs | 2 +- src/Test/L0/Worker/JobExtensionL0.cs | 6 +-- src/Test/L0/Worker/JobRunnerL0.cs | 4 +- .../L0/Worker/PipelineDirectoryManagerL0.cs | 4 +- src/Test/L0/Worker/StepsRunnerL0.cs | 2 +- src/Test/L0/Worker/TrackingManagerL0.cs | 6 +-- src/Test/L0/Worker/VariablesL0.cs | 16 +++--- src/Test/L0/Worker/WorkerL0.cs | 8 +-- 99 files changed, 412 insertions(+), 412 deletions(-) diff --git a/src/Runner.Common/ActionCommand.cs b/src/Runner.Common/ActionCommand.cs index 68ded964c57..0ed1362acd8 100644 --- a/src/Runner.Common/ActionCommand.cs +++ b/src/Runner.Common/ActionCommand.cs @@ -31,7 +31,7 @@ public sealed class ActionCommand new EscapeMapping(token: "%", replacement: "%25"), }; - private readonly Dictionary _properties = new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _properties = new(StringComparer.OrdinalIgnoreCase); public const string Prefix = "##["; public const string _commandKey = "::"; diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index 7cce70f601d..b3cb26a86be 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -74,12 +74,12 @@ public string RepoOrOrgName { get { - Uri accountUri = new Uri(this.ServerUrl); + Uri accountUri = new(this.ServerUrl); string repoOrOrgName = string.Empty; if (accountUri.Host.EndsWith(".githubusercontent.com", StringComparison.OrdinalIgnoreCase)) { - Uri gitHubUrl = new Uri(this.GitHubUrl); + Uri gitHubUrl = new(this.GitHubUrl); // Use the "NWO part" from the GitHub URL path repoOrOrgName = gitHubUrl.AbsolutePath.Trim('/'); diff --git a/src/Runner.Common/ExtensionManager.cs b/src/Runner.Common/ExtensionManager.cs index 6f1bfc5d7b6..340b5267625 100644 --- a/src/Runner.Common/ExtensionManager.cs +++ b/src/Runner.Common/ExtensionManager.cs @@ -14,7 +14,7 @@ public interface IExtensionManager : IRunnerService public sealed class ExtensionManager : RunnerService, IExtensionManager { - private readonly ConcurrentDictionary> _cache = new ConcurrentDictionary>(); + private readonly ConcurrentDictionary> _cache = new(); public List GetExtensions() where T : class, IExtension { diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index e8eab19891a..9d5aefdf8c6 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -51,12 +51,12 @@ public sealed class HostContext : EventListener, IObserver, private static int _defaultLogRetentionDays = 30; private static int[] _vssHttpMethodEventIds = new int[] { 1, 2, 3, 4, 5, 6, 7, 8, 9, 24 }; private static int[] _vssHttpCredentialEventIds = new int[] { 11, 13, 14, 15, 16, 17, 18, 20, 21, 22, 27, 29 }; - private readonly ConcurrentDictionary _serviceInstances = new ConcurrentDictionary(); - private readonly ConcurrentDictionary _serviceTypes = new ConcurrentDictionary(); + private readonly ConcurrentDictionary _serviceInstances = new(); + private readonly ConcurrentDictionary _serviceTypes = new(); private readonly ISecretMasker _secretMasker = new SecretMasker(); - private readonly List _userAgents = new List() { new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version) }; - private CancellationTokenSource _runnerShutdownTokenSource = new CancellationTokenSource(); - private object _perfLock = new object(); + private readonly List _userAgents = new() { new ProductInfoHeaderValue($"GitHubActionsRunner-{BuildConstants.RunnerPackage.PackageName}", BuildConstants.RunnerPackage.Version) }; + private CancellationTokenSource _runnerShutdownTokenSource = new(); + private object _perfLock = new(); private Tracing _trace; private Tracing _actionsHttpTrace; private Tracing _netcoreHttpTrace; @@ -66,7 +66,7 @@ public sealed class HostContext : EventListener, IObserver, private IDisposable _diagListenerSubscription; private StartupType _startupType; private string _perfFile; - private RunnerWebProxy _webProxy = new RunnerWebProxy(); + private RunnerWebProxy _webProxy = new(); public event EventHandler Unloading; public CancellationToken RunnerShutdownToken => _runnerShutdownTokenSource.Token; diff --git a/src/Runner.Common/HostTraceListener.cs b/src/Runner.Common/HostTraceListener.cs index e022eb0e47b..90ef9938ae9 100644 --- a/src/Runner.Common/HostTraceListener.cs +++ b/src/Runner.Common/HostTraceListener.cs @@ -164,7 +164,7 @@ private StreamWriter CreatePageLogWriter() { if (_enableLogRetention) { - DirectoryInfo diags = new DirectoryInfo(_logFileDirectory); + DirectoryInfo diags = new(_logFileDirectory); var logs = diags.GetFiles($"{_logFilePrefix}*.log"); foreach (var log in logs) { diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 3da4400a9a6..d84614c15c3 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -39,19 +39,19 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private Guid _jobTimelineRecordId; // queue for web console line - private readonly ConcurrentQueue _webConsoleLineQueue = new ConcurrentQueue(); + private readonly ConcurrentQueue _webConsoleLineQueue = new(); // queue for file upload (log file or attachment) - private readonly ConcurrentQueue _fileUploadQueue = new ConcurrentQueue(); + private readonly ConcurrentQueue _fileUploadQueue = new(); // queue for timeline or timeline record update (one queue per timeline) - private readonly ConcurrentDictionary> _timelineUpdateQueue = new ConcurrentDictionary>(); + private readonly ConcurrentDictionary> _timelineUpdateQueue = new(); // indicate how many timelines we have, we will process _timelineUpdateQueue base on the order of timeline in this list - private readonly List _allTimelines = new List(); + private readonly List _allTimelines = new(); // bufferd timeline records that fail to update - private readonly Dictionary> _bufferedRetryRecords = new Dictionary>(); + private readonly Dictionary> _bufferedRetryRecords = new(); // Task for each queue's dequeue process private Task _webConsoleLineDequeueTask; @@ -61,8 +61,8 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // common private IJobServer _jobServer; private Task[] _allDequeueTasks; - private readonly TaskCompletionSource _jobCompletionSource = new TaskCompletionSource(); - private readonly TaskCompletionSource _jobRecordUpdated = new TaskCompletionSource(); + private readonly TaskCompletionSource _jobCompletionSource = new(); + private readonly TaskCompletionSource _jobRecordUpdated = new(); private bool _queueInProcess = false; public TaskCompletionSource JobRecordUpdated => _jobRecordUpdated; @@ -237,8 +237,8 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) } // Group consolelines by timeline record of each step - Dictionary> stepsConsoleLines = new Dictionary>(); - List stepRecordIds = new List(); // We need to keep lines in order + Dictionary> stepsConsoleLines = new(); + List stepRecordIds = new(); // We need to keep lines in order int linesCounter = 0; ConsoleLineInfo lineInfo; while (_webConsoleLineQueue.TryDequeue(out lineInfo)) @@ -264,7 +264,7 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { // Split consolelines into batch, each batch will container at most 100 lines. int batchCounter = 0; - List> batchedLines = new List>(); + List> batchedLines = new(); foreach (var line in stepsConsoleLines[stepRecordId]) { var currentBatch = batchedLines.ElementAtOrDefault(batchCounter); @@ -338,7 +338,7 @@ private async Task ProcessFilesUploadQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { - List filesToUpload = new List(); + List filesToUpload = new(); UploadFileInfo dequeueFile; while (_fileUploadQueue.TryDequeue(out dequeueFile)) { @@ -398,13 +398,13 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) { - List pendingUpdates = new List(); + List pendingUpdates = new(); foreach (var timeline in _allTimelines) { ConcurrentQueue recordQueue; if (_timelineUpdateQueue.TryGetValue(timeline, out recordQueue)) { - List records = new List(); + List records = new(); TimelineRecord record; while (recordQueue.TryDequeue(out record)) { @@ -426,7 +426,7 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) // we need track whether we have new sub-timeline been created on the last run. // if so, we need continue update timeline record even we on the last run. bool pendingSubtimelineUpdate = false; - List mainTimelineRecordsUpdateErrors = new List(); + List mainTimelineRecordsUpdateErrors = new(); if (pendingUpdates.Count > 0) { foreach (var update in pendingUpdates) @@ -529,7 +529,7 @@ private List MergeTimelineRecords(List timelineR return timelineRecords; } - Dictionary dict = new Dictionary(); + Dictionary dict = new(); foreach (TimelineRecord rec in timelineRecords) { if (rec == null) diff --git a/src/Runner.Common/ProcessChannel.cs b/src/Runner.Common/ProcessChannel.cs index ca493c9628e..d0e574681bd 100644 --- a/src/Runner.Common/ProcessChannel.cs +++ b/src/Runner.Common/ProcessChannel.cs @@ -76,7 +76,7 @@ public async Task SendAsync(MessageType messageType, string body, CancellationTo public async Task ReceiveAsync(CancellationToken cancellationToken) { - WorkerMessage result = new WorkerMessage(MessageType.NotInitialized, string.Empty); + WorkerMessage result = new(MessageType.NotInitialized, string.Empty); result.MessageType = (MessageType)await _readStream.ReadInt32Async(cancellationToken); result.Body = await _readStream.ReadStringAsync(cancellationToken); Trace.Info($"Receiving message of length {result.Body.Length}, with hash '{IOUtil.GetSha256Hash(result.Body)}'"); diff --git a/src/Runner.Common/ProcessExtensions.cs b/src/Runner.Common/ProcessExtensions.cs index 5e3bbd35ba1..5ccda3e8abf 100644 --- a/src/Runner.Common/ProcessExtensions.cs +++ b/src/Runner.Common/ProcessExtensions.cs @@ -291,7 +291,7 @@ public static class LinuxProcessExtensions public static string GetEnvironmentVariable(this Process process, IHostContext hostContext, string variable) { var trace = hostContext.GetTrace(nameof(LinuxProcessExtensions)); - Dictionary env = new Dictionary(); + Dictionary env = new(); if (Directory.Exists("/proc")) { @@ -322,8 +322,8 @@ public static string GetEnvironmentVariable(this Process process, IHostContext h // It doesn't escape '=' or ' ', so we can't parse the output into a dictionary of all envs. // So we only look for the env you request, in the format of variable=value. (it won't work if you variable contains = or space) trace.Info($"Read env from output of `ps e -p {process.Id} -o command`"); - List psOut = new List(); - object outputLock = new object(); + List psOut = new(); + object outputLock = new(); using (var p = hostContext.CreateService()) { p.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs stdout) diff --git a/src/Runner.Common/Terminal.cs b/src/Runner.Common/Terminal.cs index cf52d4154d9..bade576a095 100644 --- a/src/Runner.Common/Terminal.cs +++ b/src/Runner.Common/Terminal.cs @@ -81,7 +81,7 @@ public string ReadSecret() } // Trace whether a value was entered. - string val = new String(chars.ToArray()); + string val = new(chars.ToArray()); if (!string.IsNullOrEmpty(val)) { HostContext.SecretMasker.AddValue(val); diff --git a/src/Runner.Common/TraceManager.cs b/src/Runner.Common/TraceManager.cs index ebbaefa89ed..f935aaf8bbe 100644 --- a/src/Runner.Common/TraceManager.cs +++ b/src/Runner.Common/TraceManager.cs @@ -14,7 +14,7 @@ public interface ITraceManager : IDisposable public sealed class TraceManager : ITraceManager { - private readonly ConcurrentDictionary _sources = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _sources = new(StringComparer.OrdinalIgnoreCase); private readonly HostTraceListener _hostTraceListener; private TraceSetting _traceSetting; private ISecretMasker _secretMasker; diff --git a/src/Runner.Listener/Checks/CheckUtil.cs b/src/Runner.Listener/Checks/CheckUtil.cs index b2e58b41c73..0802e11b024 100644 --- a/src/Runner.Listener/Checks/CheckUtil.cs +++ b/src/Runner.Listener/Checks/CheckUtil.cs @@ -347,8 +347,8 @@ await processInvoker.ExecuteAsync( public sealed class HttpEventSourceListener : EventListener { private readonly List _logs; - private readonly object _lock = new object(); - private readonly Dictionary> _ignoredEvent = new Dictionary> + private readonly object _lock = new(); + private readonly Dictionary> _ignoredEvent = new() { { "Microsoft-System-Net-Http", diff --git a/src/Runner.Listener/Checks/NodeJsCheck.cs b/src/Runner.Listener/Checks/NodeJsCheck.cs index 29fb282e07b..df750fe8bd2 100644 --- a/src/Runner.Listener/Checks/NodeJsCheck.cs +++ b/src/Runner.Listener/Checks/NodeJsCheck.cs @@ -86,7 +86,7 @@ private async Task CheckNodeJs(string url, string pat, bool extraCA result.Logs.Add($"{DateTime.UtcNow.ToString("O")} ***************************************************************************************************************"); // Request to github.com or ghes server - Uri requestUrl = new Uri(url); + Uri requestUrl = new(url); var env = new Dictionary() { { "HOSTNAME", requestUrl.Host }, diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index ca0ea1b7da5..44039bb5cb8 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -11,7 +11,7 @@ namespace GitHub.Runner.Listener { public sealed class CommandSettings { - private readonly Dictionary _envArgs = new Dictionary(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary _envArgs = new(StringComparer.OrdinalIgnoreCase); private readonly CommandLineParser _parser; private readonly IPromptManager _promptManager; private readonly Tracing _trace; @@ -26,7 +26,7 @@ public sealed class CommandSettings }; // Valid flags and args for specific command - key: command, value: array of valid flags and args - private readonly Dictionary validOptions = new Dictionary + private readonly Dictionary validOptions = new() { // Valid configure flags and args [Constants.Runner.CommandLine.Commands.Configure] = @@ -137,7 +137,7 @@ public CommandSettings(IHostContext context, string[] args) // Validate commandline parser result public List Validate() { - List unknowns = new List(); + List unknowns = new(); // detect unknown commands unknowns.AddRange(_parser.Commands.Where(x => !validOptions.Keys.Contains(x, StringComparer.OrdinalIgnoreCase))); diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 410bec8f6e8..72b574873ea 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -86,7 +86,7 @@ public async Task ConfigureAsync(CommandSettings command) throw new InvalidOperationException("Cannot configure the runner because it is already configured. To reconfigure the runner, run 'config.cmd remove' or './config.sh remove' first."); } - RunnerSettings runnerSettings = new RunnerSettings(); + RunnerSettings runnerSettings = new(); // Loop getting url and creds until you can connect ICredentialProvider credProvider = null; @@ -521,7 +521,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate) { - TaskAgent agent = new TaskAgent(agentName) + TaskAgent agent = new(agentName) { Authorization = new TaskAgentAuthorization { diff --git a/src/Runner.Listener/Configuration/CredentialManager.cs b/src/Runner.Listener/Configuration/CredentialManager.cs index e13d2951012..327961794e5 100644 --- a/src/Runner.Listener/Configuration/CredentialManager.cs +++ b/src/Runner.Listener/Configuration/CredentialManager.cs @@ -18,7 +18,7 @@ public interface ICredentialManager : IRunnerService public class CredentialManager : RunnerService, ICredentialManager { - public static readonly Dictionary CredentialTypes = new Dictionary(StringComparer.OrdinalIgnoreCase) + public static readonly Dictionary CredentialTypes = new(StringComparer.OrdinalIgnoreCase) { { Constants.Configuration.OAuth, typeof(OAuthCredential)}, { Constants.Configuration.OAuthAccessToken, typeof(OAuthAccessTokenCredential)}, diff --git a/src/Runner.Listener/Configuration/CredentialProvider.cs b/src/Runner.Listener/Configuration/CredentialProvider.cs index 304c1177006..def579a0daf 100644 --- a/src/Runner.Listener/Configuration/CredentialProvider.cs +++ b/src/Runner.Listener/Configuration/CredentialProvider.cs @@ -48,7 +48,7 @@ public override VssCredentials GetVssCredentials(IHostContext context) ArgUtil.NotNullOrEmpty(token, nameof(token)); trace.Info("token retrieved: {0} chars", token.Length); - VssCredentials creds = new VssCredentials(new VssOAuthAccessTokenCredential(token), CredentialPromptType.DoNotPrompt); + VssCredentials creds = new(new VssOAuthAccessTokenCredential(token), CredentialPromptType.DoNotPrompt); trace.Info("cred created"); return creds; diff --git a/src/Runner.Listener/Configuration/ServiceControlManager.cs b/src/Runner.Listener/Configuration/ServiceControlManager.cs index 9349484c5ee..e8a749484c6 100644 --- a/src/Runner.Listener/Configuration/ServiceControlManager.cs +++ b/src/Runner.Listener/Configuration/ServiceControlManager.cs @@ -44,7 +44,7 @@ public void CalculateServiceName(RunnerSettings settings, string serviceNamePatt } // For the service name, replace any characters outside of the alpha-numeric set and ".", "_", "-" with "-" - Regex regex = new Regex(@"[^0-9a-zA-Z._\-]"); + Regex regex = new(@"[^0-9a-zA-Z._\-]"); string repoOrOrgName = regex.Replace(settings.RepoOrOrgName, "-"); serviceName = StringUtil.Format(serviceNamePattern, repoOrOrgName, settings.AgentName); diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index 9b4b2d6f477..1e5306bd0af 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -37,8 +37,8 @@ public interface IJobDispatcher : IRunnerService // and the server will not send another job while this one is still running. public sealed class JobDispatcher : RunnerService, IJobDispatcher { - private static Regex _invalidJsonRegex = new Regex(@"invalid\ Json\ at\ position\ '(\d+)':", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private readonly Lazy> _localRunJobResult = new Lazy>(); + private static Regex _invalidJsonRegex = new(@"invalid\ Json\ at\ position\ '(\d+)':", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private readonly Lazy> _localRunJobResult = new(); private int _poolId; IConfigurationStore _configurationStore; @@ -47,14 +47,14 @@ public sealed class JobDispatcher : RunnerService, IJobDispatcher private static readonly string _workerProcessName = $"Runner.Worker{IOUtil.ExeExtension}"; // this is not thread-safe - private readonly Queue _jobDispatchedQueue = new Queue(); - private readonly ConcurrentDictionary _jobInfos = new ConcurrentDictionary(); + private readonly Queue _jobDispatchedQueue = new(); + private readonly ConcurrentDictionary _jobInfos = new(); // allow up to 30sec for any data to be transmitted over the process channel // timeout limit can be overwritten by environment GITHUB_ACTIONS_RUNNER_CHANNEL_TIMEOUT private TimeSpan _channelTimeout; - private TaskCompletionSource _runOnceJobCompleted = new TaskCompletionSource(); + private TaskCompletionSource _runOnceJobCompleted = new(); public event EventHandler JobStatus; @@ -111,7 +111,7 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce } } - WorkerDispatcher newDispatch = new WorkerDispatcher(jobRequestMessage.JobId, jobRequestMessage.RequestId); + WorkerDispatcher newDispatch = new(jobRequestMessage.JobId, jobRequestMessage.RequestId); if (runOnce) { Trace.Info("Start dispatcher for one time used runner."); @@ -357,7 +357,7 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc term.WriteLine($"{DateTime.UtcNow:u}: Running job: {message.JobDisplayName}"); // first job request renew succeed. - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); + TaskCompletionSource firstJobRequestRenewed = new(); var notification = HostContext.GetService(); // lock renew cancellation token. @@ -398,8 +398,8 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc HostContext.WritePerfCounter($"JobRequestRenewed_{requestId.ToString()}"); Task workerProcessTask = null; - object _outputLock = new object(); - List workerOutput = new List(); + object _outputLock = new(); + List workerOutput = new(); using (var processChannel = HostContext.CreateService()) using (var processInvoker = HostContext.CreateService()) { @@ -936,7 +936,7 @@ private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequest var runnerServer = HostContext.GetService(); int completeJobRequestRetryLimit = 5; - List exceptions = new List(); + List exceptions = new(); while (completeJobRequestRetryLimit-- > 0) { try @@ -1039,7 +1039,7 @@ private class WorkerDispatcher : IDisposable public Task WorkerDispatch { get; set; } public CancellationTokenSource WorkerCancellationTokenSource { get; private set; } public CancellationTokenSource WorkerCancelTimeoutKillTokenSource { get; private set; } - private readonly object _lock = new object(); + private readonly object _lock = new(); public WorkerDispatcher(Guid jobId, long requestId) { diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index acea728c7c5..274967ac36f 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -38,7 +38,7 @@ public sealed class MessageListener : RunnerService, IMessageListener private readonly TimeSpan _sessionCreationRetryInterval = TimeSpan.FromSeconds(30); private readonly TimeSpan _sessionConflictRetryLimit = TimeSpan.FromMinutes(4); private readonly TimeSpan _clockSkewRetryLimit = TimeSpan.FromMinutes(30); - private readonly Dictionary _sessionCreationExceptionTracker = new Dictionary(); + private readonly Dictionary _sessionCreationExceptionTracker = new(); private TaskAgentStatus runnerStatus = TaskAgentStatus.Online; private CancellationTokenSource _getMessagesTokenSource; @@ -198,7 +198,7 @@ public async Task GetNextMessageAsync(CancellationToken token) bool encounteringError = false; int continuousError = 0; string errorMessage = string.Empty; - Stopwatch heartbeat = new Stopwatch(); + Stopwatch heartbeat = new(); heartbeat.Restart(); while (true) { diff --git a/src/Runner.Listener/Program.cs b/src/Runner.Listener/Program.cs index 1482600ce95..d4d5e43eb34 100644 --- a/src/Runner.Listener/Program.cs +++ b/src/Runner.Listener/Program.cs @@ -16,7 +16,7 @@ public static int Main(string[] args) // Add environment variables from .env file LoadAndSetEnv(); - using (HostContext context = new HostContext("Runner")) + using (HostContext context = new("Runner")) { return MainAsync(context, args).GetAwaiter().GetResult(); } diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 20235d51d78..b6f9946f797 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -28,7 +28,7 @@ public sealed class Runner : RunnerService, IRunner private IMessageListener _listener; private ITerminal _term; private bool _inConfigStage; - private ManualResetEvent _completedCommand = new ManualResetEvent(false); + private ManualResetEvent _completedCommand = new(false); public override void Initialize(IHostContext hostContext) { diff --git a/src/Runner.Listener/SelfUpdater.cs b/src/Runner.Listener/SelfUpdater.cs index 07db370f634..23ba0a0cff6 100644 --- a/src/Runner.Listener/SelfUpdater.cs +++ b/src/Runner.Listener/SelfUpdater.cs @@ -32,14 +32,14 @@ public class SelfUpdater : RunnerService, ISelfUpdater private static string _platform = BuildConstants.RunnerPackage.PackageName; private static string _dotnetRuntime = "dotnetRuntime"; private static string _externals = "externals"; - private readonly Dictionary _contentHashes = new Dictionary(); + private readonly Dictionary _contentHashes = new(); private PackageMetadata _targetPackage; private ITerminal _terminal; private IRunnerServer _runnerServer; private int _poolId; private int _agentId; - private readonly ConcurrentQueue _updateTrace = new ConcurrentQueue(); + private readonly ConcurrentQueue _updateTrace = new(); private Task _cloneAndCalculateContentHashTask; private string _dotnetRuntimeCloneDirectory; private string _externalsCloneDirectory; @@ -134,7 +134,7 @@ public async Task SelfUpdate(AgentRefreshMessage updateMessage, IJobDispat string flagFile = "update.finished"; IOUtil.DeleteFile(flagFile); // kick off update script - Process invokeScript = new Process(); + Process invokeScript = new(); #if OS_WINDOWS invokeScript.StartInfo.FileName = WhichUtil.Which("cmd.exe", trace: Trace); invokeScript.StartInfo.Arguments = $"/c \"{updateScript}\""; @@ -191,9 +191,9 @@ private async Task UpdateNeeded(string targetVersion, CancellationToken to } Trace.Info($"Version '{_targetPackage.Version}' of '{_targetPackage.Type}' package available in server."); - PackageVersion serverVersion = new PackageVersion(_targetPackage.Version); + PackageVersion serverVersion = new(_targetPackage.Version); Trace.Info($"Current running runner version is {BuildConstants.RunnerPackage.Version}"); - PackageVersion runnerVersion = new PackageVersion(BuildConstants.RunnerPackage.Version); + PackageVersion runnerVersion = new(BuildConstants.RunnerPackage.Version); return serverVersion.CompareTo(runnerVersion) > 0; } @@ -476,7 +476,7 @@ private async Task DownLoadRunner(string downloadDirectory, string packa long downloadSize = 0; //open zip stream in async mode - using (HttpClient httpClient = new HttpClient(HostContext.CreateHttpClientHandler())) + using (HttpClient httpClient = new(HostContext.CreateHttpClientHandler())) { if (!string.IsNullOrEmpty(_targetPackage.Token)) { @@ -486,7 +486,7 @@ private async Task DownLoadRunner(string downloadDirectory, string packa Trace.Info($"Downloading {packageDownloadUrl}"); - using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) + using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: 4096, useAsync: true)) using (Stream result = await httpClient.GetStreamAsync(packageDownloadUrl)) { //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). diff --git a/src/Runner.PluginHost/Program.cs b/src/Runner.PluginHost/Program.cs index 2496132586c..1cacb6a2802 100644 --- a/src/Runner.PluginHost/Program.cs +++ b/src/Runner.PluginHost/Program.cs @@ -12,7 +12,7 @@ namespace GitHub.Runner.PluginHost { public static class Program { - private static CancellationTokenSource tokenSource = new CancellationTokenSource(); + private static CancellationTokenSource tokenSource = new(); private static string executingAssemblyLocation = string.Empty; public static int Main(string[] args) diff --git a/src/Runner.Plugins/Artifact/DownloadArtifact.cs b/src/Runner.Plugins/Artifact/DownloadArtifact.cs index 7e5f6e27753..81f654cfb74 100644 --- a/src/Runner.Plugins/Artifact/DownloadArtifact.cs +++ b/src/Runner.Plugins/Artifact/DownloadArtifact.cs @@ -63,7 +63,7 @@ public async Task RunAsync( string containerPath = actionsStorageArtifact.Name; // In actions storage artifacts, name equals the path long containerId = actionsStorageArtifact.ContainerId; - FileContainerServer fileContainerServer = new FileContainerServer(context.VssConnection, projectId: new Guid(), containerId, containerPath); + FileContainerServer fileContainerServer = new(context.VssConnection, projectId: new Guid(), containerId, containerPath); await fileContainerServer.DownloadFromContainerAsync(context, targetPath, token); context.Output("Artifact download finished."); diff --git a/src/Runner.Plugins/Artifact/FileContainerServer.cs b/src/Runner.Plugins/Artifact/FileContainerServer.cs index c2b8fb3a2ce..043bc3becb8 100644 --- a/src/Runner.Plugins/Artifact/FileContainerServer.cs +++ b/src/Runner.Plugins/Artifact/FileContainerServer.cs @@ -23,10 +23,10 @@ public class FileContainerServer //81920 is the default used by System.IO.Stream.CopyTo and is under the large object heap threshold (85k). private const int _defaultCopyBufferSize = 81920; - private readonly ConcurrentQueue _fileUploadQueue = new ConcurrentQueue(); - private readonly ConcurrentQueue _fileDownloadQueue = new ConcurrentQueue(); - private readonly ConcurrentDictionary> _fileUploadTraceLog = new ConcurrentDictionary>(); - private readonly ConcurrentDictionary> _fileUploadProgressLog = new ConcurrentDictionary>(); + private readonly ConcurrentQueue _fileUploadQueue = new(); + private readonly ConcurrentQueue _fileDownloadQueue = new(); + private readonly ConcurrentDictionary> _fileUploadTraceLog = new(); + private readonly ConcurrentDictionary> _fileUploadProgressLog = new(); private readonly FileContainerHttpClient _fileContainerHttpClient; private CancellationTokenSource _uploadCancellationTokenSource; @@ -67,7 +67,7 @@ public async Task DownloadFromContainerAsync( CancellationToken cancellationToken) { // Find out all container items need to be processed - List containerItems = new List(); + List containerItems = new(); int retryCount = 0; while (retryCount < 3) { @@ -106,7 +106,7 @@ public async Task DownloadFromContainerAsync( // Create all required empty folders and emptry files, gather a list of files that we need to download from server. int foldersCreated = 0; int emptryFilesCreated = 0; - List downloadFiles = new List(); + List downloadFiles = new(); foreach (var item in containerItems.OrderBy(x => x.Path)) { if (!item.Path.StartsWith(_containerPath, StringComparison.OrdinalIgnoreCase)) @@ -306,7 +306,7 @@ private async Task ParallelDownloadAsync(RunnerActionPluginExecu Task downloadMonitor = DownloadReportingAsync(context, files.Count(), token); // Start parallel download tasks. - List> parallelDownloadingTasks = new List>(); + List> parallelDownloadingTasks = new(); for (int downloader = 0; downloader < concurrentDownloads; downloader++) { parallelDownloadingTasks.Add(DownloadAsync(context, downloader, token)); @@ -358,7 +358,7 @@ private async Task ParallelUploadAsync(RunnerActionPluginExecution Task uploadMonitor = UploadReportingAsync(context, files.Count(), _uploadCancellationTokenSource.Token); // Start parallel upload tasks. - List> parallelUploadingTasks = new List>(); + List> parallelUploadingTasks = new(); for (int uploader = 0; uploader < concurrentUploads; uploader++) { parallelUploadingTasks.Add(UploadAsync(context, uploader, _uploadCancellationTokenSource.Token)); @@ -381,8 +381,8 @@ private async Task ParallelUploadAsync(RunnerActionPluginExecution private async Task DownloadAsync(RunnerActionPluginExecutionContext context, int downloaderId, CancellationToken token) { - List failedFiles = new List(); - Stopwatch downloadTimer = new Stopwatch(); + List failedFiles = new(); + Stopwatch downloadTimer = new(); while (_fileDownloadQueue.TryDequeue(out DownloadInfo fileToDownload)) { token.ThrowIfCancellationRequested(); @@ -396,7 +396,7 @@ private async Task DownloadAsync(RunnerActionPluginExecutionCont { context.Debug($"Start downloading file: '{fileToDownload.ItemPath}' (Downloader {downloaderId})"); downloadTimer.Restart(); - using (FileStream fs = new FileStream(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) + using (FileStream fs = new(fileToDownload.LocalPath, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var downloadStream = await _fileContainerHttpClient.DownloadFileAsync(_containerId, fileToDownload.ItemPath, token, _projectId)) { await downloadStream.CopyToAsync(fs, _defaultCopyBufferSize, token); @@ -453,10 +453,10 @@ private async Task DownloadAsync(RunnerActionPluginExecutionCont private async Task UploadAsync(RunnerActionPluginExecutionContext context, int uploaderId, CancellationToken token) { - List failedFiles = new List(); + List failedFiles = new(); long uploadedSize = 0; string fileToUpload; - Stopwatch uploadTimer = new Stopwatch(); + Stopwatch uploadTimer = new(); while (_fileUploadQueue.TryDequeue(out fileToUpload)) { token.ThrowIfCancellationRequested(); diff --git a/src/Runner.Plugins/Artifact/PublishArtifact.cs b/src/Runner.Plugins/Artifact/PublishArtifact.cs index 59f9a04a5d9..19b316fe0c9 100644 --- a/src/Runner.Plugins/Artifact/PublishArtifact.cs +++ b/src/Runner.Plugins/Artifact/PublishArtifact.cs @@ -68,7 +68,7 @@ public async Task RunAsync( context.Output($"Uploading artifact '{artifactName}' from '{fullPath}' for run #{buildId}"); - FileContainerServer fileContainerHelper = new FileContainerServer(context.VssConnection, projectId: Guid.Empty, containerId, artifactName); + FileContainerServer fileContainerHelper = new(context.VssConnection, projectId: Guid.Empty, containerId, artifactName); var propertiesDictionary = new Dictionary(StringComparer.OrdinalIgnoreCase); long size = 0; @@ -87,7 +87,7 @@ public async Task RunAsync( // Definition ID is a dummy value only used by HTTP client routing purposes int definitionId = 1; - PipelinesServer pipelinesHelper = new PipelinesServer(context.VssConnection); + PipelinesServer pipelinesHelper = new(context.VssConnection); var artifact = await pipelinesHelper.AssociateActionsStorageArtifactAsync( definitionId, diff --git a/src/Runner.Plugins/Repository/GitCliManager.cs b/src/Runner.Plugins/Repository/GitCliManager.cs index 338e17a6e76..cadba0004fd 100644 --- a/src/Runner.Plugins/Repository/GitCliManager.cs +++ b/src/Runner.Plugins/Repository/GitCliManager.cs @@ -18,7 +18,7 @@ public class GitCliManager #else private static readonly Encoding s_encoding = null; #endif - private readonly Dictionary gitEnv = new Dictionary(StringComparer.OrdinalIgnoreCase) + private readonly Dictionary gitEnv = new(StringComparer.OrdinalIgnoreCase) { { "GIT_TERMINAL_PROMPT", "0" }, }; @@ -92,11 +92,11 @@ public async Task LoadGitExecutionInfo(RunnerActionPluginExecutionContext contex } // required 2.0, all git operation commandline args need min git version 2.0 - Version minRequiredGitVersion = new Version(2, 0); + Version minRequiredGitVersion = new(2, 0); EnsureGitVersion(minRequiredGitVersion, throwOnNotMatch: true); // suggest user upgrade to 2.9 for better git experience - Version recommendGitVersion = new Version(2, 9); + Version recommendGitVersion = new(2, 9); if (!EnsureGitVersion(recommendGitVersion, throwOnNotMatch: false)) { context.Output($"To get a better Git experience, upgrade your Git to at least version '{recommendGitVersion}'. Your current Git version is '{gitVersion}'."); @@ -430,7 +430,7 @@ public async Task GitGetFetchUrl(RunnerActionPluginExecutionContext context context.Debug($"Inspect remote.origin.url for repository under {repositoryPath}"); Uri fetchUrl = null; - List outputStrings = new List(); + List outputStrings = new(); int exitCode = await ExecuteGitCommandAsync(context, repositoryPath, "config", "--get remote.origin.url", outputStrings); if (exitCode != 0) @@ -477,7 +477,7 @@ public async Task GitConfigExist(RunnerActionPluginExecutionContext contex context.Debug($"Checking git config {configKey} exist or not"); // ignore any outputs by redirect them into a string list, since the output might contains secrets. - List outputStrings = new List(); + List outputStrings = new(); int exitcode = await ExecuteGitCommandAsync(context, repositoryPath, "config", StringUtil.Format($"--get-all {configKey}"), outputStrings); return exitcode == 0; @@ -539,7 +539,7 @@ public async Task GitVersion(RunnerActionPluginExecutionContext context string runnerWorkspace = context.GetRunnerContext("workspace"); ArgUtil.Directory(runnerWorkspace, "runnerWorkspace"); Version version = null; - List outputStrings = new List(); + List outputStrings = new(); int exitCode = await ExecuteGitCommandAsync(context, runnerWorkspace, "version", null, outputStrings); context.Output($"{string.Join(Environment.NewLine, outputStrings)}"); if (exitCode == 0) @@ -550,7 +550,7 @@ public async Task GitVersion(RunnerActionPluginExecutionContext context { string verString = outputStrings.First(); // we interested about major.minor.patch version - Regex verRegex = new Regex("\\d+\\.\\d+(\\.\\d+)?", RegexOptions.IgnoreCase); + Regex verRegex = new("\\d+\\.\\d+(\\.\\d+)?", RegexOptions.IgnoreCase); var matchResult = verRegex.Match(verString); if (matchResult.Success && !string.IsNullOrEmpty(matchResult.Value)) { @@ -572,7 +572,7 @@ public async Task GitLfsVersion(RunnerActionPluginExecutionContext cont string runnerWorkspace = context.GetRunnerContext("workspace"); ArgUtil.Directory(runnerWorkspace, "runnerWorkspace"); Version version = null; - List outputStrings = new List(); + List outputStrings = new(); int exitCode = await ExecuteGitCommandAsync(context, runnerWorkspace, "lfs version", null, outputStrings); context.Output($"{string.Join(Environment.NewLine, outputStrings)}"); if (exitCode == 0) @@ -583,7 +583,7 @@ public async Task GitLfsVersion(RunnerActionPluginExecutionContext cont { string verString = outputStrings.First(); // we interested about major.minor.patch version - Regex verRegex = new Regex("\\d+\\.\\d+(\\.\\d+)?", RegexOptions.IgnoreCase); + Regex verRegex = new("\\d+\\.\\d+(\\.\\d+)?", RegexOptions.IgnoreCase); var matchResult = verRegex.Match(verString); if (matchResult.Success && !string.IsNullOrEmpty(matchResult.Value)) { diff --git a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs index f567a1338b8..a63eaaf3e2c 100644 --- a/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs +++ b/src/Runner.Plugins/Repository/v1.0/GitSourceProvider.cs @@ -21,7 +21,7 @@ public sealed class GitHubSourceProvider private const string _remotePullRefsPrefix = "refs/remotes/pull/"; // min git version that support add extra auth header. - private Version _minGitVersionSupportAuthHeader = new Version(2, 9); + private Version _minGitVersionSupportAuthHeader = new(2, 9); #if OS_WINDOWS // min git version that support override sslBackend setting. @@ -29,7 +29,7 @@ public sealed class GitHubSourceProvider #endif // min git-lfs version that support add extra auth header. - private Version _minGitLfsVersionSupportAuthHeader = new Version(2, 1); + private Version _minGitLfsVersionSupportAuthHeader = new(2, 1); private void RequirementCheck(RunnerActionPluginExecutionContext executionContext, GitCliManager gitCommandManager, bool checkGitLfs) { @@ -83,7 +83,7 @@ public async Task GetSourceAsync( var githubUrl = executionContext.GetGitHubContext("server_url"); var githubUri = new Uri(!string.IsNullOrEmpty(githubUrl) ? githubUrl : "https://github.com"); var portInfo = githubUri.IsDefaultPort ? string.Empty : $":{githubUri.Port}"; - Uri repositoryUrl = new Uri($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}"); + Uri repositoryUrl = new($"{githubUri.Scheme}://{githubUri.Host}{portInfo}/{repoFullName}"); if (!repositoryUrl.IsAbsoluteUri) { throw new InvalidOperationException("Repository url need to be an absolute uri."); @@ -121,7 +121,7 @@ public async Task GetSourceAsync( executionContext.Debug($"gitLfsSupport={gitLfsSupport}"); // Initialize git command manager with additional environment variables. - Dictionary gitEnv = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary gitEnv = new(StringComparer.OrdinalIgnoreCase); // Disable prompting for git credential manager gitEnv["GCM_INTERACTIVE"] = "Never"; @@ -141,7 +141,7 @@ public async Task GetSourceAsync( gitEnv[formattedKey] = variable.Value?.Value ?? string.Empty; } - GitCliManager gitCommandManager = new GitCliManager(gitEnv); + GitCliManager gitCommandManager = new(gitEnv); await gitCommandManager.LoadGitExecutionInfo(executionContext); // Make sure the build machine met all requirements for the git repository @@ -293,8 +293,8 @@ public async Task GetSourceAsync( await RemoveGitConfig(executionContext, gitCommandManager, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty); } - List additionalFetchArgs = new List(); - List additionalLfsFetchArgs = new List(); + List additionalFetchArgs = new(); + List additionalLfsFetchArgs = new(); // add accessToken as basic auth header to handle auth challenge. if (!string.IsNullOrEmpty(accessToken)) @@ -320,7 +320,7 @@ public async Task GetSourceAsync( } } - List additionalFetchSpecs = new List(); + List additionalFetchSpecs = new(); additionalFetchSpecs.Add("+refs/heads/*:refs/remotes/origin/*"); if (IsPullRequest(sourceBranch)) @@ -395,7 +395,7 @@ public async Task GetSourceAsync( throw new InvalidOperationException($"Git submodule sync failed with exit code: {exitCode_submoduleSync}"); } - List additionalSubmoduleUpdateArgs = new List(); + List additionalSubmoduleUpdateArgs = new(); if (!string.IsNullOrEmpty(accessToken)) { diff --git a/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs b/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs index df8bd33639c..8f7c27df02a 100644 --- a/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs +++ b/src/Runner.Plugins/Repository/v1.0/RepositoryPlugin.cs @@ -12,7 +12,7 @@ namespace GitHub.Runner.Plugins.Repository.v1_0 { public class CheckoutTask : IRunnerActionPlugin { - private readonly Regex _validSha1 = new Regex(@"\b[0-9a-f]{40}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled, TimeSpan.FromSeconds(2)); + private readonly Regex _validSha1 = new(@"\b[0-9a-f]{40}\b", RegexOptions.IgnoreCase | RegexOptions.CultureInvariant | RegexOptions.Compiled, TimeSpan.FromSeconds(2)); public async Task RunAsync(RunnerActionPluginExecutionContext executionContext, CancellationToken token) { diff --git a/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs b/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs index b2f50f45813..e138cb1db17 100644 --- a/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs +++ b/src/Runner.Plugins/Repository/v1.1/GitSourceProvider.cs @@ -22,7 +22,7 @@ public sealed class GitHubSourceProvider private const string _tagRefsPrefix = "refs/tags/"; // min git version that support add extra auth header. - private Version _minGitVersionSupportAuthHeader = new Version(2, 9); + private Version _minGitVersionSupportAuthHeader = new(2, 9); #if OS_WINDOWS // min git version that support override sslBackend setting. @@ -30,7 +30,7 @@ public sealed class GitHubSourceProvider #endif // min git-lfs version that support add extra auth header. - private Version _minGitLfsVersionSupportAuthHeader = new Version(2, 1); + private Version _minGitLfsVersionSupportAuthHeader = new(2, 1); public static string ProblemMatcher => @" { @@ -62,9 +62,9 @@ public async Task GetSourceAsync( { // Validate args. ArgUtil.NotNull(executionContext, nameof(executionContext)); - Dictionary configModifications = new Dictionary(); + Dictionary configModifications = new(); executionContext.Output($"Syncing repository: {repoFullName}"); - Uri repositoryUrl = new Uri($"https://github.com/{repoFullName}"); + Uri repositoryUrl = new($"https://github.com/{repoFullName}"); if (!repositoryUrl.IsAbsoluteUri) { throw new InvalidOperationException("Repository url need to be an absolute uri."); @@ -102,7 +102,7 @@ public async Task GetSourceAsync( executionContext.Debug($"gitLfsSupport={gitLfsSupport}"); // Initialize git command manager with additional environment variables. - Dictionary gitEnv = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary gitEnv = new(StringComparer.OrdinalIgnoreCase); // Disable git prompt gitEnv["GIT_TERMINAL_PROMPT"] = "0"; @@ -125,7 +125,7 @@ public async Task GetSourceAsync( gitEnv[formattedKey] = variable.Value?.Value ?? string.Empty; } - GitCliManager gitCommandManager = new GitCliManager(gitEnv); + GitCliManager gitCommandManager = new(gitEnv); await gitCommandManager.LoadGitExecutionInfo(executionContext); // Make sure the build machine met all requirements for the git repository @@ -277,8 +277,8 @@ public async Task GetSourceAsync( await RemoveGitConfig(executionContext, gitCommandManager, targetPath, $"http.{repositoryUrl.AbsoluteUri}.extraheader", string.Empty); } - List additionalFetchArgs = new List(); - List additionalLfsFetchArgs = new List(); + List additionalFetchArgs = new(); + List additionalLfsFetchArgs = new(); // Add http.https://github.com.extraheader=... to gitconfig // accessToken as basic auth header to handle any auth challenge from github.com @@ -303,7 +303,7 @@ public async Task GetSourceAsync( } } - List additionalFetchSpecs = new List(); + List additionalFetchSpecs = new(); additionalFetchSpecs.Add("+refs/heads/*:refs/remotes/origin/*"); if (IsPullRequest(sourceBranch)) @@ -378,7 +378,7 @@ public async Task GetSourceAsync( throw new InvalidOperationException($"Git submodule sync failed with exit code: {exitCode_submoduleSync}"); } - List additionalSubmoduleUpdateArgs = new List(); + List additionalSubmoduleUpdateArgs = new(); int exitCode_submoduleUpdate = await gitCommandManager.GitSubmoduleUpdate(executionContext, targetPath, fetchDepth, string.Join(" ", additionalSubmoduleUpdateArgs), checkoutNestedSubmodules, cancellationToken); if (exitCode_submoduleUpdate != 0) @@ -404,7 +404,7 @@ public async Task CleanupAsync(RunnerActionPluginExecutionContext executionConte executionContext.Output($"Cleanup cached git credential from {repositoryPath}."); // Initialize git command manager - GitCliManager gitCommandManager = new GitCliManager(); + GitCliManager gitCommandManager = new(); await gitCommandManager.LoadGitExecutionInfo(executionContext); executionContext.Debug("Remove any extraheader setting from git config."); @@ -499,7 +499,7 @@ private async Task RemoveGitConfig(RunnerActionPluginExecutionContext executionC string gitConfig = Path.Combine(targetPath, ".git/config"); if (File.Exists(gitConfig)) { - List safeGitConfig = new List(); + List safeGitConfig = new(); var gitConfigContents = File.ReadAllLines(gitConfig); foreach (var line in gitConfigContents) { diff --git a/src/Runner.Sdk/ActionPlugin.cs b/src/Runner.Sdk/ActionPlugin.cs index 0a0f33ab457..35ffa9ab431 100644 --- a/src/Runner.Sdk/ActionPlugin.cs +++ b/src/Runner.Sdk/ActionPlugin.cs @@ -23,7 +23,7 @@ public class RunnerActionPluginExecutionContext : ITraceWriter private readonly string DebugEnvironmentalVariable = "ACTIONS_STEP_DEBUG"; private VssConnection _connection; private RunnerWebProxy _webProxy; - private readonly object _stdoutLock = new object(); + private readonly object _stdoutLock = new(); private readonly ITraceWriter _trace; // for unit tests public RunnerActionPluginExecutionContext() @@ -220,7 +220,7 @@ private string Escape(string input) return input; } - private Dictionary _commandEscapeMappings = new Dictionary(StringComparer.OrdinalIgnoreCase) + private Dictionary _commandEscapeMappings = new(StringComparer.OrdinalIgnoreCase) { { ";", "%3B" diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index 78a9f2dd27e..e303863b59b 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -24,18 +24,18 @@ public sealed class ProcessInvoker : IDisposable private Stopwatch _stopWatch; private int _asyncStreamReaderCount = 0; private bool _waitingOnStreams = false; - private readonly AsyncManualResetEvent _outputProcessEvent = new AsyncManualResetEvent(); - private readonly TaskCompletionSource _processExitedCompletionSource = new TaskCompletionSource(); - private readonly CancellationTokenSource _processStandardInWriteCancellationTokenSource = new CancellationTokenSource(); - private readonly ConcurrentQueue _errorData = new ConcurrentQueue(); - private readonly ConcurrentQueue _outputData = new ConcurrentQueue(); + private readonly AsyncManualResetEvent _outputProcessEvent = new(); + private readonly TaskCompletionSource _processExitedCompletionSource = new(); + private readonly CancellationTokenSource _processStandardInWriteCancellationTokenSource = new(); + private readonly ConcurrentQueue _errorData = new(); + private readonly ConcurrentQueue _outputData = new(); private readonly TimeSpan _sigintTimeout = TimeSpan.FromMilliseconds(7500); private readonly TimeSpan _sigtermTimeout = TimeSpan.FromMilliseconds(2500); private ITraceWriter Trace { get; set; } private class AsyncManualResetEvent { - private volatile TaskCompletionSource m_tcs = new TaskCompletionSource(); + private volatile TaskCompletionSource m_tcs = new(); public Task WaitAsync() { return m_tcs.Task; } @@ -387,8 +387,8 @@ private void Dispose(bool disposing) private void ProcessOutput() { - List errorData = new List(); - List outputData = new List(); + List errorData = new(); + List outputData = new(); string errorLine; while (_errorData.TryDequeue(out errorLine)) diff --git a/src/Runner.Sdk/RunnerWebProxy.cs b/src/Runner.Sdk/RunnerWebProxy.cs index abe0506b297..516b1dcec16 100644 --- a/src/Runner.Sdk/RunnerWebProxy.cs +++ b/src/Runner.Sdk/RunnerWebProxy.cs @@ -23,9 +23,9 @@ public class RunnerWebProxy : IWebProxy private string _httpsProxyPassword; private string _noProxyString; - private readonly List _noProxyList = new List(); - private readonly HashSet _noProxyUnique = new HashSet(StringComparer.OrdinalIgnoreCase); - private readonly Regex _validIpRegex = new Regex("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", RegexOptions.Compiled); + private readonly List _noProxyList = new(); + private readonly HashSet _noProxyUnique = new(StringComparer.OrdinalIgnoreCase); + private readonly Regex _validIpRegex = new("^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$", RegexOptions.Compiled); public string HttpProxyAddress => _httpProxyAddress; public string HttpProxyUsername => _httpProxyUsername; diff --git a/src/Runner.Sdk/Util/IOUtil.cs b/src/Runner.Sdk/Util/IOUtil.cs index 94a674da3b4..d956988192a 100644 --- a/src/Runner.Sdk/Util/IOUtil.cs +++ b/src/Runner.Sdk/Util/IOUtil.cs @@ -52,7 +52,7 @@ public static string GetSha256Hash(string path) using (SHA256 sha256hash = SHA256.Create()) { byte[] data = sha256hash.ComputeHash(Encoding.UTF8.GetBytes(hashString)); - StringBuilder sBuilder = new StringBuilder(); + StringBuilder sBuilder = new(); for (int i = 0; i < data.Length; i++) { sBuilder.Append(data[i].ToString("x2")); @@ -77,7 +77,7 @@ public static void DeleteDirectory(string path, CancellationToken cancellationTo public static void DeleteDirectory(string path, bool contentsOnly, bool continueOnContentDeleteError, CancellationToken cancellationToken) { ArgUtil.NotNullOrEmpty(path, nameof(path)); - DirectoryInfo directory = new DirectoryInfo(path); + DirectoryInfo directory = new(path); if (!directory.Exists) { return; @@ -363,12 +363,12 @@ public static void CopyDirectory(string source, string target, CancellationToken Directory.CreateDirectory(target); // Get the file contents of the directory to copy. - DirectoryInfo sourceDir = new DirectoryInfo(source); + DirectoryInfo sourceDir = new(source); foreach (FileInfo sourceFile in sourceDir.GetFiles() ?? new FileInfo[0]) { // Check if the file already exists. cancellationToken.ThrowIfCancellationRequested(); - FileInfo targetFile = new FileInfo(Path.Combine(target, sourceFile.Name)); + FileInfo targetFile = new(Path.Combine(target, sourceFile.Name)); if (!targetFile.Exists || sourceFile.Length != targetFile.Length || sourceFile.LastWriteTime != targetFile.LastWriteTime) diff --git a/src/Runner.Sdk/Util/StringUtil.cs b/src/Runner.Sdk/Util/StringUtil.cs index e55fd7f4624..3110e7e21de 100644 --- a/src/Runner.Sdk/Util/StringUtil.cs +++ b/src/Runner.Sdk/Util/StringUtil.cs @@ -9,7 +9,7 @@ namespace GitHub.Runner.Sdk public static class StringUtil { private static readonly object[] s_defaultFormatArgs = new object[] { null }; - private static Lazy s_serializerSettings = new Lazy(() => + private static Lazy s_serializerSettings = new(() => { var settings = new VssJsonMediaTypeFormatter().SerializerSettings; settings.DateParseHandling = DateParseHandling.None; diff --git a/src/Runner.Sdk/Util/UrlUtil.cs b/src/Runner.Sdk/Util/UrlUtil.cs index 02e35a8f846..4e74c98201e 100644 --- a/src/Runner.Sdk/Util/UrlUtil.cs +++ b/src/Runner.Sdk/Util/UrlUtil.cs @@ -21,7 +21,7 @@ public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string return baseUrl; } - UriBuilder credUri = new UriBuilder(baseUrl); + UriBuilder credUri = new(baseUrl); // ensure we have a username, uribuild will throw if username is empty but password is not. if (string.IsNullOrEmpty(username)) diff --git a/src/Runner.Sdk/Util/VssUtil.cs b/src/Runner.Sdk/Util/VssUtil.cs index 2194e28ff03..6bfc7f131f4 100644 --- a/src/Runner.Sdk/Util/VssUtil.cs +++ b/src/Runner.Sdk/Util/VssUtil.cs @@ -76,7 +76,7 @@ public static VssConnection CreateConnection( // settings are applied to an HttpRequestMessage. settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture); - VssConnection connection = new VssConnection(serverUri, new VssHttpMessageHandler(credentials, settings), additionalDelegatingHandler); + VssConnection connection = new(serverUri, new VssHttpMessageHandler(credentials, settings), additionalDelegatingHandler); return connection; } @@ -116,7 +116,7 @@ public static RawConnection CreateRawConnection( // settings are applied to an HttpRequestMessage. settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture); - RawConnection connection = new RawConnection(serverUri, new RawHttpMessageHandler(credentials.ToOAuthCredentials(), settings), additionalDelegatingHandler); + RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.ToOAuthCredentials(), settings), additionalDelegatingHandler); return connection; } diff --git a/src/Runner.Worker/ActionCommandManager.cs b/src/Runner.Worker/ActionCommandManager.cs index e301515da02..33fab1f2141 100644 --- a/src/Runner.Worker/ActionCommandManager.cs +++ b/src/Runner.Worker/ActionCommandManager.cs @@ -21,9 +21,9 @@ public interface IActionCommandManager : IRunnerService public sealed class ActionCommandManager : RunnerService, IActionCommandManager { private const string _stopCommand = "stop-commands"; - private readonly Dictionary _commandExtensions = new Dictionary(StringComparer.OrdinalIgnoreCase); - private readonly HashSet _registeredCommands = new HashSet(StringComparer.OrdinalIgnoreCase); - private readonly object _commandSerializeLock = new object(); + private readonly Dictionary _commandExtensions = new(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _registeredCommands = new(StringComparer.OrdinalIgnoreCase); + private readonly object _commandSerializeLock = new(); private bool _stopProcessCommand = false; private string _stopToken = null; @@ -618,7 +618,7 @@ public void ProcessCommand(IExecutionContext context, string inputLine, ActionCo context.Debug("Enhanced Annotations not enabled on the server. The 'title', 'end_line', and 'end_column' fields are unsupported."); } - Issue issue = new Issue() + Issue issue = new() { Category = "General", Type = this.Type, diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 1ba983de7aa..6a07c9aa0d9 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -52,16 +52,16 @@ public sealed class ActionManager : RunnerService, IActionManager private const int _defaultCopyBufferSize = 81920; private const string _dotcomApiUrl = "https://api.github.com"; - private readonly Dictionary _cachedActionContainers = new Dictionary(); + private readonly Dictionary _cachedActionContainers = new(); public Dictionary CachedActionContainers => _cachedActionContainers; - private readonly Dictionary> _cachedEmbeddedPreSteps = new Dictionary>(); + private readonly Dictionary> _cachedEmbeddedPreSteps = new(); public Dictionary> CachedEmbeddedPreSteps => _cachedEmbeddedPreSteps; - private readonly Dictionary> _cachedEmbeddedStepIds = new Dictionary>(); + private readonly Dictionary> _cachedEmbeddedStepIds = new(); public Dictionary> CachedEmbeddedStepIds => _cachedEmbeddedStepIds; - private readonly Dictionary> _cachedEmbeddedPostSteps = new Dictionary>(); + private readonly Dictionary> _cachedEmbeddedPostSteps = new(); public Dictionary> CachedEmbeddedPostSteps => _cachedEmbeddedPostSteps; public async Task PrepareActionsAsync(IExecutionContext executionContext, IEnumerable steps, Guid rootStepId = default(Guid)) @@ -791,7 +791,7 @@ private async Task DownloadRepositoryActionAsync(IExecutionContext executionCont try { //open zip stream in async mode - using (FileStream fs = new FileStream(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) + using (FileStream fs = new(archiveFile, FileMode.Create, FileAccess.Write, FileShare.None, bufferSize: _defaultFileStreamBufferSize, useAsync: true)) using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) { diff --git a/src/Runner.Worker/ActionManifestManager.cs b/src/Runner.Worker/ActionManifestManager.cs index 00cc36f6af9..e14be1295ee 100644 --- a/src/Runner.Worker/ActionManifestManager.cs +++ b/src/Runner.Worker/ActionManifestManager.cs @@ -53,7 +53,7 @@ public override void Initialize(IHostContext hostContext) public ActionDefinitionData Load(IExecutionContext executionContext, string manifestFile) { var templateContext = CreateTemplateContext(executionContext); - ActionDefinitionData actionDefinition = new ActionDefinitionData(); + ActionDefinitionData actionDefinition = new(); // Clean up file name real quick // Instead of using Regex which can be computationally expensive, diff --git a/src/Runner.Worker/ConditionTraceWriter.cs b/src/Runner.Worker/ConditionTraceWriter.cs index 1f98987ccd9..a298113ebd3 100644 --- a/src/Runner.Worker/ConditionTraceWriter.cs +++ b/src/Runner.Worker/ConditionTraceWriter.cs @@ -10,7 +10,7 @@ public sealed class ConditionTraceWriter : ObjectTemplating::ITraceWriter { private readonly IExecutionContext _executionContext; private readonly Tracing _trace; - private readonly StringBuilder _traceBuilder = new StringBuilder(); + private readonly StringBuilder _traceBuilder = new(); public string Trace => _traceBuilder.ToString(); diff --git a/src/Runner.Worker/Container/ContainerInfo.cs b/src/Runner.Worker/Container/ContainerInfo.cs index 32e55eb3c40..72cd0ada992 100644 --- a/src/Runner.Worker/Container/ContainerInfo.cs +++ b/src/Runner.Worker/Container/ContainerInfo.cs @@ -15,7 +15,7 @@ public class ContainerInfo private IDictionary _userPortMappings; private List _portMappings; private IDictionary _environmentVariables; - private List _pathMappings = new List(); + private List _pathMappings = new(); public ContainerInfo() { diff --git a/src/Runner.Worker/Container/DockerCommandManager.cs b/src/Runner.Worker/Container/DockerCommandManager.cs index a0c158bdf68..64f42f3bbfd 100644 --- a/src/Runner.Worker/Container/DockerCommandManager.cs +++ b/src/Runner.Worker/Container/DockerCommandManager.cs @@ -60,7 +60,7 @@ public async Task DockerVersion(IExecutionContext context) context.Output($"Docker client API version: {clientVersionStr}"); // we interested about major.minor.patch version - Regex verRegex = new Regex("\\d+\\.\\d+(\\.\\d+)?", RegexOptions.IgnoreCase); + Regex verRegex = new("\\d+\\.\\d+(\\.\\d+)?", RegexOptions.IgnoreCase); Version serverVersion = null; var serverVersionMatchResult = verRegex.Match(serverVersionStr); @@ -309,7 +309,7 @@ public async Task DockerExec(IExecutionContext context, string containerId, string arg = $"exec {options} {containerId} {command}".Trim(); context.Command($"{DockerPath} {arg}"); - object outputLock = new object(); + object outputLock = new(); var processInvoker = HostContext.CreateService(); processInvoker.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message) { @@ -447,7 +447,7 @@ private async Task> ExecuteDockerCommandAsync(IExecutionContext con string arg = $"{command} {options}".Trim(); context.Command($"{DockerPath} {arg}"); - List output = new List(); + List output = new(); var processInvoker = HostContext.CreateService(); processInvoker.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message) { diff --git a/src/Runner.Worker/Container/DockerUtil.cs b/src/Runner.Worker/Container/DockerUtil.cs index 97a53759bc7..e8b094298cb 100644 --- a/src/Runner.Worker/Container/DockerUtil.cs +++ b/src/Runner.Worker/Container/DockerUtil.cs @@ -6,8 +6,8 @@ namespace GitHub.Runner.Worker.Container { public class DockerUtil { - private static readonly Regex QuoteEscape = new Regex(@"(\\*)" + "\"", RegexOptions.Compiled); - private static readonly Regex EndOfStringEscape = new Regex(@"(\\+)$", RegexOptions.Compiled); + private static readonly Regex QuoteEscape = new(@"(\\*)" + "\"", RegexOptions.Compiled); + private static readonly Regex EndOfStringEscape = new(@"(\\+)$", RegexOptions.Compiled); public static List ParseDockerPort(IList portMappingLines) { @@ -19,7 +19,7 @@ public static List ParseDockerPort(IList portMappingLines) //"TARGET_PORT/PROTO -> HOST:HOST_PORT" string pattern = $"^(?<{targetPort}>\\d+)/(?<{proto}>\\w+) -> (?<{host}>.+):(?<{hostPort}>\\d+)$"; - List portMappings = new List(); + List portMappings = new(); foreach (var line in portMappingLines) { Match m = Regex.Match(line, pattern, RegexOptions.None, TimeSpan.FromSeconds(1)); diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index b0da204715c..79cc8cf7643 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -354,8 +354,8 @@ private async Task> ExecuteCommandAsync(IExecutionContext context, { context.Command($"{command} {arg}"); - List outputs = new List(); - object outputLock = new object(); + List outputs = new(); + object outputLock = new(); var processInvoker = HostContext.CreateService(); processInvoker.OutputDataReceived += delegate (object sender, ProcessDataReceivedEventArgs message) { @@ -562,7 +562,7 @@ private async Task AssertCompatibleOS(IExecutionContext executionContext) #if OS_WINDOWS Version requiredDockerEngineAPIVersion = new Version(1, 30); // Docker-EE version 17.6 #else - Version requiredDockerEngineAPIVersion = new Version(1, 35); // Docker-CE version 17.12 + Version requiredDockerEngineAPIVersion = new(1, 35); // Docker-CE version 17.12 #endif if (dockerVersion.ServerVersion < requiredDockerEngineAPIVersion) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 79e0335fda9..d7ff528b1d3 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -122,10 +122,10 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private const int _maxIssueCountInTelemetry = 3; // Only send the first 3 issues to telemetry private const int _maxIssueMessageLengthInTelemetry = 256; // Only send the first 256 characters of issue message to telemetry - private readonly TimelineRecord _record = new TimelineRecord(); - private readonly Dictionary _detailRecords = new Dictionary(); - private readonly object _loggerLock = new object(); - private readonly object _matchersLock = new object(); + private readonly TimelineRecord _record = new(); + private readonly Dictionary _detailRecords = new(); + private readonly object _loggerLock = new(); + private readonly object _matchersLock = new(); private event OnMatcherChanged _onMatcherChanged; @@ -140,7 +140,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private bool _expandedForPostJob = false; private int _childTimelineRecordOrder = 0; private CancellationTokenSource _cancellationTokenSource; - private TaskCompletionSource _forceCompleted = new TaskCompletionSource(); + private TaskCompletionSource _forceCompleted = new(); private bool _throttlingReported = false; // only job level ExecutionContext will track throttling delay. diff --git a/src/Runner.Worker/Expressions/HashFilesFunction.cs b/src/Runner.Worker/Expressions/HashFilesFunction.cs index 572e7b251b7..290f41239f5 100644 --- a/src/Runner.Worker/Expressions/HashFilesFunction.cs +++ b/src/Runner.Worker/Expressions/HashFilesFunction.cs @@ -32,7 +32,7 @@ protected sealed override Object EvaluateCore( string githubWorkspace = workspaceData.Value; bool followSymlink = false; - List patterns = new List(); + List patterns = new(); var firstParameter = true; foreach (var parameter in Parameters) { diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index f320705eba0..f92850cc65b 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -6,7 +6,7 @@ namespace GitHub.Runner.Worker { public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextData { - private readonly HashSet _contextEnvAllowlist = new HashSet(StringComparer.OrdinalIgnoreCase) + private readonly HashSet _contextEnvAllowlist = new(StringComparer.OrdinalIgnoreCase) { "action_path", "action_ref", diff --git a/src/Runner.Worker/Handlers/OutputManager.cs b/src/Runner.Worker/Handlers/OutputManager.cs index a67e35e2126..8b97c867d95 100644 --- a/src/Runner.Worker/Handlers/OutputManager.cs +++ b/src/Runner.Worker/Handlers/OutputManager.cs @@ -16,16 +16,16 @@ public sealed class OutputManager : IDisposable private const string _colorCodePrefix = "\033["; private const int _maxAttempts = 3; private const string _timeoutKey = "GITHUB_ACTIONS_RUNNER_ISSUE_MATCHER_TIMEOUT"; - private static readonly Regex _colorCodeRegex = new Regex(@"\x0033\[[0-9;]*m?", RegexOptions.Compiled | RegexOptions.CultureInvariant); + private static readonly Regex _colorCodeRegex = new(@"\x0033\[[0-9;]*m?", RegexOptions.Compiled | RegexOptions.CultureInvariant); private readonly IActionCommandManager _commandManager; private readonly ContainerInfo _container; private readonly IExecutionContext _executionContext; private readonly int _failsafe = 50; - private readonly object _matchersLock = new object(); + private readonly object _matchersLock = new(); private readonly TimeSpan _timeout; private IssueMatcher[] _matchers = Array.Empty(); // Mapping that indicates whether a directory belongs to the workflow repository - private readonly Dictionary _directoryMap = new Dictionary(); + private readonly Dictionary _directoryMap = new(); public OutputManager(IExecutionContext executionContext, IActionCommandManager commandManager, ContainerInfo container = null) { diff --git a/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs b/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs index 159a028f9fa..9a22c75901b 100644 --- a/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs +++ b/src/Runner.Worker/Handlers/ScriptHandlerHelpers.cs @@ -10,7 +10,7 @@ namespace GitHub.Runner.Worker.Handlers { internal static class ScriptHandlerHelpers { - private static readonly Dictionary _defaultArguments = new Dictionary(StringComparer.OrdinalIgnoreCase) + private static readonly Dictionary _defaultArguments = new(StringComparer.OrdinalIgnoreCase) { ["cmd"] = "/D /E:ON /V:OFF /S /C \"CALL \"{0}\"\"", ["pwsh"] = "-command \". '{0}'\"", @@ -20,7 +20,7 @@ internal static class ScriptHandlerHelpers ["python"] = "{0}" }; - private static readonly Dictionary _extensions = new Dictionary(StringComparer.OrdinalIgnoreCase) + private static readonly Dictionary _extensions = new(StringComparer.OrdinalIgnoreCase) { ["cmd"] = ".cmd", ["pwsh"] = ".ps1", diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 14004ecaf25..58e1b11371a 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -39,10 +39,10 @@ public interface IJobExtension : IRunnerService public sealed class JobExtension : RunnerService, IJobExtension { - private readonly HashSet _existingProcesses = new HashSet(StringComparer.OrdinalIgnoreCase); + private readonly HashSet _existingProcesses = new(StringComparer.OrdinalIgnoreCase); private bool _processCleanup; private string _processLookupId = $"github_{Guid.NewGuid()}"; - private CancellationTokenSource _diskSpaceCheckToken = new CancellationTokenSource(); + private CancellationTokenSource _diskSpaceCheckToken = new(); private Task _diskSpaceCheckTask = null; // Download all required actions. @@ -59,8 +59,8 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.StepTelemetry.Type = "runner"; context.StepTelemetry.Action = "setup_job"; - List preJobSteps = new List(); - List jobSteps = new List(); + List preJobSteps = new(); + List jobSteps = new(); using (var register = jobContext.CancellationToken.Register(() => { context.CancelToken(); })) { try @@ -386,7 +386,7 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel data: (object)jobHookData)); } - List steps = new List(); + List steps = new(); steps.AddRange(preJobSteps); steps.AddRange(jobSteps); @@ -674,7 +674,7 @@ private async Task CheckDiskSpaceAsync(IExecutionContext context, CancellationTo private Dictionary SnapshotProcesses() { - Dictionary snapshot = new Dictionary(); + Dictionary snapshot = new(); foreach (var proc in Process.GetProcesses()) { try diff --git a/src/Runner.Worker/Program.cs b/src/Runner.Worker/Program.cs index 4f5cf68bd9c..c4f23f943ec 100644 --- a/src/Runner.Worker/Program.cs +++ b/src/Runner.Worker/Program.cs @@ -11,7 +11,7 @@ public static class Program { public static int Main(string[] args) { - using (HostContext context = new HostContext("Worker")) + using (HostContext context = new("Worker")) { return MainAsync(context, args).GetAwaiter().GetResult(); } diff --git a/src/Runner.Worker/RunnerPluginManager.cs b/src/Runner.Worker/RunnerPluginManager.cs index 78f7ff38fb6..f103482857e 100644 --- a/src/Runner.Worker/RunnerPluginManager.cs +++ b/src/Runner.Worker/RunnerPluginManager.cs @@ -26,7 +26,7 @@ public interface IRunnerPluginManager : IRunnerService public sealed class RunnerPluginManager : RunnerService, IRunnerPluginManager { - private readonly Dictionary _actionPlugins = new Dictionary(StringComparer.OrdinalIgnoreCase) + private readonly Dictionary _actionPlugins = new(StringComparer.OrdinalIgnoreCase) { { "checkout", @@ -97,7 +97,7 @@ public async Task RunPluginActionAsync(IExecutionContext context, string plugin, string arguments = $"action \"{plugin}\""; // construct plugin context - RunnerActionPluginExecutionContext pluginContext = new RunnerActionPluginExecutionContext + RunnerActionPluginExecutionContext pluginContext = new() { Inputs = inputs, Endpoints = context.Global.Endpoints, diff --git a/src/Runner.Worker/StepsContext.cs b/src/Runner.Worker/StepsContext.cs index fe967253790..5dcf258b4fa 100644 --- a/src/Runner.Worker/StepsContext.cs +++ b/src/Runner.Worker/StepsContext.cs @@ -16,8 +16,8 @@ namespace GitHub.Runner.Worker /// public sealed class StepsContext { - private static readonly Regex _propertyRegex = new Regex("^[a-zA-Z_][a-zA-Z0-9_]*$", RegexOptions.Compiled); - private readonly DictionaryContextData _contextData = new DictionaryContextData(); + private static readonly Regex _propertyRegex = new("^[a-zA-Z_][a-zA-Z0-9_]*$", RegexOptions.Compiled); + private readonly DictionaryContextData _contextData = new(); /// /// Clears memory for a composite action's isolated "steps" context, after the action diff --git a/src/Runner.Worker/TrackingManager.cs b/src/Runner.Worker/TrackingManager.cs index b5eb7c8756e..cb3ead6c9c1 100644 --- a/src/Runner.Worker/TrackingManager.cs +++ b/src/Runner.Worker/TrackingManager.cs @@ -32,7 +32,7 @@ public TrackingConfig Create( Trace.Entering(); // Create the new tracking config. - TrackingConfig config = new TrackingConfig(executionContext); + TrackingConfig config = new(executionContext); WriteToFile(file, config); return config; } diff --git a/src/Runner.Worker/Variables.cs b/src/Runner.Worker/Variables.cs index fb8458475b2..3284bb7e154 100644 --- a/src/Runner.Worker/Variables.cs +++ b/src/Runner.Worker/Variables.cs @@ -14,9 +14,9 @@ namespace GitHub.Runner.Worker public sealed class Variables { private readonly IHostContext _hostContext; - private readonly ConcurrentDictionary _variables = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + private readonly ConcurrentDictionary _variables = new(StringComparer.OrdinalIgnoreCase); private readonly ISecretMasker _secretMasker; - private readonly object _setLock = new object(); + private readonly object _setLock = new(); private readonly Tracing _trace; public IEnumerable AllVariables @@ -43,7 +43,7 @@ public Variables(IHostContext hostContext, IDictionary co } // Initialize the variable dictionary. - List variables = new List(); + List variables = new(); foreach (var variable in copy) { if (!string.IsNullOrWhiteSpace(variable.Key)) diff --git a/src/Runner.Worker/Worker.cs b/src/Runner.Worker/Worker.cs index b26ae9451a9..4784c169314 100644 --- a/src/Runner.Worker/Worker.cs +++ b/src/Runner.Worker/Worker.cs @@ -22,10 +22,10 @@ public interface IWorker : IRunnerService public sealed class Worker : RunnerService, IWorker { private readonly TimeSpan _workerStartTimeout = TimeSpan.FromSeconds(30); - private ManualResetEvent _completedCommand = new ManualResetEvent(false); + private ManualResetEvent _completedCommand = new(false); // Do not mask the values of these secrets - private static HashSet SecretVariableMaskWhitelist = new HashSet(StringComparer.OrdinalIgnoreCase) + private static HashSet SecretVariableMaskWhitelist = new(StringComparer.OrdinalIgnoreCase) { Constants.Variables.Actions.StepDebug, Constants.Variables.Actions.RunnerDebug diff --git a/src/Test/L0/CommandLineParserL0.cs b/src/Test/L0/CommandLineParserL0.cs index 074660fe748..19ab497fa35 100644 --- a/src/Test/L0/CommandLineParserL0.cs +++ b/src/Test/L0/CommandLineParserL0.cs @@ -15,7 +15,7 @@ public void CanConstruct() { Tracing trace = hc.GetTrace(); - CommandLineParser clp = new CommandLineParser(hc, secretArgNames: new string[0]); + CommandLineParser clp = new(hc, secretArgNames: new string[0]); trace.Info("Constructed"); Assert.NotNull(clp); @@ -30,7 +30,7 @@ public void MasksSecretArgs() using (TestHostContext hc = CreateTestContext()) { // Arrange. - CommandLineParser clp = new CommandLineParser( + CommandLineParser clp = new( hc, secretArgNames: new[] { "SecretArg1", "SecretArg2" }); @@ -61,7 +61,7 @@ public void ParsesCommands() { Tracing trace = hc.GetTrace(); - CommandLineParser clp = new CommandLineParser(hc, secretArgNames: new string[0]); + CommandLineParser clp = new(hc, secretArgNames: new string[0]); trace.Info("Constructed."); clp.Parse(new string[] { "cmd1", "cmd2", "--arg1", "arg1val", "badcmd" }); @@ -81,7 +81,7 @@ public void ParsesArgs() { Tracing trace = hc.GetTrace(); - CommandLineParser clp = new CommandLineParser(hc, secretArgNames: new string[0]); + CommandLineParser clp = new(hc, secretArgNames: new string[0]); trace.Info("Constructed."); clp.Parse(new string[] { "cmd1", "--arg1", "arg1val", "--arg2", "arg2val" }); @@ -105,7 +105,7 @@ public void ParsesFlags() { Tracing trace = hc.GetTrace(); - CommandLineParser clp = new CommandLineParser(hc, secretArgNames: new string[0]); + CommandLineParser clp = new(hc, secretArgNames: new string[0]); trace.Info("Constructed."); clp.Parse(new string[] { "cmd1", "--flag1", "--arg1", "arg1val", "--flag2" }); @@ -120,7 +120,7 @@ public void ParsesFlags() private TestHostContext CreateTestContext([CallerMemberName] string testName = "") { - TestHostContext hc = new TestHostContext(this, testName); + TestHostContext hc = new(this, testName); return hc; } } diff --git a/src/Test/L0/ConstantGenerationL0.cs b/src/Test/L0/ConstantGenerationL0.cs index f2ad6ab23e9..204248516d7 100644 --- a/src/Test/L0/ConstantGenerationL0.cs +++ b/src/Test/L0/ConstantGenerationL0.cs @@ -12,7 +12,7 @@ public sealed class ConstantGenerationL0 [Trait("Category", "Runner")] public void BuildConstantGenerateSucceed() { - List validPackageNames = new List() + List validPackageNames = new() { "win-x64", "win-x86", diff --git a/src/Test/L0/Container/ContainerInfoL0.cs b/src/Test/L0/Container/ContainerInfoL0.cs index 5873652ad1d..6dd74d42cf8 100644 --- a/src/Test/L0/Container/ContainerInfoL0.cs +++ b/src/Test/L0/Container/ContainerInfoL0.cs @@ -12,10 +12,10 @@ public sealed class ContainerInfoL0 public void MountVolumeConstructorParsesStringInput() { // Arrange - MountVolume target = new MountVolume("/dst/dir"); // Maps anonymous Docker volume into target dir - MountVolume source_target = new MountVolume("/src/dir:/dst/dir"); // Maps source to target dir - MountVolume target_ro = new MountVolume("/dst/dir:ro"); - MountVolume source_target_ro = new MountVolume("/src/dir:/dst/dir:ro"); + MountVolume target = new("/dst/dir"); // Maps anonymous Docker volume into target dir + MountVolume source_target = new("/src/dir:/dst/dir"); // Maps source to target dir + MountVolume target_ro = new("/dst/dir:ro"); + MountVolume source_target_ro = new("/src/dir:/dst/dir:ro"); // Assert Assert.Null(target.SourceVolumePath); diff --git a/src/Test/L0/DotnetsdkDownloadScriptL0.cs b/src/Test/L0/DotnetsdkDownloadScriptL0.cs index ea81d6acee1..9d3fbd26c69 100644 --- a/src/Test/L0/DotnetsdkDownloadScriptL0.cs +++ b/src/Test/L0/DotnetsdkDownloadScriptL0.cs @@ -21,7 +21,7 @@ public async Task EnsureDotnetsdkBashDownloadScriptUpToDate() string shDownloadUrl = "https://dot.net/v1/dotnet-install.sh"; - using (HttpClient downloadClient = new HttpClient()) + using (HttpClient downloadClient = new()) { var response = await downloadClient.GetAsync("https://www.bing.com"); if (!response.IsSuccessStatusCode) @@ -51,7 +51,7 @@ public async Task EnsureDotnetsdkPowershellDownloadScriptUpToDate() string ps1DownloadUrl = "https://dot.net/v1/dotnet-install.ps1"; - using (HttpClient downloadClient = new HttpClient()) + using (HttpClient downloadClient = new()) { var response = await downloadClient.GetAsync("https://www.bing.com"); if (!response.IsSuccessStatusCode) diff --git a/src/Test/L0/ExtensionManagerL0.cs b/src/Test/L0/ExtensionManagerL0.cs index 37eb7e65795..98207ab59b4 100644 --- a/src/Test/L0/ExtensionManagerL0.cs +++ b/src/Test/L0/ExtensionManagerL0.cs @@ -13,7 +13,7 @@ public sealed class ExtensionManagerL0 [Trait("Category", "Common")] public void LoadsTypeFromString() { - using (TestHostContext tc = new TestHostContext(this)) + using (TestHostContext tc = new(this)) { // Arrange. var manager = new ExtensionManager(); @@ -34,7 +34,7 @@ public void LoadsTypeFromString() [Trait("Category", "Common")] public void LoadsTypes() { - using (TestHostContext tc = new TestHostContext(this)) + using (TestHostContext tc = new(this)) { // Arrange. var manager = new ExtensionManager(); diff --git a/src/Test/L0/Listener/CommandSettingsL0.cs b/src/Test/L0/Listener/CommandSettingsL0.cs index 014ed5a721e..249d050895e 100644 --- a/src/Test/L0/Listener/CommandSettingsL0.cs +++ b/src/Test/L0/Listener/CommandSettingsL0.cs @@ -9,7 +9,7 @@ namespace GitHub.Runner.Common.Tests { public sealed class CommandSettingsL0 { - private readonly Mock _promptManager = new Mock(); + private readonly Mock _promptManager = new(); // It is sufficient to test one arg only. All individual args are tested by the PromptsFor___ methods. // The PromptsFor___ methods suffice to cover the interesting differences between each of the args. @@ -879,7 +879,7 @@ public void ValidateGoodArgCommandCombination(string validCommand, string arg, s private TestHostContext CreateTestContext([CallerMemberName] string testName = "") { - TestHostContext hc = new TestHostContext(this, testName); + TestHostContext hc = new(this, testName); hc.SetSingleton(_promptManager.Object); return hc; } diff --git a/src/Test/L0/Listener/Configuration/ArgumentValidatorTestsL0.cs b/src/Test/L0/Listener/Configuration/ArgumentValidatorTestsL0.cs index 6f3a62d6207..68128a645e1 100644 --- a/src/Test/L0/Listener/Configuration/ArgumentValidatorTestsL0.cs +++ b/src/Test/L0/Listener/Configuration/ArgumentValidatorTestsL0.cs @@ -10,7 +10,7 @@ public sealed class ArgumentValidatorTestsL0 [Trait("Category", "ArgumentValidator")] public void ServerUrlValidator() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Assert.True(Validators.ServerUrlValidator("http://servername")); Assert.False(Validators.ServerUrlValidator("Fail")); @@ -23,7 +23,7 @@ public void ServerUrlValidator() [Trait("Category", "ArgumentValidator")] public void AuthSchemeValidator() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Assert.True(Validators.AuthSchemeValidator("OAuth")); Assert.False(Validators.AuthSchemeValidator("Fail")); @@ -35,7 +35,7 @@ public void AuthSchemeValidator() [Trait("Category", "ArgumentValidator")] public void NonEmptyValidator() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Assert.True(Validators.NonEmptyValidator("test")); Assert.False(Validators.NonEmptyValidator(string.Empty)); diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index 20ec96a784a..a3fdd5603e8 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -44,7 +44,7 @@ public class ConfigurationManagerL0 private int _defaultRunnerGroupId = 1; private int _secondRunnerGroupId = 2; private RSACryptoServiceProvider rsa = null; - private RunnerSettings _configMgrAgentSettings = new RunnerSettings(); + private RunnerSettings _configMgrAgentSettings = new(); public ConfigurationManagerL0() { @@ -113,7 +113,7 @@ public ConfigurationManagerL0() private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { - TestHostContext tc = new TestHostContext(this, testName); + TestHostContext tc = new(this, testName); tc.SetSingleton(_credMgr.Object); tc.SetSingleton(_promptManager.Object); tc.SetSingleton(_store.Object); diff --git a/src/Test/L0/Listener/Configuration/PromptManagerTestsL0.cs b/src/Test/L0/Listener/Configuration/PromptManagerTestsL0.cs index cfc8e03a3a2..286fc0edb79 100644 --- a/src/Test/L0/Listener/Configuration/PromptManagerTestsL0.cs +++ b/src/Test/L0/Listener/Configuration/PromptManagerTestsL0.cs @@ -12,8 +12,8 @@ public class PromptManagerTestsL0 { private readonly string _argName = "SomeArgName"; private readonly string _description = "Some description"; - private readonly PromptManager _promptManager = new PromptManager(); - private readonly Mock _terminal = new Mock(); + private readonly PromptManager _promptManager = new(); + private readonly Mock _terminal = new(); private readonly string _unattendedExceptionMessage = "Invalid configuration provided for SomeArgName. Terminating unattended configuration."; [Fact] @@ -21,7 +21,7 @@ public class PromptManagerTestsL0 [Trait("Category", "PromptManager")] public void FallsBackToDefault() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. _terminal @@ -46,7 +46,7 @@ public void FallsBackToDefault() [Trait("Category", "PromptManager")] public void FallsBackToDefaultWhenTrimmed() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. _terminal @@ -71,7 +71,7 @@ public void FallsBackToDefaultWhenTrimmed() [Trait("Category", "PromptManager")] public void FallsBackToDefaultWhenUnattended() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. _terminal @@ -98,7 +98,7 @@ public void FallsBackToDefaultWhenUnattended() [Trait("Category", "PromptManager")] public void Prompts() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. _terminal @@ -123,7 +123,7 @@ public void Prompts() [Trait("Category", "PromptManager")] public void PromptsAgainWhenEmpty() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var readLineValues = new Queue(new[] { string.Empty, "Some prompt value" }); @@ -150,7 +150,7 @@ public void PromptsAgainWhenEmpty() [Trait("Category", "PromptManager")] public void PromptsAgainWhenFailsValidation() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var readLineValues = new Queue(new[] { "Some invalid prompt value", "Some valid prompt value" }); @@ -177,7 +177,7 @@ public void PromptsAgainWhenFailsValidation() [Trait("Category", "PromptManager")] public void ThrowsWhenUnattended() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. _terminal diff --git a/src/Test/L0/Listener/Configuration/RunnerCredentialL0.cs b/src/Test/L0/Listener/Configuration/RunnerCredentialL0.cs index 57aa88dce17..a838aa537b8 100644 --- a/src/Test/L0/Listener/Configuration/RunnerCredentialL0.cs +++ b/src/Test/L0/Listener/Configuration/RunnerCredentialL0.cs @@ -14,7 +14,7 @@ public override VssCredentials GetVssCredentials(IHostContext context) trace.Info("GetVssCredentials()"); var loginCred = new VssOAuthAccessTokenCredential("sometoken"); - VssCredentials creds = new VssCredentials(loginCred); + VssCredentials creds = new(loginCred); trace.Verbose("cred created"); return creds; diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index f06005f1374..f1a74d5fe36 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -30,7 +30,7 @@ public JobDispatcherL0() private Pipelines.AgentJobRequestMessage CreateJobRequestMessage() { - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); var result = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "someJob", "someJob", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -101,10 +101,10 @@ public async void DispatcherRenewJobRequest() int count = 0; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequest)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); - TaskAgentJobRequest request = new TaskAgentJobRequest(); + TaskAgentJobRequest request = new(); PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(lockUntilProperty); lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); @@ -159,10 +159,10 @@ public async void DispatcherRenewJobRequestStopOnJobNotFoundExceptions() int count = 0; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnJobNotFoundExceptions)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); - TaskAgentJobRequest request = new TaskAgentJobRequest(); + TaskAgentJobRequest request = new(); PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(lockUntilProperty); lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); @@ -218,10 +218,10 @@ public async void DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions() int count = 0; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); - TaskAgentJobRequest request = new TaskAgentJobRequest(); + TaskAgentJobRequest request = new(); PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(lockUntilProperty); lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); @@ -279,8 +279,8 @@ public async void RenewJobRequestNewAgentNameUpdatesSettings() var reservedAgent = new TaskAgentReference { Name = newName }; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); var request = new Mock(); request.Object.ReservedAgent = reservedAgent; @@ -335,8 +335,8 @@ public async void RenewJobRequestSameAgentNameIgnored() var reservedAgent = new TaskAgentReference { Name = newName }; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); var request = new Mock(); request.Object.ReservedAgent = reservedAgent; @@ -388,8 +388,8 @@ public async void RenewJobRequestNullAgentNameIgnored() var oldSettings = new RunnerSettings { AgentName = oldName }; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); var request = new Mock(); PropertyInfo lockUntilProperty = request.Object.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); @@ -441,10 +441,10 @@ public async void DispatcherRenewJobRequestRecoverFromExceptions() int count = 0; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestRecoverFromExceptions)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); - TaskAgentJobRequest request = new TaskAgentJobRequest(); + TaskAgentJobRequest request = new(); PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(lockUntilProperty); lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); @@ -502,10 +502,10 @@ public async void DispatcherRenewJobRequestFirstRenewRetrySixTimes() int count = 0; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestFirstRenewRetrySixTimes)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); - TaskAgentJobRequest request = new TaskAgentJobRequest(); + TaskAgentJobRequest request = new(); PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(lockUntilProperty); lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); @@ -557,10 +557,10 @@ public async void DispatcherRenewJobRequestStopOnExpiredRequest() int count = 0; var trace = hc.GetTrace(nameof(DispatcherRenewJobRequestStopOnExpiredRequest)); - TaskCompletionSource firstJobRequestRenewed = new TaskCompletionSource(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); - TaskAgentJobRequest request = new TaskAgentJobRequest(); + TaskAgentJobRequest request = new(); PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(lockUntilProperty); lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); diff --git a/src/Test/L0/Listener/MessageListenerL0.cs b/src/Test/L0/Listener/MessageListenerL0.cs index 09725a1a3f2..8ce1dfe0f97 100644 --- a/src/Test/L0/Listener/MessageListenerL0.cs +++ b/src/Test/L0/Listener/MessageListenerL0.cs @@ -36,7 +36,7 @@ public MessageListenerL0() private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { - TestHostContext tc = new TestHostContext(this, testName); + TestHostContext tc = new(this, testName); tc.SetSingleton(_config.Object); tc.SetSingleton(_runnerServer.Object); tc.SetSingleton(_credMgr.Object); @@ -68,7 +68,7 @@ public async void CreatesSession() _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); // Act. - MessageListener listener = new MessageListener(); + MessageListener listener = new(); listener.Initialize(tc); bool result = await listener.CreateSessionAsync(tokenSource.Token); @@ -112,7 +112,7 @@ public async void DeleteSession() _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); // Act. - MessageListener listener = new MessageListener(); + MessageListener listener = new(); listener.Initialize(tc); bool result = await listener.CreateSessionAsync(tokenSource.Token); @@ -159,7 +159,7 @@ public async void GetNextMessage() _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); // Act. - MessageListener listener = new MessageListener(); + MessageListener listener = new(); listener.Initialize(tc); bool result = await listener.CreateSessionAsync(tokenSource.Token); @@ -241,7 +241,7 @@ public async void CreateSessionWithOriginalCredential() _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); // Act. - MessageListener listener = new MessageListener(); + MessageListener listener = new(); listener.Initialize(tc); bool result = await listener.CreateSessionAsync(tokenSource.Token); @@ -285,7 +285,7 @@ public async void SkipDeleteSession_WhenGetNextMessageGetTaskAgentAccessTokenExp _store.Setup(x => x.GetMigratedCredentials()).Returns(default(CredentialData)); // Act. - MessageListener listener = new MessageListener(); + MessageListener listener = new(); listener.Initialize(tc); bool result = await listener.CreateSessionAsync(tokenSource.Token); diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index a1c283a9f0a..f35f7e766fe 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -39,7 +39,7 @@ public RunnerL0() private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) { - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = null; Guid jobId = Guid.NewGuid(); return new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -155,7 +155,7 @@ public async void TestRunAsync() } } - public static TheoryData RunAsServiceTestData = new TheoryData() + public static TheoryData RunAsServiceTestData = new() { // staring with run command, configured as run as service, should start the runner { new [] { "run" }, true, Times.Once() }, diff --git a/src/Test/L0/Listener/SelfUpdaterL0.cs b/src/Test/L0/Listener/SelfUpdaterL0.cs index 43afdae43a4..1bf450366c1 100644 --- a/src/Test/L0/Listener/SelfUpdaterL0.cs +++ b/src/Test/L0/Listener/SelfUpdaterL0.cs @@ -22,8 +22,8 @@ public sealed class SelfUpdaterL0 private Mock _term; private Mock _configStore; private Mock _jobDispatcher; - private AgentRefreshMessage _refreshMessage = new AgentRefreshMessage(1, "2.299.0"); - private List _trimmedPackages = new List(); + private AgentRefreshMessage _refreshMessage = new(1, "2.299.0"); + private List _trimmedPackages = new(); #if !OS_WINDOWS private string _packageUrl = null; @@ -52,7 +52,7 @@ private async Task FetchLatestRunner() if (response.StatusCode == System.Net.HttpStatusCode.Redirect) { var redirectUrl = response.Headers.Location.ToString(); - Regex regex = new Regex(@"/runner/releases/tag/v(?\d+\.\d+\.\d+)"); + Regex regex = new(@"/runner/releases/tag/v(?\d+\.\d+\.\d+)"); var match = regex.Match(redirectUrl); if (match.Success) { diff --git a/src/Test/L0/PackagesTrimL0.cs b/src/Test/L0/PackagesTrimL0.cs index 391e8574581..da3a44a0c26 100644 --- a/src/Test/L0/PackagesTrimL0.cs +++ b/src/Test/L0/PackagesTrimL0.cs @@ -20,7 +20,7 @@ public sealed class PackagesTrimL0 [Trait("Category", "Common")] public async Task RunnerLayoutParts_NewFilesCrossAll() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); var runnerCoreAssetsFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnercoreassets"); @@ -53,7 +53,7 @@ public async Task RunnerLayoutParts_NewFilesCrossAll() [Trait("Category", "Common")] public async Task RunnerLayoutParts_OverlapFiles() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); var runnerCoreAssetsFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnercoreassets"); @@ -77,7 +77,7 @@ public async Task RunnerLayoutParts_OverlapFiles() [Trait("Category", "Common")] public async Task RunnerLayoutParts_NewRunnerCoreAssets() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); var runnerCoreAssetsFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnercoreassets"); @@ -120,7 +120,7 @@ public async Task RunnerLayoutParts_NewRunnerCoreAssets() [Trait("Category", "Common")] public async Task RunnerLayoutParts_NewDotnetRuntimeAssets() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); var runnerDotnetRuntimeFile = Path.Combine(TestUtil.GetSrcPath(), @"Misc/runnerdotnetruntimeassets"); @@ -156,7 +156,7 @@ public async Task RunnerLayoutParts_NewDotnetRuntimeAssets() [Trait("Category", "Common")] public async Task RunnerLayoutParts_CheckDotnetRuntimeHash() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); var dotnetRuntimeHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/dotnetRuntime/{BuildConstants.RunnerPackage.PackageName}"); @@ -217,7 +217,7 @@ public async Task RunnerLayoutParts_CheckDotnetRuntimeHash() [Trait("Category", "Common")] public async Task RunnerLayoutParts_CheckExternalsHash() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); var externalsHashFile = Path.Combine(TestUtil.GetSrcPath(), $"Misc/contentHash/externals/{BuildConstants.RunnerPackage.PackageName}"); diff --git a/src/Test/L0/PagingLoggerL0.cs b/src/Test/L0/PagingLoggerL0.cs index 799428aa6eb..5317b4f9953 100644 --- a/src/Test/L0/PagingLoggerL0.cs +++ b/src/Test/L0/PagingLoggerL0.cs @@ -19,7 +19,7 @@ public PagingLoggerL0() private void CleanLogFolder() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { //clean test data if any old test forgot string pagesFolder = Path.Combine(hc.GetDirectory(WellKnownDirectory.Diag), PagingLogger.PagingFolder); diff --git a/src/Test/L0/ProcessExtensionL0.cs b/src/Test/L0/ProcessExtensionL0.cs index 1fd01e304d5..9708c14953a 100644 --- a/src/Test/L0/ProcessExtensionL0.cs +++ b/src/Test/L0/ProcessExtensionL0.cs @@ -16,7 +16,7 @@ public sealed class ProcessExtensionL0 [Trait("Category", "Common")] public async Task SuccessReadProcessEnv() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); diff --git a/src/Test/L0/ProcessInvokerL0.cs b/src/Test/L0/ProcessInvokerL0.cs index 11dcf8d2ae0..d3d1b54866a 100644 --- a/src/Test/L0/ProcessInvokerL0.cs +++ b/src/Test/L0/ProcessInvokerL0.cs @@ -63,7 +63,7 @@ await processInvoker.ExecuteAsync( [Trait("Category", "Common")] public async Task SuccessExitsWithCodeZero() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -86,7 +86,7 @@ public async Task SuccessExitsWithCodeZero() [Trait("Category", "Common")] public async Task SetCIEnv() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { var existingCI = Environment.GetEnvironmentVariable("CI"); try @@ -134,7 +134,7 @@ public async Task SetCIEnv() [Trait("Category", "Common")] public async Task KeepExistingCIEnv() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { var existingCI = Environment.GetEnvironmentVariable("CI"); try @@ -185,7 +185,7 @@ public async Task KeepExistingCIEnv() public async Task TestCancel() { const int SecondsToRun = 20; - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) using (var tokenSource = new CancellationTokenSource()) { Tracing trace = hc.GetTrace(); @@ -234,13 +234,13 @@ public async Task TestCancel() [Trait("Category", "Common")] public async Task RedirectSTDINCloseStream() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + CancellationTokenSource cancellationTokenSource = new(); Int32 exitCode = -1; Channel redirectSTDIN = Channel.CreateUnbounded(new UnboundedChannelOptions() { SingleReader = true, SingleWriter = true }); - List stdout = new List(); + List stdout = new(); redirectSTDIN.Writer.TryWrite("Single line of STDIN"); var processInvoker = new ProcessInvokerWrapper(); @@ -284,13 +284,13 @@ public async Task RedirectSTDINCloseStream() [Trait("Category", "Common")] public async Task RedirectSTDINKeepStreamOpen() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); - CancellationTokenSource cancellationTokenSource = new CancellationTokenSource(); + CancellationTokenSource cancellationTokenSource = new(); Int32 exitCode = -1; Channel redirectSTDIN = Channel.CreateUnbounded(new UnboundedChannelOptions() { SingleReader = true, SingleWriter = true }); - List stdout = new List(); + List stdout = new(); redirectSTDIN.Writer.TryWrite("Single line of STDIN"); var processInvoker = new ProcessInvokerWrapper(); @@ -339,7 +339,7 @@ public async Task OomScoreAdjIsWriten_Default() string testProcPath = $"/proc/{Process.GetCurrentProcess().Id}/oom_score_adj"; if (File.Exists(testProcPath)) { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) using (var tokenSource = new CancellationTokenSource()) { Tracing trace = hc.GetTrace(); @@ -375,7 +375,7 @@ public async Task OomScoreAdjIsWriten_FromEnv() string testProcPath = $"/proc/{Process.GetCurrentProcess().Id}/oom_score_adj"; if (File.Exists(testProcPath)) { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) using (var tokenSource = new CancellationTokenSource()) { Tracing trace = hc.GetTrace(); @@ -415,7 +415,7 @@ public async Task OomScoreAdjIsInherited() { int testProcOomScoreAdj = 123; File.WriteAllText(testProcPath, testProcOomScoreAdj.ToString()); - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) using (var tokenSource = new CancellationTokenSource()) { Tracing trace = hc.GetTrace(); diff --git a/src/Test/L0/RunnerWebProxyL0.cs b/src/Test/L0/RunnerWebProxyL0.cs index 621dbd96ff7..649f06197a7 100644 --- a/src/Test/L0/RunnerWebProxyL0.cs +++ b/src/Test/L0/RunnerWebProxyL0.cs @@ -11,9 +11,9 @@ namespace GitHub.Runner.Common.Tests { public sealed class RunnerWebProxyL0 { - private static readonly Regex NewHttpClientHandlerRegex = new Regex("New\\s+HttpClientHandler\\s*\\(", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private static readonly Regex NewHttpClientRegex = new Regex("New\\s+HttpClient\\s*\\(\\s*\\)", RegexOptions.Compiled | RegexOptions.IgnoreCase); - private static readonly List SkippedFiles = new List() + private static readonly Regex NewHttpClientHandlerRegex = new("New\\s+HttpClientHandler\\s*\\(", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly Regex NewHttpClientRegex = new("New\\s+HttpClient\\s*\\(\\s*\\)", RegexOptions.Compiled | RegexOptions.IgnoreCase); + private static readonly List SkippedFiles = new() { "Runner.Common\\HostContext.cs", "Runner.Common/HostContext.cs", @@ -39,7 +39,7 @@ public void IsNotUseRawHttpClientHandler() "*.cs", SearchOption.AllDirectories)); - List badCode = new List(); + List badCode = new(); foreach (string sourceFile in sourceFiles) { // Skip skipped files. @@ -86,7 +86,7 @@ public void IsNotUseRawHttpClient() "*.cs", SearchOption.AllDirectories)); - List badCode = new List(); + List badCode = new(); foreach (string sourceFile in sourceFiles) { // Skip skipped files. diff --git a/src/Test/L0/ServiceControlManagerL0.cs b/src/Test/L0/ServiceControlManagerL0.cs index f94ded572c2..4d5ab543425 100644 --- a/src/Test/L0/ServiceControlManagerL0.cs +++ b/src/Test/L0/ServiceControlManagerL0.cs @@ -11,7 +11,7 @@ public sealed class ServiceControlManagerL0 [Trait("Category", "Service")] public void CalculateServiceName() { - RunnerSettings settings = new RunnerSettings(); + RunnerSettings settings = new(); settings.AgentName = "thisiskindofalongrunnerabcde"; settings.ServerUrl = "https://example.githubusercontent.com/12345678901234567890123456789012345678901234567890"; @@ -22,7 +22,7 @@ public void CalculateServiceName() using (TestHostContext hc = CreateTestContext()) { - ServiceControlManager scm = new ServiceControlManager(); + ServiceControlManager scm = new(); scm.Initialize(hc); scm.CalculateServiceName( @@ -50,7 +50,7 @@ public void CalculateServiceName() [Trait("Category", "Service")] public void CalculateServiceName80Chars() { - RunnerSettings settings = new RunnerSettings(); + RunnerSettings settings = new(); settings.AgentName = "thisiskindofalongrunnernabcde"; settings.ServerUrl = "https://example.githubusercontent.com/12345678901234567890123456789012345678901234567890"; @@ -61,7 +61,7 @@ public void CalculateServiceName80Chars() using (TestHostContext hc = CreateTestContext()) { - ServiceControlManager scm = new ServiceControlManager(); + ServiceControlManager scm = new(); scm.Initialize(hc); scm.CalculateServiceName( @@ -169,7 +169,7 @@ public void CalculateServiceNameSanitizeOutOfRangeChars() [Trait("Category", "Service")] public void CalculateServiceNameLimitsServiceNameTo150Chars() { - RunnerSettings settings = new RunnerSettings(); + RunnerSettings settings = new(); settings.AgentName = "thisisareallyreallylongbutstillvalidagentnameiamusingforthisexampletotestverylongnamelimits"; settings.ServerUrl = "https://example.githubusercontent.com/12345678901234567890123456789012345678901234567890"; @@ -180,7 +180,7 @@ public void CalculateServiceNameLimitsServiceNameTo150Chars() using (TestHostContext hc = CreateTestContext()) { - ServiceControlManager scm = new ServiceControlManager(); + ServiceControlManager scm = new(); scm.Initialize(hc); scm.CalculateServiceName( @@ -206,7 +206,7 @@ public void CalculateServiceNameLimitsServiceNameTo150Chars() private TestHostContext CreateTestContext([CallerMemberName] string testName = "") { - TestHostContext hc = new TestHostContext(this, testName); + TestHostContext hc = new(this, testName); return hc; } diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index ce1ec01bcbe..a88b9170ea1 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -17,12 +17,12 @@ namespace GitHub.Runner.Common.Tests { public sealed class TestHostContext : IHostContext, IDisposable { - private readonly ConcurrentDictionary> _serviceInstances = new ConcurrentDictionary>(); - private readonly ConcurrentDictionary _serviceSingletons = new ConcurrentDictionary(); + private readonly ConcurrentDictionary> _serviceInstances = new(); + private readonly ConcurrentDictionary _serviceSingletons = new(); private readonly ITraceManager _traceManager; private readonly Terminal _term; private readonly SecretMasker _secretMasker; - private CancellationTokenSource _runnerShutdownTokenSource = new CancellationTokenSource(); + private CancellationTokenSource _runnerShutdownTokenSource = new(); private string _suiteName; private string _testName; private Tracing _trace; @@ -86,9 +86,9 @@ public StartupType StartupType } } - public List UserAgents => new List() { new ProductInfoHeaderValue("L0Test", "0.0") }; + public List UserAgents => new() { new ProductInfoHeaderValue("L0Test", "0.0") }; - public RunnerWebProxy WebProxy => new RunnerWebProxy(); + public RunnerWebProxy WebProxy => new(); public async Task Delay(TimeSpan delay, CancellationToken token) { diff --git a/src/Test/L0/Util/ArgUtilL0.cs b/src/Test/L0/Util/ArgUtilL0.cs index e0e0b34545d..5dd29de8e3f 100644 --- a/src/Test/L0/Util/ArgUtilL0.cs +++ b/src/Test/L0/Util/ArgUtilL0.cs @@ -11,7 +11,7 @@ public sealed class ArgUtilL0 [Trait("Category", "Common")] public void Equal_MatchesObjectEquality() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -29,12 +29,12 @@ public void Equal_MatchesObjectEquality() [Trait("Category", "Common")] public void Equal_MatchesReferenceEquality() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); // Arrange. - object expected = new object(); + object expected = new(); object actual = expected; // Act/Assert. @@ -47,7 +47,7 @@ public void Equal_MatchesReferenceEquality() [Trait("Category", "Common")] public void Equal_MatchesStructEquality() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -65,12 +65,12 @@ public void Equal_MatchesStructEquality() [Trait("Category", "Common")] public void Equal_ThrowsWhenActualObjectIsNull() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); // Arrange. - object expected = new object(); + object expected = new(); object actual = null; // Act/Assert. @@ -86,13 +86,13 @@ public void Equal_ThrowsWhenActualObjectIsNull() [Trait("Category", "Common")] public void Equal_ThrowsWhenExpectedObjectIsNull() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); // Arrange. object expected = null; - object actual = new object(); + object actual = new(); // Act/Assert. Assert.Throws(() => @@ -107,13 +107,13 @@ public void Equal_ThrowsWhenExpectedObjectIsNull() [Trait("Category", "Common")] public void Equal_ThrowsWhenObjectsAreNotEqual() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); // Arrange. - object expected = new object(); - object actual = new object(); + object expected = new(); + object actual = new(); // Act/Assert. Assert.Throws(() => @@ -128,7 +128,7 @@ public void Equal_ThrowsWhenObjectsAreNotEqual() [Trait("Category", "Common")] public void Equal_ThrowsWhenStructsAreNotEqual() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); diff --git a/src/Test/L0/Util/IOUtilL0.cs b/src/Test/L0/Util/IOUtilL0.cs index 6a990c7898e..6a504d19221 100644 --- a/src/Test/L0/Util/IOUtilL0.cs +++ b/src/Test/L0/Util/IOUtilL0.cs @@ -15,7 +15,7 @@ public sealed class IOUtilL0 [Trait("Category", "Common")] public void Delete_DeletesDirectory() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -49,7 +49,7 @@ public void Delete_DeletesDirectory() [Trait("Category", "Common")] public void Delete_DeletesFile() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -83,7 +83,7 @@ public void Delete_DeletesFile() [Trait("Category", "Common")] public void DeleteDirectory_DeletesDirectoriesRecursively() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -115,7 +115,7 @@ public void DeleteDirectory_DeletesDirectoriesRecursively() [Trait("Category", "Common")] public async Task DeleteDirectory_DeletesDirectoryReparsePointChain() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -179,7 +179,7 @@ public async Task DeleteDirectory_DeletesDirectoryReparsePointChain() [Trait("Category", "Common")] public async Task DeleteDirectory_DeletesDirectoryReparsePointsBeforeDirectories() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -230,7 +230,7 @@ public async Task DeleteDirectory_DeletesDirectoryReparsePointsBeforeDirectories [Trait("Category", "Common")] public void DeleteDirectory_DeletesFilesRecursively() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -264,7 +264,7 @@ public void DeleteDirectory_DeletesFilesRecursively() [Trait("Category", "Common")] public void DeleteDirectory_DeletesReadOnlyDirectories() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -305,7 +305,7 @@ public void DeleteDirectory_DeletesReadOnlyDirectories() [Trait("Category", "Common")] public void DeleteDirectory_DeletesReadOnlyRootDirectory() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -341,7 +341,7 @@ public void DeleteDirectory_DeletesReadOnlyRootDirectory() [Trait("Category", "Common")] public void DeleteDirectory_DeletesReadOnlyFiles() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -381,7 +381,7 @@ public void DeleteDirectory_DeletesReadOnlyFiles() [Trait("Category", "Common")] public async Task DeleteDirectory_DoesNotFollowDirectoryReparsePoint() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -428,7 +428,7 @@ public async Task DeleteDirectory_DoesNotFollowDirectoryReparsePoint() [Trait("Category", "Common")] public async Task DeleteDirectory_DoesNotFollowNestLevel1DirectoryReparsePoint() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -477,7 +477,7 @@ public async Task DeleteDirectory_DoesNotFollowNestLevel1DirectoryReparsePoint() [Trait("Category", "Common")] public async Task DeleteDirectory_DoesNotFollowNestLevel2DirectoryReparsePoint() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -528,7 +528,7 @@ public async Task DeleteDirectory_DoesNotFollowNestLevel2DirectoryReparsePoint() [Trait("Category", "Common")] public void DeleteDirectory_IgnoresFile() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -563,7 +563,7 @@ public void DeleteDirectory_IgnoresFile() [Trait("Category", "Common")] public void DeleteFile_DeletesFile() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -597,7 +597,7 @@ public void DeleteFile_DeletesFile() [Trait("Category", "Common")] public void DeleteFile_DeletesReadOnlyFile() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -637,7 +637,7 @@ public void DeleteFile_DeletesReadOnlyFile() [Trait("Category", "Common")] public void DeleteFile_IgnoresDirectory() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -670,7 +670,7 @@ public void DeleteFile_IgnoresDirectory() [Trait("Category", "Common")] public void GetRelativePath() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -768,7 +768,7 @@ public void GetRelativePath() [Trait("Category", "Common")] public void ResolvePath() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -867,7 +867,7 @@ public void ResolvePath() [Trait("Category", "Common")] public void ValidateExecutePermission_DoesNotExceedFailsafe() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -896,7 +896,7 @@ public void ValidateExecutePermission_DoesNotExceedFailsafe() [Trait("Category", "Common")] public void ValidateExecutePermission_ExceedsFailsafe() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); diff --git a/src/Test/L0/Util/StringUtilL0.cs b/src/Test/L0/Util/StringUtilL0.cs index c67b3922df2..e6618187478 100644 --- a/src/Test/L0/Util/StringUtilL0.cs +++ b/src/Test/L0/Util/StringUtilL0.cs @@ -12,7 +12,7 @@ public class StringUtilL0 [Trait("Category", "Common")] public void FormatAlwaysCallsFormat() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -48,7 +48,7 @@ public void FormatAlwaysCallsFormat() [Trait("Category", "Common")] public void FormatHandlesFormatException() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); @@ -79,7 +79,7 @@ public void FormatHandlesFormatException() [Trait("Category", "Common")] public void FormatUsesInvariantCulture() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. CultureInfo originalCulture = CultureInfo.CurrentCulture; @@ -105,7 +105,7 @@ public void FormatUsesInvariantCulture() [Trait("Category", "Common")] public void ConvertNullOrEmptryStringToBool() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. string nullString = null; @@ -126,7 +126,7 @@ public void ConvertNullOrEmptryStringToBool() [Trait("Category", "Common")] public void ConvertNullOrEmptryStringToDefaultBool() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. string nullString = null; @@ -147,7 +147,7 @@ public void ConvertNullOrEmptryStringToDefaultBool() [Trait("Category", "Common")] public void ConvertStringToBool() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. string trueString1 = "1"; diff --git a/src/Test/L0/Util/TaskResultUtilL0.cs b/src/Test/L0/Util/TaskResultUtilL0.cs index b12d07c6ab4..331e71555dc 100644 --- a/src/Test/L0/Util/TaskResultUtilL0.cs +++ b/src/Test/L0/Util/TaskResultUtilL0.cs @@ -12,7 +12,7 @@ public class TaskResultUtilL0 public void TaskResultReturnCodeTranslate() { // Arrange. - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Act. TaskResult abandon = TaskResultUtil.TranslateFromReturnCode(TaskResultUtil.TranslateToReturnCode(TaskResult.Abandoned)); @@ -57,7 +57,7 @@ public void TaskResultReturnCodeTranslate() public void TaskResultsMerge() { // Arrange. - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { TaskResult merged; diff --git a/src/Test/L0/Util/VssUtilL0.cs b/src/Test/L0/Util/VssUtilL0.cs index 30ed8569219..2965dcc068d 100644 --- a/src/Test/L0/Util/VssUtilL0.cs +++ b/src/Test/L0/Util/VssUtilL0.cs @@ -12,7 +12,7 @@ public sealed class VssUtilL0 [Trait("Category", "Common")] public void VerifyOverwriteVssConnectionSetting() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { Tracing trace = hc.GetTrace(); diff --git a/src/Test/L0/Util/WhichUtilL0.cs b/src/Test/L0/Util/WhichUtilL0.cs index 7271bc283bd..c6a05714442 100644 --- a/src/Test/L0/Util/WhichUtilL0.cs +++ b/src/Test/L0/Util/WhichUtilL0.cs @@ -13,7 +13,7 @@ public sealed class WhichUtilL0 [Trait("Category", "Common")] public void UseWhichFindGit() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { //Arrange Tracing trace = hc.GetTrace(); @@ -33,7 +33,7 @@ public void UseWhichFindGit() [Trait("Category", "Common")] public void WhichReturnsNullWhenNotFound() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { //Arrange Tracing trace = hc.GetTrace(); @@ -53,7 +53,7 @@ public void WhichReturnsNullWhenNotFound() [Trait("Category", "Common")] public void WhichThrowsWhenRequireAndNotFound() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { //Arrange Tracing trace = hc.GetTrace(); @@ -76,7 +76,7 @@ public void WhichThrowsWhenRequireAndNotFound() [Trait("Category", "Common")] public void WhichHandleFullyQualifiedPath() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { //Arrange Tracing trace = hc.GetTrace(); diff --git a/src/Test/L0/Worker/ActionCommandL0.cs b/src/Test/L0/Worker/ActionCommandL0.cs index 4429976f2bb..314c437fa72 100644 --- a/src/Test/L0/Worker/ActionCommandL0.cs +++ b/src/Test/L0/Worker/ActionCommandL0.cs @@ -18,7 +18,7 @@ public void CommandParserTest() string message; ActionCommand test; ActionCommand verify; - HashSet commands = new HashSet() { "do-something" }; + HashSet commands = new() { "do-something" }; //##[do-something k1=v1;]msg message = "##[do-something k1=v1;]msg"; test = new ActionCommand("do-something") @@ -99,7 +99,7 @@ public void CommandParserV2Test() string message; ActionCommand test; ActionCommand verify; - HashSet commands = new HashSet() { "do-something" }; + HashSet commands = new() { "do-something" }; //::do-something k1=v1;]msg message = "::do-something k1=v1,::msg"; test = new ActionCommand("do-something") diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index cc74397e2ba..cef826fd94e 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -228,8 +228,8 @@ public void EchoProcessCommandDebugOn() { // Set up a few things // 1. Job request message (with ACTIONS_STEP_DEBUG = true) - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 971d051cad6..07cf9b48700 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -1028,7 +1028,7 @@ public void LoadsContainerRegistryActionDefinition() //Arrange Setup(); - Pipelines.ActionStep instance = new Pipelines.ActionStep() + Pipelines.ActionStep instance = new() { Id = Guid.NewGuid(), Reference = new Pipelines.ContainerRegistryReference() @@ -1065,7 +1065,7 @@ public void LoadsScriptActionDefinition() //Arrange Setup(); - Pipelines.ActionStep instance = new Pipelines.ActionStep() + Pipelines.ActionStep instance = new() { Id = Guid.NewGuid(), Reference = new Pipelines.ScriptReference() @@ -1137,7 +1137,7 @@ public void LoadsContainerActionDefinitionDockerfile() Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1236,7 +1236,7 @@ public void LoadsContainerActionDefinitionRegistry() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1328,7 +1328,7 @@ public void LoadsNode12ActionDefinition() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1397,7 +1397,7 @@ public void LoadsNode16ActionDefinition() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1479,7 +1479,7 @@ public void LoadsNodeActionDefinitionYaml() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1555,7 +1555,7 @@ public void LoadsContainerActionDefinitionDockerfile_SelfRepo() Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1653,7 +1653,7 @@ public void LoadsContainerActionDefinitionRegistry_SelfRepo() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1745,7 +1745,7 @@ public void LoadsNodeActionDefinition_SelfRepo() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1814,7 +1814,7 @@ public void LoadsNodeActionDefinition_Cleanup() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1893,7 +1893,7 @@ public void LoadsContainerActionDefinitionDockerfile_Cleanup() Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; @@ -1983,7 +1983,7 @@ public void LoadsPluginActionDefinition() Assert.Equal(directory, definition.Directory); Assert.NotNull(definition.Data); Assert.NotNull(definition.Data.Inputs); // inputs - Dictionary inputDefaults = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary inputDefaults = new(StringComparer.OrdinalIgnoreCase); foreach (var input in definition.Data.Inputs) { var name = input.Key.AssertString("key").Value; diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index 1f77e1114c5..31d0743659b 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -34,7 +34,7 @@ public sealed class ActionRunnerL0 private IActionManifestManager _actionManifestManager; private Mock _fileCommandManager; - private DictionaryContextData _context = new DictionaryContextData(); + private DictionaryContextData _context = new(); [Fact] [Trait("Level", "L0")] @@ -60,7 +60,7 @@ public async void MergeDefaultInputs() _actionRunner.Action = action; - Dictionary finialInputs = new Dictionary(); + Dictionary finialInputs = new(); _handlerFactory.Setup(x => x.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback((IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference actionReference, IStepHost stepHost, ActionExecutionData data, Dictionary inputs, Dictionary environment, Variables runtimeVariables, string taskDirectory, List localActionContainerSetupSteps) => { @@ -106,7 +106,7 @@ public async void WriteEventPayload() _actionRunner.Action = action; - Dictionary finialInputs = new Dictionary(); + Dictionary finialInputs = new(); _handlerFactory.Setup(x => x.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback((IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference actionReference, IStepHost stepHost, ActionExecutionData data, Dictionary inputs, Dictionary environment, Variables runtimeVariables, string taskDirectory, List localActionContainerSetupSteps) => { @@ -307,7 +307,7 @@ public async void WarnInvalidInputs() _actionRunner.Action = action; - Dictionary finialInputs = new Dictionary(); + Dictionary finialInputs = new(); _handlerFactory.Setup(x => x.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback((IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference actionReference, IStepHost stepHost, ActionExecutionData data, Dictionary inputs, Dictionary environment, Variables runtimeVariables, string taskDirectory, List localActionContainerSetupSteps) => { @@ -358,7 +358,7 @@ public async void SetGitHubContextActionRepoRef() _actionRunner.Action = action; - Dictionary finialInputs = new Dictionary(); + Dictionary finialInputs = new(); _handlerFactory.Setup(x => x.Create(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny>(), It.IsAny>(), It.IsAny(), It.IsAny(), It.IsAny>())) .Callback((IExecutionContext executionContext, Pipelines.ActionStepDefinitionReference actionReference, IStepHost stepHost, ActionExecutionData data, Dictionary inputs, Dictionary environment, Variables runtimeVariables, string taskDirectory, List localActionContainerSetupSteps) => { diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index e0819511bc7..1cec0d4daa5 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -22,12 +22,12 @@ public sealed class ContainerOperationProviderL0 private ContainerOperationProvider containerOperationProvider; private Mock serverQueue; private Mock pagingLogger; - private List healthyDockerStatus = new List { "healthy" }; - private List emptyDockerStatus = new List { string.Empty }; - private List unhealthyDockerStatus = new List { "unhealthy" }; - private List dockerLogs = new List { "log1", "log2", "log3" }; + private List healthyDockerStatus = new() { "healthy" }; + private List emptyDockerStatus = new() { string.Empty }; + private List unhealthyDockerStatus = new() { "unhealthy" }; + private List dockerLogs = new() { "log1", "log2", "log3" }; - List containers = new List(); + List containers = new(); [Fact] [Trait("Level", "L0")] diff --git a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs index 46fe5260882..67631ed4bd7 100644 --- a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs +++ b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs @@ -189,8 +189,8 @@ private TestHostContext Setup([CallerMemberName] string name = "") _issues = new List>(); // Setup a job request - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "Summary Job"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 670fece20c4..937d964e867 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -25,8 +25,8 @@ public void AddIssue_CountWarningsErrors() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -101,8 +101,8 @@ public void ApplyContinueOnError_CheckResultAndOutcome() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -157,8 +157,8 @@ public void AddIssue_TrimMessageSize() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -211,8 +211,8 @@ public void AddIssue_AddStepAndLineNumberInformation() using (TestHostContext hc = CreateTestContext()) { - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -260,8 +260,8 @@ public void Debug_Multilines() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -311,8 +311,8 @@ public void RegisterPostJobAction_ShareState() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -409,8 +409,8 @@ public void RegisterPostJobAction_NotRegisterPostTwice() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -493,8 +493,8 @@ public void ActionResult_Lowercase() { using (TestHostContext hc = CreateTestContext()) { - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -548,8 +548,8 @@ public void PublishStepTelemetry_RegularStep_NoOpt() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -591,8 +591,8 @@ public void PublishStepTelemetry_RegularStep() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -655,8 +655,8 @@ public void PublishStepTelemetry_EmbeddedStep() using (TestHostContext hc = CreateTestContext()) { // Arrange: Create a job request message. - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); @@ -851,8 +851,8 @@ public void ActionVariables_AddedToVarsContext() { using (TestHostContext hc = CreateTestContext()) { - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); - TimelineReference timeline = new TimelineReference(); + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); Guid jobId = Guid.NewGuid(); string jobName = "some job name"; var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); diff --git a/src/Test/L0/Worker/HandlerFactoryL0.cs b/src/Test/L0/Worker/HandlerFactoryL0.cs index f046d9803c8..18cf8e8ff07 100644 --- a/src/Test/L0/Worker/HandlerFactoryL0.cs +++ b/src/Test/L0/Worker/HandlerFactoryL0.cs @@ -56,7 +56,7 @@ public void IsNodeVersionUpgraded(string inputVersion, string serverFeatureFlag, { variables["DistributedTask.ForceGithubJavascriptActionsToNode16"] = serverFeatureFlag; } - Variables serverVariables = new Variables(hc, variables); + Variables serverVariables = new(hc, variables); // Workflow opt-out var workflowVariables = new Dictionary(); diff --git a/src/Test/L0/Worker/JobExtensionL0.cs b/src/Test/L0/Worker/JobExtensionL0.cs index e3955c65398..ce7dca1a51b 100644 --- a/src/Test/L0/Worker/JobExtensionL0.cs +++ b/src/Test/L0/Worker/JobExtensionL0.cs @@ -67,10 +67,10 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " } _tokenSource = new CancellationTokenSource(); - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = new Timeline(Guid.NewGuid()); - List steps = new List() + List steps = new() { new Pipelines.ActionStep() { @@ -101,7 +101,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " Guid jobId = Guid.NewGuid(); _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, "test", "test", null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), steps, null, null, null, null); - GitHubContext github = new GitHubContext(); + GitHubContext github = new(); github["repository"] = new Pipelines.ContextData.StringContextData("actions/runner"); github["secret_source"] = new Pipelines.ContextData.StringContextData("Actions"); _message.ContextData.Add("github", github); diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index 1bb7f1ac099..152c816aa2b 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -15,7 +15,7 @@ public sealed class JobRunnerL0 { private IExecutionContext _jobEc; private JobRunner _jobRunner; - private List _initResult = new List(); + private List _initResult = new(); private Pipelines.AgentJobRequestMessage _message; private CancellationTokenSource _tokenSource; private Mock _jobServer; @@ -55,7 +55,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " _jobRunner = new JobRunner(); _jobRunner.Initialize(hc); - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TaskOrchestrationPlanReference plan = new(); TimelineReference timeline = new Timeline(Guid.NewGuid()); Guid jobId = Guid.NewGuid(); _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); diff --git a/src/Test/L0/Worker/PipelineDirectoryManagerL0.cs b/src/Test/L0/Worker/PipelineDirectoryManagerL0.cs index 4cc6d4bca12..d2ad4a89e33 100644 --- a/src/Test/L0/Worker/PipelineDirectoryManagerL0.cs +++ b/src/Test/L0/Worker/PipelineDirectoryManagerL0.cs @@ -199,13 +199,13 @@ private TestHostContext Setup( [CallerMemberName] string name = "") { // Setup the host context. - TestHostContext hc = new TestHostContext(this, name); + TestHostContext hc = new(this, name); // Setup the execution context. _ec = new Mock(); _ec.Setup(x => x.Global).Returns(new GlobalContext()); - GitHubContext githubContext = new GitHubContext(); + GitHubContext githubContext = new(); _ec.Setup(x => x.GetGitHubContext("repository")).Returns("actions/runner"); // Store the expected tracking file path. diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index e090dd1c086..3ff1e304a4a 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -27,7 +27,7 @@ public sealed class StepsRunnerL0 private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { var hc = new TestHostContext(this, testName); - Dictionary variablesToCopy = new Dictionary(); + Dictionary variablesToCopy = new(); _variables = new Variables( hostContext: hc, copy: variablesToCopy); diff --git a/src/Test/L0/Worker/TrackingManagerL0.cs b/src/Test/L0/Worker/TrackingManagerL0.cs index 8b2c1bffc46..a016943371c 100644 --- a/src/Test/L0/Worker/TrackingManagerL0.cs +++ b/src/Test/L0/Worker/TrackingManagerL0.cs @@ -16,14 +16,14 @@ public sealed class TrackingManagerL0 public TestHostContext Setup([CallerMemberName] string name = "") { // Setup the host context. - TestHostContext hc = new TestHostContext(this, name); + TestHostContext hc = new(this, name); // Create a random work path. _workFolder = hc.GetDirectory(WellKnownDirectory.Work); // Setup the execution context. _ec = new Mock(); - GitHubContext githubContext = new GitHubContext(); + GitHubContext githubContext = new(); _ec.Setup(x => x.GetGitHubContext("repository")).Returns("actions/runner"); // Setup the tracking manager. @@ -113,7 +113,7 @@ public void UpdatesTrackingConfigJobRunProperties() using (TestHostContext hc = Setup()) { // Arrange. - TrackingConfig config = new TrackingConfig() { RepositoryName = "actions/runner" }; + TrackingConfig config = new() { RepositoryName = "actions/runner" }; string trackingFile = Path.Combine(_workFolder, "trackingconfig.json"); // Act. diff --git a/src/Test/L0/Worker/VariablesL0.cs b/src/Test/L0/Worker/VariablesL0.cs index 5b7d50b4db3..9c9d24538e8 100644 --- a/src/Test/L0/Worker/VariablesL0.cs +++ b/src/Test/L0/Worker/VariablesL0.cs @@ -14,7 +14,7 @@ public sealed class VariablesL0 [Trait("Category", "Worker")] public void Constructor_AppliesMaskHints() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var copy = new Dictionary @@ -36,7 +36,7 @@ public void Constructor_AppliesMaskHints() [Trait("Category", "Worker")] public void Constructor_HandlesNullValue() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var copy = new Dictionary @@ -59,7 +59,7 @@ public void Constructor_HandlesNullValue() [Trait("Category", "Worker")] public void Constructor_SetsNullAsEmpty() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var copy = new Dictionary @@ -80,7 +80,7 @@ public void Constructor_SetsNullAsEmpty() [Trait("Category", "Worker")] public void Constructor_SetsOrdinalIgnoreCaseComparer() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. CultureInfo currentCulture = CultureInfo.CurrentCulture; @@ -115,7 +115,7 @@ public void Constructor_SetsOrdinalIgnoreCaseComparer() [Trait("Category", "Worker")] public void Constructor_SkipVariableWithEmptyName() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var copy = new Dictionary @@ -139,7 +139,7 @@ public void Constructor_SkipVariableWithEmptyName() [Trait("Category", "Worker")] public void Get_ReturnsNullIfNotFound() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var variables = new Variables(hc, new Dictionary()); @@ -157,7 +157,7 @@ public void Get_ReturnsNullIfNotFound() [Trait("Category", "Worker")] public void GetBoolean_DoesNotThrowWhenNull() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var variables = new Variables(hc, new Dictionary()); @@ -175,7 +175,7 @@ public void GetBoolean_DoesNotThrowWhenNull() [Trait("Category", "Worker")] public void GetEnum_DoesNotThrowWhenNull() { - using (TestHostContext hc = new TestHostContext(this)) + using (TestHostContext hc = new(this)) { // Arrange. var variables = new Variables(hc, new Dictionary()); diff --git a/src/Test/L0/Worker/WorkerL0.cs b/src/Test/L0/Worker/WorkerL0.cs index 80e5eaa0aa4..fe1f9c02c19 100644 --- a/src/Test/L0/Worker/WorkerL0.cs +++ b/src/Test/L0/Worker/WorkerL0.cs @@ -25,17 +25,17 @@ public WorkerL0() private Pipelines.AgentJobRequestMessage CreateJobRequestMessage(string jobName) { - TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference() { PlanId = Guid.NewGuid() }; + TaskOrchestrationPlanReference plan = new() { PlanId = Guid.NewGuid() }; TimelineReference timeline = null; - Dictionary variables = new Dictionary(StringComparer.OrdinalIgnoreCase); + Dictionary variables = new(StringComparer.OrdinalIgnoreCase); variables[Constants.Variables.System.Culture] = "en-US"; - Pipelines.JobResources resources = new Pipelines.JobResources(); + Pipelines.JobResources resources = new(); var serviceEndpoint = new ServiceEndpoint(); serviceEndpoint.Authorization = new EndpointAuthorization(); serviceEndpoint.Authorization.Parameters.Add("nullValue", null); resources.Endpoints.Add(serviceEndpoint); - List actions = new List(); + List actions = new(); actions.Add(new Pipelines.ActionStep() { Id = Guid.NewGuid(), From 985c95b982b9b00b3190d1882c8920c11acb91c1 Mon Sep 17 00:00:00 2001 From: Cory Miller <13227161+cory-miller@users.noreply.github.com> Date: Tue, 18 Oct 2022 12:14:27 -0400 Subject: [PATCH 039/145] Add linter workflow (#2213) --- .github/workflows/lint.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/workflows/lint.yml diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml new file mode 100644 index 00000000000..b35bee3c7d9 --- /dev/null +++ b/.github/workflows/lint.yml @@ -0,0 +1,24 @@ +name: Lint + +on: + pull_request: + branches: [ main ] + +jobs: + build: + name: Lint + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v3 + with: + # Ensure full list of changed files within `super-linter` + fetch-depth: 0 + - name: Run linters + uses: github/super-linter@v4 + env: + DEFAULT_BRANCH: ${{ github.base_ref }} + EDITORCONFIG_FILE_NAME: .editorconfig + LINTER_RULES_PATH: /src/ + VALIDATE_ALL_CODEBASE: false + VALIDATE_CSHARP: true From d26dc55f4ba296a144c805e6326938d1d9399df2 Mon Sep 17 00:00:00 2001 From: Cory Miller <13227161+cory-miller@users.noreply.github.com> Date: Thu, 20 Oct 2022 18:59:39 -0400 Subject: [PATCH 040/145] Disable linter errors (#2216) --- .github/workflows/lint.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index b35bee3c7d9..9df0fcc0895 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,6 +18,7 @@ jobs: uses: github/super-linter@v4 env: DEFAULT_BRANCH: ${{ github.base_ref }} + DISABLE_ERRORS: true EDITORCONFIG_FILE_NAME: .editorconfig LINTER_RULES_PATH: /src/ VALIDATE_ALL_CODEBASE: false From 351e962c3135e53bda0290f0d10f70f57a99d414 Mon Sep 17 00:00:00 2001 From: Cory Miller <13227161+cory-miller@users.noreply.github.com> Date: Wed, 26 Oct 2022 08:48:23 -0400 Subject: [PATCH 041/145] Add generateServiceConfig option for configure command (#2226) For runners that are already configured but need to add systemd services after the fact, a new command option is added to generate the file only. Once generated, ./svc.sh install and other commands will be available. This also adds support for falling back to the tenant name in the Actions URL in cases when the GitHub URL is not provided, such as for our hosted larger runners. --- src/Runner.Common/ConfigurationStore.cs | 5 +- src/Runner.Common/Constants.cs | 1 + src/Runner.Listener/CommandSettings.cs | 6 +- .../Configuration/ConfigurationManager.cs | 21 ++++ .../Configuration/ConfigurationManagerL0.cs | 98 +++++++++++++++++++ 5 files changed, 127 insertions(+), 4 deletions(-) diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index b3cb26a86be..7daa4d66fdd 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -77,14 +77,15 @@ public string RepoOrOrgName Uri accountUri = new(this.ServerUrl); string repoOrOrgName = string.Empty; - if (accountUri.Host.EndsWith(".githubusercontent.com", StringComparison.OrdinalIgnoreCase)) + if (accountUri.Host.EndsWith(".githubusercontent.com", StringComparison.OrdinalIgnoreCase) && !string.IsNullOrEmpty(this.GitHubUrl)) { Uri gitHubUrl = new(this.GitHubUrl); // Use the "NWO part" from the GitHub URL path repoOrOrgName = gitHubUrl.AbsolutePath.Trim('/'); } - else + + if (string.IsNullOrEmpty(repoOrOrgName)) { repoOrOrgName = accountUri.AbsolutePath.Split('/', StringSplitOptions.RemoveEmptyEntries).FirstOrDefault(); } diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 33ab3ce0451..2c3a6e2996a 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -128,6 +128,7 @@ public static class Flags public static readonly string Check = "check"; public static readonly string Commit = "commit"; public static readonly string Ephemeral = "ephemeral"; + public static readonly string GenerateServiceConfig = "generateServiceConfig"; public static readonly string Help = "help"; public static readonly string Replace = "replace"; public static readonly string DisableUpdate = "disableupdate"; diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index 44039bb5cb8..4d00f1d4592 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -34,6 +34,7 @@ public sealed class CommandSettings { Constants.Runner.CommandLine.Flags.DisableUpdate, Constants.Runner.CommandLine.Flags.Ephemeral, + Constants.Runner.CommandLine.Flags.GenerateServiceConfig, Constants.Runner.CommandLine.Flags.Replace, Constants.Runner.CommandLine.Flags.RunAsService, Constants.Runner.CommandLine.Flags.Unattended, @@ -79,11 +80,12 @@ public sealed class CommandSettings // Flags. public bool Check => TestFlag(Constants.Runner.CommandLine.Flags.Check); public bool Commit => TestFlag(Constants.Runner.CommandLine.Flags.Commit); + public bool DisableUpdate => TestFlag(Constants.Runner.CommandLine.Flags.DisableUpdate); + public bool Ephemeral => TestFlag(Constants.Runner.CommandLine.Flags.Ephemeral); + public bool GenerateServiceConfig => TestFlag(Constants.Runner.CommandLine.Flags.GenerateServiceConfig); public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help); public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended); public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version); - public bool Ephemeral => TestFlag(Constants.Runner.CommandLine.Flags.Ephemeral); - public bool DisableUpdate => TestFlag(Constants.Runner.CommandLine.Flags.DisableUpdate); // Keep this around since customers still relies on it public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once); diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 72b574873ea..25217f9bee4 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -81,6 +81,27 @@ public async Task ConfigureAsync(CommandSettings command) _term.WriteLine("--------------------------------------------------------------------------------"); Trace.Info(nameof(ConfigureAsync)); + + if (command.GenerateServiceConfig) + { +#if OS_LINUX + if (!IsConfigured()) + { + throw new InvalidOperationException("--generateServiceConfig requires that the runner is already configured. For configuring a new runner as a service, run './config.sh'."); + } + + RunnerSettings settings = _store.GetSettings(); + + Trace.Info($"generate service config for runner: {settings.AgentId}"); + var controlManager = HostContext.GetService(); + controlManager.GenerateScripts(settings); + + return; +#else + throw new NotSupportedException("--generateServiceConfig is only supported on Linux."); +#endif + } + if (IsConfigured()) { throw new InvalidOperationException("Cannot configure the runner because it is already configured. To reconfigure the runner, run 'config.cmd remove' or './config.sh remove' first."); diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index a3fdd5603e8..ca6e90b6bf7 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -234,5 +234,103 @@ public async Task ConfigureErrorOnMissingRunnerGroup() _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny(), It.Is(p => p == TaskAgentPoolType.Automation)), Times.Exactly(1)); } } + +#if OS_LINUX + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "ConfigurationManagement")] + public async Task ConfigureRunnerServiceFailsOnUnconfiguredRunners() + { + using (TestHostContext tc = CreateTestContext()) + { + Tracing trace = tc.GetTrace(); + + trace.Info("Creating config manager"); + IConfigurationManager configManager = new ConfigurationManager(); + configManager.Initialize(tc); + + trace.Info("Preparing command line arguments"); + var command = new CommandSettings( + tc, + new[] + { + "configure", + "--generateServiceConfig", + }); + trace.Info("Constructed"); + _store.Setup(x => x.IsConfigured()).Returns(false); + + trace.Info("Ensuring service generation mode fails when on un-configured runners"); + var ex = await Assert.ThrowsAsync(() => configManager.ConfigureAsync(command)); + + Assert.Contains("requires that the runner is already configured", ex.Message); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "ConfigurationManagement")] + public async Task ConfigureRunnerServiceCreatesService() + { + using (TestHostContext tc = CreateTestContext()) + { + Tracing trace = tc.GetTrace(); + + trace.Info("Creating config manager"); + IConfigurationManager configManager = new ConfigurationManager(); + configManager.Initialize(tc); + + trace.Info("Preparing command line arguments"); + var command = new CommandSettings( + tc, + new[] + { + "configure", + "--generateServiceConfig", + }); + trace.Info("Constructed"); + + _store.Setup(x => x.IsConfigured()).Returns(true); + + trace.Info("Ensuring service generation mode fails when on un-configured runners"); + await configManager.ConfigureAsync(command); + + _serviceControlManager.Verify(x => x.GenerateScripts(It.IsAny()), Times.Once); + } + } +#endif + +#if !OS_LINUX + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "ConfigurationManagement")] + public async Task ConfigureRunnerServiceFailsOnUnsupportedPlatforms() + { + using (TestHostContext tc = CreateTestContext()) + { + Tracing trace = tc.GetTrace(); + + trace.Info("Creating config manager"); + IConfigurationManager configManager = new ConfigurationManager(); + configManager.Initialize(tc); + + trace.Info("Preparing command line arguments"); + var command = new CommandSettings( + tc, + new[] + { + "configure", + "--generateServiceConfig", + }); + trace.Info("Constructed"); + _store.Setup(x => x.IsConfigured()).Returns(true); + + trace.Info("Ensuring service generation mode fails on unsupported runner platforms"); + var ex = await Assert.ThrowsAsync(() => configManager.ConfigureAsync(command)); + + Assert.Contains("only supported on Linux", ex.Message); + } + } +#endif } } From b4f0c45a3a063e14bfcfbe11334ce2378224d6a2 Mon Sep 17 00:00:00 2001 From: Tauhid Anjum Date: Mon, 31 Oct 2022 19:21:33 +0530 Subject: [PATCH 042/145] Setting debug using GitHub Action variables (#2234) --- src/Runner.Worker/ExecutionContext.cs | 32 ++++++++- src/Test/L0/Worker/ExecutionContextL0.cs | 89 ++++++++++++++++++++++++ 2 files changed, 119 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index d7ff528b1d3..f4aa14d01d1 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -686,8 +686,11 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Endpoints Global.Endpoints = message.Resources.Endpoints; - // Variables - Global.Variables = new Variables(HostContext, message.Variables); + // Ser debug using vars context if debug variables are not already present. + var variables = message.Variables; + SetDebugUsingVars(variables, message.ContextData); + + Global.Variables = new Variables(HostContext, variables); if (Global.Variables.GetBoolean("DistributedTask.ForceInternalNodeVersionOnRunnerTo12") ?? false) { @@ -1077,6 +1080,31 @@ private IExecutionContext CreatePostChild(string displayName, Dictionary variables, IDictionary contextData) + { + if (contextData != null && + contextData.TryGetValue(PipelineTemplateConstants.Vars, out var varsPipelineContextData) && + varsPipelineContextData != null && + varsPipelineContextData is DictionaryContextData varsContextData) + { + // Set debug variables only when StepDebug/RunnerDebug variables are not present. + if (!variables.ContainsKey(Constants.Variables.Actions.StepDebug) && + varsContextData.TryGetValue(Constants.Variables.Actions.StepDebug, out var stepDebugValue) && + stepDebugValue is StringContextData) + { + variables[Constants.Variables.Actions.StepDebug] = stepDebugValue.ToString(); + } + + if (!variables.ContainsKey(Constants.Variables.Actions.RunnerDebug) && + varsContextData.TryGetValue(Constants.Variables.Actions.RunnerDebug, out var runDebugValue) && + runDebugValue is StringContextData) + { + variables[Constants.Variables.Actions.RunnerDebug] = runDebugValue.ToString(); + } + } + } + public void ApplyContinueOnError(TemplateToken continueOnErrorToken) { if (Result != TaskResult.Failed) diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 937d964e867..3aef7fb58ff 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -889,6 +889,95 @@ public void ActionVariables_AddedToVarsContext() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void ActionVariables_DebugUsingVars() + { + using (TestHostContext hc = CreateTestContext()) + { + TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TimelineReference timeline = new TimelineReference(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + + var inputVarsContext = new DictionaryContextData(); + + inputVarsContext[Constants.Variables.Actions.StepDebug] = new StringContextData("true"); + inputVarsContext[Constants.Variables.Actions.RunnerDebug] = new StringContextData("true"); + jobRequest.ContextData["vars"] = inputVarsContext; + + // Arrange: Setup the paging logger. + var pagingLogger1 = new Mock(); + var jobServerQueue = new Mock(); + hc.EnqueueInstance(pagingLogger1.Object); + hc.SetSingleton(jobServerQueue.Object); + + var jobContext = new Runner.Worker.ExecutionContext(); + jobContext.Initialize(hc); + + jobContext.InitializeJob(jobRequest, CancellationToken.None); + + + Assert.Equal("true", jobContext.Global.Variables.Get(Constants.Variables.Actions.StepDebug)); + Assert.Equal("true", jobContext.Global.Variables.Get(Constants.Variables.Actions.RunnerDebug)); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void ActionVariables_SecretsPrecedenceForDebugUsingVars() + { + using (TestHostContext hc = CreateTestContext()) + { + TaskOrchestrationPlanReference plan = new TaskOrchestrationPlanReference(); + TimelineReference timeline = new TimelineReference(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + + var inputVarsContext = new DictionaryContextData(); + + inputVarsContext[Constants.Variables.Actions.StepDebug] = new StringContextData("true"); + inputVarsContext[Constants.Variables.Actions.RunnerDebug] = new StringContextData("true"); + jobRequest.ContextData["vars"] = inputVarsContext; + + jobRequest.Variables[Constants.Variables.Actions.StepDebug] = "false"; + jobRequest.Variables[Constants.Variables.Actions.RunnerDebug] = "false"; + + // Arrange: Setup the paging logger. + var pagingLogger1 = new Mock(); + var jobServerQueue = new Mock(); + hc.EnqueueInstance(pagingLogger1.Object); + hc.SetSingleton(jobServerQueue.Object); + + var jobContext = new Runner.Worker.ExecutionContext(); + jobContext.Initialize(hc); + + jobContext.InitializeJob(jobRequest, CancellationToken.None); + + + Assert.Equal("false", jobContext.Global.Variables.Get(Constants.Variables.Actions.StepDebug)); + Assert.Equal("false", jobContext.Global.Variables.Get(Constants.Variables.Actions.RunnerDebug)); + } + } + private bool ExpressionValuesAssertEqual(DictionaryContextData expect, DictionaryContextData actual) { foreach (var key in expect.Keys.ToList()) From d019e5b9951c78027a29299f4e0a536393fdbb20 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 2 Nov 2022 11:27:37 +0100 Subject: [PATCH 043/145] Allow the container.image to be empty (#2235) * Allow container.image to be null/empty * Remove unnecessary workflow type * Update src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> * Rename ConvertToJobContainer to ConvertToContainer It converts serviceContainers too * Remove unused parameter AllowExpressions was always false * Allow service in yaml with empty string for value * Don't attempt to start services without an image * Improve error messages * Revert "Remove unused parameter" This reverts commit f4ab2d50d0cbf7287460c206cec817434839c0c7. * Revert "Rename ConvertToJobContainer to ConvertToContainer" This reverts commit ffc62f834328af5c8509dfaff75e4cbdcc4e8238. * Fix revert merge conflict * Removed info logs * Added info log for services without container images Co-authored-by: moonblade Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Co-authored-by: Ava Stancu --- src/Runner.Worker/JobExtension.cs | 5 +++++ .../Pipelines/ObjectTemplating/PipelineTemplateConverter.cs | 2 +- src/Sdk/DTPipelines/workflow-v1.0.json | 4 ++-- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 58e1b11371a..246274734d9 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -220,6 +220,11 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel { var networkAlias = pair.Key; var serviceContainer = pair.Value; + if (serviceContainer == null) + { + context.Output($"The service '{networkAlias}' will not be started because the container definition has an empty image."); + continue; + } jobContext.Global.ServiceContainers.Add(new Container.ContainerInfo(HostContext, serviceContainer, false, networkAlias)); } } diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs index 69f9a3d32cf..506a7d2689c 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateConverter.cs @@ -316,7 +316,7 @@ internal static JobContainer ConvertToJobContainer( if (String.IsNullOrEmpty(result.Image)) { - context.Error(value, "Container image cannot be empty"); + return null; } return result; diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index 6ae301fbd4c..a3745029b3d 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -381,7 +381,7 @@ "container-mapping": { "mapping": { "properties": { - "image": "non-empty-string", + "image": "string", "options": "non-empty-string", "env": "container-env", "ports": "sequence-of-non-empty-string", @@ -414,7 +414,7 @@ "vars" ], "one-of": [ - "non-empty-string", + "string", "container-mapping" ] }, From aae4dd81af88b1ab6886a8c3a6f4bb2d83288a21 Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:28:14 +0100 Subject: [PATCH 044/145] run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233) * run.sh installs SIGINT and SIGTERM traps to gracefully stop runner * replaced trap to force wait for child processes and to send kill to run-helper instead of Runner.Listener * run with signal handling if RUNNER_MANUALLY_TRAP_SIG is set * Update src/Misc/layoutroot/run.sh Co-authored-by: Tingluo Huang Co-authored-by: Tingluo Huang --- src/Misc/layoutroot/run.sh | 62 ++++++++++++++++++++++++++++++-------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/src/Misc/layoutroot/run.sh b/src/Misc/layoutroot/run.sh index ca9e5247d61..e6610a7f82e 100755 --- a/src/Misc/layoutroot/run.sh +++ b/src/Misc/layoutroot/run.sh @@ -9,16 +9,52 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -# run the helper process which keep the listener alive -while :; -do - cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh - "$DIR"/run-helper.sh $* - returnCode=$? - if [[ $returnCode -eq 2 ]]; then - echo "Restarting runner..." - else - echo "Exiting runner..." - exit 0 - fi -done + +run() { + # run the helper process which keep the listener alive + while :; + do + cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh + "$DIR"/run-helper.sh $* + returnCode=$? + if [[ $returnCode -eq 2 ]]; then + echo "Restarting runner..." + else + echo "Exiting runner..." + exit 0 + fi + done +} + +runWithManualTrap() { + # Set job control + set -m + + trap 'kill -INT -$PID' INT TERM + + # run the helper process which keep the listener alive + while :; + do + cp -f "$DIR"/run-helper.sh.template "$DIR"/run-helper.sh + "$DIR"/run-helper.sh $* & + PID=$! + wait -f $PID + returnCode=$? + if [[ $returnCode -eq 2 ]]; then + echo "Restarting runner..." + else + echo "Exiting runner..." + # Unregister signal handling before exit + trap - INT TERM + # wait for last parts to be logged + wait $PID + exit 0 + fi + done +} + +if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then + run +else + runWithManualTrap +fi \ No newline at end of file From fe189bd063a2edcf3803ccbd964cb5dab9447752 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 2 Nov 2022 09:40:19 -0400 Subject: [PATCH 045/145] Use Global.Variables instead of JobContext and include action path/ref in the message. (#2214) * Use Global.Variables instead of JobContext and include action path/ref in the message. * encoding * . --- .editorconfig | 1 + .../Handlers/NodeScriptActionHandler.cs | 27 ++++++++++++++----- src/Runner.Worker/JobRunner.cs | 6 ++--- src/Runner.Worker/Variables.cs | 10 +++++-- 4 files changed, 32 insertions(+), 12 deletions(-) diff --git a/.editorconfig b/.editorconfig index 37c44b8cfb8..76e6ed4e620 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,6 +1,7 @@ # https://editorconfig.org/ [*] +charset = utf-8 # Set default charset to utf-8 insert_final_newline = true # ensure all files end with a single newline trim_trailing_whitespace = true # attempt to remove trailing whitespace on save diff --git a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs index fc743f1b113..fb77137d95b 100644 --- a/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs +++ b/src/Runner.Worker/Handlers/NodeScriptActionHandler.cs @@ -1,4 +1,5 @@ -using System; +using System; +using System.Collections.Generic; using System.IO; using System.Linq; using System.Text; @@ -7,8 +8,8 @@ using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; -using GitHub.Runner.Sdk; using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Container.ContainerHooks; @@ -137,13 +138,25 @@ public async Task RunAsync(ActionRunStage stage) if (Data.NodeVersion == "node12" && (ExecutionContext.Global.Variables.GetBoolean(Constants.Runner.Features.Node12Warning) ?? false)) { - if (!ExecutionContext.JobContext.ContainsKey("Node12ActionsWarnings")) + var repoAction = Action as RepositoryPathReference; + var warningActions = new HashSet(); + if (ExecutionContext.Global.Variables.TryGetValue("Node12ActionsWarnings", out var node12Warnings)) { - ExecutionContext.JobContext["Node12ActionsWarnings"] = new ArrayContextData(); + warningActions = StringUtil.ConvertFromJson>(node12Warnings); } - var repoAction = Action as RepositoryPathReference; - var actionDisplayName = new StringContextData(repoAction.Name ?? repoAction.Path); // local actions don't have a 'Name' - ExecutionContext.JobContext["Node12ActionsWarnings"].AssertArray("Node12ActionsWarnings").Add(actionDisplayName); + + var repoActionFullName = ""; + if (string.IsNullOrEmpty(repoAction.Name)) + { + repoActionFullName = repoAction.Path; // local actions don't have a 'Name' + } + else + { + repoActionFullName = $"{repoAction.Name}/{repoAction.Path ?? string.Empty}".TrimEnd('/') + $"@{repoAction.Ref}"; + } + + warningActions.Add(repoActionFullName); + ExecutionContext.Global.Variables.Set("Node12ActionsWarnings", StringUtil.ConvertToJson(warningActions)); } using (var stdoutManager = new OutputManager(ExecutionContext, ActionCommandManager)) diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index ac2ebd42a38..562ef9068fd 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -258,9 +258,9 @@ private async Task CompleteJobAsync(IJobServer jobServer, IExecution } } - if (jobContext.JobContext.ContainsKey("Node12ActionsWarnings")) + if (jobContext.Global.Variables.TryGetValue("Node12ActionsWarnings", out var node12Warnings)) { - var actions = string.Join(", ", jobContext.JobContext["Node12ActionsWarnings"].AssertArray("Node12ActionsWarnings").Select(action => action.ToString())); + var actions = string.Join(", ", StringUtil.ConvertFromJson>(node12Warnings)); jobContext.Warning(string.Format(Constants.Runner.Node12DetectedAfterEndOfLife, actions)); } diff --git a/src/Runner.Worker/Variables.cs b/src/Runner.Worker/Variables.cs index 3284bb7e154..2969bf6101f 100644 --- a/src/Runner.Worker/Variables.cs +++ b/src/Runner.Worker/Variables.cs @@ -1,10 +1,10 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; -using GitHub.DistributedTask.WebApi; using GitHub.DistributedTask.Logging; using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; @@ -136,6 +136,12 @@ public string Get(string name) return null; } + public void Set(string name, string val) + { + ArgUtil.NotNullOrEmpty(name, nameof(name)); + _variables[name] = new Variable(name, val, false); + } + public bool TryGetValue(string name, out string val) { Variable variable; From fde61366e0874901c7951251bb744d6059f5e52b Mon Sep 17 00:00:00 2001 From: Cory Miller <13227161+cory-miller@users.noreply.github.com> Date: Wed, 2 Nov 2022 14:55:03 -0400 Subject: [PATCH 046/145] prepare release notes for 2.299.0 (#2239) --- releaseNote.md | 15 +++-- src/Test/L0/Listener/SelfUpdaterL0.cs | 82 +++++++++++++-------------- src/runnerversion | 2 +- 3 files changed, 52 insertions(+), 47 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 10b9a6301a4..e7af4964fc1 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,13 +1,18 @@ ## Features -- [REVERTED] Service containers startup error logs are now included in workflow's logs (#2110) - - Reverted due to https://github.com/actions/runner/issues/2173 +- Displays the error logs in dedicated sub-sections of the Initialize containers section (#2182) +- Add generateServiceConfig option for configure command (#2226) +- Setting debug using GitHub Action variables (#2234) +- run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233) + ## Bugs -- Fixed missing SHA for Windows arm64 release archive (#2171) +- Use Global.Variables instead of JobContext and include action path/ref in the message. (#2214) ## Misc -- Added a feature flag to start warning on `save-state` and `set-output` deprecation (#2164) -- Prepare supporting `vars` in workflow templates (#2096) +- Allow '--disableupdate' in create-latest-svc.sh (#2201) +- Fix markup for support link (#2114) +- Add runner devcontainer (#2187) +- Setup linter for Runner (#2211, #2213, #2216) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/Test/L0/Listener/SelfUpdaterL0.cs b/src/Test/L0/Listener/SelfUpdaterL0.cs index 1bf450366c1..2bbdf6b4d9e 100644 --- a/src/Test/L0/Listener/SelfUpdaterL0.cs +++ b/src/Test/L0/Listener/SelfUpdaterL0.cs @@ -22,7 +22,7 @@ public sealed class SelfUpdaterL0 private Mock _term; private Mock _configStore; private Mock _jobDispatcher; - private AgentRefreshMessage _refreshMessage = new(1, "2.299.0"); + private AgentRefreshMessage _refreshMessage = new(1, "2.999.0"); private List _trimmedPackages = new(); #if !OS_WINDOWS @@ -77,8 +77,8 @@ private async Task FetchLatestRunner() _trimmedPackages = StringUtil.ConvertFromJson>(json); } - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl })); } @@ -127,13 +127,13 @@ public async void TestSelfUpdateAsync() { var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); } finally { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); } } } @@ -219,8 +219,8 @@ public async void TestSelfUpdateAsync_DownloadRetry() hc.SetSingleton(_configStore.Object); hc.SetSingleton(new HttpClientHandlerFactory()); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = $"https://github.com/actions/runner/notexists" })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = $"https://github.com/actions/runner/notexists" })); var p1 = new ProcessInvokerWrapper(); p1.Initialize(hc); @@ -274,8 +274,8 @@ public async void TestSelfUpdateAsync_ValidateHash() hc.SetSingleton(_configStore.Object); hc.SetSingleton(new HttpClientHandlerFactory()); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, HashValue = "bad_hash" })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, HashValue = "bad_hash" })); var p1 = new ProcessInvokerWrapper(); p1.Initialize(hc); @@ -340,8 +340,8 @@ public async void TestSelfUpdateAsync_CloneHash_RuntimeAndExternals() hc.EnqueueInstance(p3); updater.Initialize(hc); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = new List() { new TrimmedPackageMetadata() } })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = new List() { new TrimmedPackageMetadata() } })); _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) .Callback((int p, int a, string s, string t) => @@ -354,8 +354,8 @@ public async void TestSelfUpdateAsync_CloneHash_RuntimeAndExternals() { var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); FieldInfo contentHashesProperty = updater.GetType().GetField("_contentHashes", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); Assert.NotNull(contentHashesProperty); @@ -370,8 +370,8 @@ public async void TestSelfUpdateAsync_CloneHash_RuntimeAndExternals() } finally { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); } } } @@ -486,8 +486,8 @@ public async void TestSelfUpdateAsync_UseExternalsTrimmedPackage() updater.Initialize(hc); var trim = _trimmedPackages.Where(x => !x.TrimmedContents.ContainsKey("dotnetRuntime")).ToList(); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) .Callback((int p, int a, string s, string t) => @@ -500,13 +500,13 @@ public async void TestSelfUpdateAsync_UseExternalsTrimmedPackage() { var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); } finally { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); } var traceFile = Path.GetTempFileName(); @@ -575,8 +575,8 @@ public async void TestSelfUpdateAsync_UseExternalsRuntimeTrimmedPackage() updater.Initialize(hc); var trim = _trimmedPackages.Where(x => x.TrimmedContents.ContainsKey("dotnetRuntime") && x.TrimmedContents.ContainsKey("externals")).ToList(); - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) .Callback((int p, int a, string s, string t) => @@ -589,13 +589,13 @@ public async void TestSelfUpdateAsync_UseExternalsRuntimeTrimmedPackage() { var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); } finally { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); } var traceFile = Path.GetTempFileName(); @@ -676,8 +676,8 @@ public async void TestSelfUpdateAsync_NotUseExternalsRuntimeTrimmedPackageOnHash } } - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) @@ -691,13 +691,13 @@ public async void TestSelfUpdateAsync_NotUseExternalsRuntimeTrimmedPackageOnHash { var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); } finally { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); } var traceFile = Path.GetTempFileName(); @@ -754,8 +754,8 @@ public async void TestSelfUpdateAsync_FallbackToFullPackage() package.HashValue = "mismatch"; } - _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.299.0", true, It.IsAny())) - .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.299.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); + _runnerServer.Setup(x => x.GetPackageAsync("agent", BuildConstants.RunnerPackage.PackageName, "2.999.0", true, It.IsAny())) + .Returns(Task.FromResult(new PackageMetadata() { Platform = BuildConstants.RunnerPackage.PackageName, Version = new PackageVersion("2.999.0"), DownloadUrl = _packageUrl, TrimmedPackages = trim })); _runnerServer.Setup(x => x.UpdateAgentUpdateStateAsync(1, 1, It.IsAny(), It.IsAny())) .Callback((int p, int a, string s, string t) => @@ -768,13 +768,13 @@ public async void TestSelfUpdateAsync_FallbackToFullPackage() { var result = await updater.SelfUpdate(_refreshMessage, _jobDispatcher.Object, true, hc.RunnerShutdownToken); Assert.True(result); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"))); - Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"))); + Assert.True(Directory.Exists(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"))); } finally { - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.299.0"), CancellationToken.None); - IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.299.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "bin.2.999.0"), CancellationToken.None); + IOUtil.DeleteDirectory(Path.Combine(hc.GetDirectory(WellKnownDirectory.Root), "externals.2.999.0"), CancellationToken.None); } var traceFile = Path.GetTempFileName(); diff --git a/src/runnerversion b/src/runnerversion index 6f38b5b7a45..bf5c74669fd 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.298.2 +2.299.0 From 813a252776bde7ae74f45edca43b01c6e817496e Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 2 Nov 2022 18:13:40 -0400 Subject: [PATCH 047/145] Forward parameters into run() func in run.sh. (#2240) --- src/Misc/layoutroot/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Misc/layoutroot/run.sh b/src/Misc/layoutroot/run.sh index e6610a7f82e..89902eb2e7a 100755 --- a/src/Misc/layoutroot/run.sh +++ b/src/Misc/layoutroot/run.sh @@ -54,7 +54,7 @@ runWithManualTrap() { } if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then - run + run $* else - runWithManualTrap + runWithManualTrap $* fi \ No newline at end of file From dd8bce032419b7d178954be38f5b7da4b3359126 Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Thu, 10 Nov 2022 18:08:13 +0100 Subject: [PATCH 048/145] Dockerfile and workflow change for runner image (#2250) * Workflow * add back github-token to id:image * added handle to image name * removed core require * multi line added :latest * added label * added repository_owner in IMAGE_NAME * with the release * release * markdown label description * Remove markdown desciprion * Remove double quotes in labels * Reverted back releaseVersion --- .github/workflows/publish-image.yml | 65 +++++++++++++++++++++++++++++ .github/workflows/release.yml | 49 ++++++++++++++++++++++ images/Dockerfile | 21 ++++++++++ 3 files changed, 135 insertions(+) create mode 100644 .github/workflows/publish-image.yml create mode 100644 images/Dockerfile diff --git a/.github/workflows/publish-image.yml b/.github/workflows/publish-image.yml new file mode 100644 index 00000000000..bf0dbfc27e8 --- /dev/null +++ b/.github/workflows/publish-image.yml @@ -0,0 +1,65 @@ +name: Publish Runner Image + +on: + workflow_dispatch: + inputs: + runnerVersion: + type: string + description: Version of the runner being installed + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/actions-runner + +jobs: + build: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Compute image version + id: image + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const inputRunnerVersion = "${{ github.event.inputs.runnerVersion }}" + if (inputRunnerVersion) { + console.log(`Using input runner version ${inputRunnerVersion}`) + core.setOutput('version', inputRunnerVersion); + return + } + const runnerVersion = fs.readFileSync('${{ github.workspace }}/src/runnerversion', 'utf8').replace(/\n$/g, '') + console.log(`Using runner version ${runnerVersion}`) + core.setOutput('version', runnerVersion); + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v3 + with: + context: ./images + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.image.outputs.version }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + build-args: | + RUNNER_VERSION=${{ steps.image.outputs.version }} + push: true + labels: | + org.opencontainers.image.source=${{github.server_url}}/${{github.repository}} + org.opencontainers.image.description=https://github.com/actions/runner/releases/tag/v${{ steps.image.outputs.version }} + org.opencontainers.image.licenses=MIT diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index abad70edaa7..ad9f0d77fa9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -660,3 +660,52 @@ jobs: asset_path: ${{ github.workspace }}/linux-arm64-trimmedpackages.json asset_name: actions-runner-linux-arm64-${{ steps.releaseNote.outputs.version }}-trimmedpackages.json asset_content_type: application/octet-stream + + publish-image: + needs: release + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository_owner }}/actions-runner + steps: + - name: Checkout repository + uses: actions/checkout@v3 + + - name: Compute image version + id: image + uses: actions/github-script@v6 + with: + script: | + const fs = require('fs'); + const runnerVersion = fs.readFileSync('${{ github.workspace }}/releaseVersion', 'utf8').replace(/\n$/g, '') + console.log(`Using runner version ${runnerVersion}`) + core.setOutput('version', runnerVersion); + + - name: Setup Docker buildx + uses: docker/setup-buildx-action@v2 + + - name: Log into registry ${{ env.REGISTRY }} + uses: docker/login-action@v2 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Build and push Docker image + id: build-and-push + uses: docker/build-push-action@v3 + with: + context: ./images + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.image.outputs.version }} + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:latest + build-args: | + RUNNER_VERSION=${{ steps.image.outputs.version }} + push: true + labels: | + org.opencontainers.image.source=${{github.server_url}}/${{github.repository}} + org.opencontainers.image.description=https://github.com/actions/runner/releases/tag/v${{ steps.image.outputs.version }} + org.opencontainers.image.licenses=MIT diff --git a/images/Dockerfile b/images/Dockerfile new file mode 100644 index 00000000000..411ce08eba2 --- /dev/null +++ b/images/Dockerfile @@ -0,0 +1,21 @@ +FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build + +ARG RUNNER_VERSION +ARG RUNNER_ARCH="x64" +ARG RUNNER_CONTAINER_HOOKS_VERSION=0.1.3 + +RUN apt update -y && apt install curl unzip -y + +WORKDIR /actions-runner +RUN curl -f -L -o runner.tar.gz https://github.com/actions/runner/releases/download/v${RUNNER_VERSION}/actions-runner-linux-${RUNNER_ARCH}-${RUNNER_VERSION}.tar.gz \ + && tar xzf ./runner.tar.gz \ + && rm runner.tar.gz + +RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-container-hooks/releases/download/v${RUNNER_CONTAINER_HOOKS_VERSION}/actions-runner-hooks-k8s-${RUNNER_CONTAINER_HOOKS_VERSION}.zip \ + && unzip ./runner-container-hooks.zip -d ./k8s \ + && rm runner-container-hooks.zip + +FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 + +WORKDIR /actions-runner +COPY --from=build /actions-runner . From dc9fe01b6e6470425b3d3750fcc434b948a921a6 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 15 Nov 2022 12:38:11 -0500 Subject: [PATCH 049/145] Add `RUNNER_ALLOW_RUNASROOT=1` to dockerfile (#2254) --- images/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/images/Dockerfile b/images/Dockerfile index 411ce08eba2..b87755b0d74 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -17,5 +17,7 @@ RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-c FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 +ENV RUNNER_ALLOW_RUNASROOT=1 + WORKDIR /actions-runner COPY --from=build /actions-runner . From 3351ea37715d24adc61bee2a9f0921dd464f2f4a Mon Sep 17 00:00:00 2001 From: Chris Patterson Date: Wed, 16 Nov 2022 10:25:59 -0500 Subject: [PATCH 050/145] Small change to Node.js 12 deprecation message (#2262) * Small change to Node.js 12 deprecation message * Update src/Runner.Common/Constants.cs --- src/Runner.Common/Constants.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 2c3a6e2996a..f5d489ea99d 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -164,7 +164,7 @@ public static class Features public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/"; public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`."; public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary"; - public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/. Please update the following actions to use Node.js 16: {0}"; + public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/."; } public static class RunnerEvent From fdfbdfdfaea81fbbc83cafac15ecb05771803599 Mon Sep 17 00:00:00 2001 From: Amit Rathi <55986583+amitrathi2040@users.noreply.github.com> Date: Thu, 17 Nov 2022 21:41:52 +0530 Subject: [PATCH 051/145] expose github.actor_id, github.workflow_ref & github.workflow_sha as environment variable (#2249) * expose workflow refs/sha as environment variables * fixes environment variable ordering * job_workflow_ref/sha aren't available in gh ctx --- src/Runner.Worker/GitHubContext.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/GitHubContext.cs b/src/Runner.Worker/GitHubContext.cs index f92850cc65b..c7482c2e346 100644 --- a/src/Runner.Worker/GitHubContext.cs +++ b/src/Runner.Worker/GitHubContext.cs @@ -13,6 +13,7 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "action_repository", "action", "actor", + "actor_id", "api_url", "base_ref", "env", @@ -27,8 +28,10 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "ref_protected", "ref_type", "ref", - "repository_owner", "repository", + "repository_id", + "repository_owner", + "repository_owner_id", "retention_days", "run_attempt", "run_id", @@ -39,7 +42,9 @@ public sealed class GitHubContext : DictionaryContextData, IEnvironmentContextDa "step_summary", "triggering_actor", "workflow", - "workspace", + "workflow_ref", + "workflow_sha", + "workspace" }; public IEnumerable> GetRuntimeEnvironmentVariables() From eea1f6d5b5c02ad9eee0d41d6e36803200791cd7 Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Fri, 18 Nov 2022 16:07:59 +0100 Subject: [PATCH 052/145] Updated info on process for requesting features and enhancements (#2259) --- README.md | 2 +- docs/contribute.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index a534eeb3260..7b79464264e 100644 --- a/README.md +++ b/README.md @@ -22,4 +22,4 @@ Runner releases: ## Contribute -We accept contributions in the form of issues and pull requests. [Read more here](docs/contribute.md) before contributing. +We accept contributions in the form of issues and pull requests. The runner typically requires changes across the entire system and we aim for issues in the runner to be entirely self contained and fixable here. Therefore, we will primarily handle bug issues opened in this repo and we kindly request you to create all feature and enhancement requests on the [GitHub Feedback](https://github.com/community/community/discussions/categories/actions-and-packages) page. [Read more about our guidelines here](docs/contribute.md) before contributing. diff --git a/docs/contribute.md b/docs/contribute.md index bb046f4cb9a..70ec61154ce 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -1,6 +1,6 @@ # Contributions -We welcome contributions in the form of issues and pull requests. We view the contributions and the process as the same for github and external contributors. +We welcome contributions in the form of issues and pull requests. We view the contributions and the process as the same for github and external contributors.Please note the runner typically requires changes across the entire system and we aim for issues in the runner to be entirely self contained and fixable here. Therefore, we will primarily handle bug issues opened in this repo and we kindly request you to create all feature and enhancement requests on the [GitHub Feedback](https://github.com/community/community/discussions/categories/actions-and-packages) page. > IMPORTANT: Building your own runner is critical for the dev inner loop process when contributing changes. However, only runners built and distributed by GitHub (releases) are supported in production. Be aware that workflows and orchestrations run service side with the runner being a remote process to run steps. For that reason, the service can pull the runner forward so customizations can be lost. From 1afe238e0d030a3c6f940a9539c7b985e68a0a09 Mon Sep 17 00:00:00 2001 From: Tatyana Kostromskaya <32135588+takost@users.noreply.github.com> Date: Mon, 21 Nov 2022 16:17:47 +0100 Subject: [PATCH 053/145] Support runner upgrade messages (#2231) Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> --- src/Runner.Common/RunnerServer.cs | 6 +-- src/Runner.Listener/MessageListener.cs | 1 + src/Runner.Listener/Runner.cs | 14 +++++- .../Generated/TaskAgentHttpClientBase.cs | 9 +++- .../DTWebApi/WebApi/RunnerRefreshMessage.cs | 49 +++++++++++++++++++ src/Test/L0/Listener/MessageListenerL0.cs | 10 ++-- 6 files changed, 78 insertions(+), 11 deletions(-) create mode 100644 src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs diff --git a/src/Runner.Common/RunnerServer.cs b/src/Runner.Common/RunnerServer.cs index 531375b948f..8ba6cee1bb4 100644 --- a/src/Runner.Common/RunnerServer.cs +++ b/src/Runner.Common/RunnerServer.cs @@ -38,7 +38,7 @@ public interface IRunnerServer : IRunnerService Task CreateAgentSessionAsync(Int32 poolId, TaskAgentSession session, CancellationToken cancellationToken); Task DeleteAgentMessageAsync(Int32 poolId, Int64 messageId, Guid sessionId, CancellationToken cancellationToken); Task DeleteAgentSessionAsync(Int32 poolId, Guid sessionId, CancellationToken cancellationToken); - Task GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, CancellationToken cancellationToken); + Task GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, string runnerVersion, CancellationToken cancellationToken); // job request Task GetAgentRequestAsync(int poolId, long requestId, CancellationToken cancellationToken); @@ -272,10 +272,10 @@ public Task DeleteAgentSessionAsync(Int32 poolId, Guid sessionId, CancellationTo return _messageTaskAgentClient.DeleteAgentSessionAsync(poolId, sessionId, cancellationToken: cancellationToken); } - public Task GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, CancellationToken cancellationToken) + public Task GetAgentMessageAsync(Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, string runnerVersion, CancellationToken cancellationToken) { CheckConnection(RunnerConnectionType.MessageQueue); - return _messageTaskAgentClient.GetMessageAsync(poolId, sessionId, lastMessageId, status, cancellationToken: cancellationToken); + return _messageTaskAgentClient.GetMessageAsync(poolId, sessionId, lastMessageId, status, runnerVersion, cancellationToken: cancellationToken); } //----------------------------------------------------------------- diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 274967ac36f..59eddfbb12d 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -211,6 +211,7 @@ public async Task GetNextMessageAsync(CancellationToken token) _session.SessionId, _lastMessageId, runnerStatus, + BuildConstants.RunnerPackage.Version, _getMessagesTokenSource.Token); // Decrypt the message body if the session is using encryption diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index b6f9946f797..853e0791207 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -430,12 +430,22 @@ private async Task RunAsync(RunnerSettings settings, bool runOnce = false) message = await getNextMessage; //get next message HostContext.WritePerfCounter($"MessageReceived_{message.MessageType}"); - if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) + if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase) || + string.Equals(message.MessageType, RunnerRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) { if (autoUpdateInProgress == false) { autoUpdateInProgress = true; - var runnerUpdateMessage = JsonUtility.FromString(message.Body); + AgentRefreshMessage runnerUpdateMessage = null; + if (string.Equals(message.MessageType, AgentRefreshMessage.MessageType, StringComparison.OrdinalIgnoreCase)) + { + runnerUpdateMessage = JsonUtility.FromString(message.Body); + } + else + { + var brokerRunnerUpdateMessage = JsonUtility.FromString(message.Body); + runnerUpdateMessage = new AgentRefreshMessage(brokerRunnerUpdateMessage.RunnerId, brokerRunnerUpdateMessage.TargetVersion, TimeSpan.FromSeconds(brokerRunnerUpdateMessage.TimeoutInSeconds)); + } #if DEBUG // Can mock the update for testing if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_IS_MOCK_UPDATE"))) diff --git a/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs b/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs index 439fd61c4bf..a423539f301 100644 --- a/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs +++ b/src/Sdk/DTGenerated/Generated/TaskAgentHttpClientBase.cs @@ -450,6 +450,8 @@ public virtual async Task DeleteMessageAsync( /// /// /// + /// + /// /// /// The cancellation token to cancel operation. [EditorBrowsable(EditorBrowsableState.Never)] @@ -458,6 +460,7 @@ public virtual Task GetMessageAsync( Guid sessionId, long? lastMessageId = null, TaskAgentStatus? status = null, + string runnerVersion = null, object userState = null, CancellationToken cancellationToken = default) { @@ -475,12 +478,16 @@ public virtual Task GetMessageAsync( { queryParams.Add("status", status.Value.ToString()); } + if (runnerVersion != null) + { + queryParams.Add("runnerVersion", runnerVersion); + } return SendAsync( httpMethod, locationId, routeValues: routeValues, - version: new ApiResourceVersion(5.1, 1), + version: new ApiResourceVersion(6.0, 1), queryParameters: queryParams, userState: userState, cancellationToken: cancellationToken); diff --git a/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs b/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs new file mode 100644 index 00000000000..93197c260a8 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/RunnerRefreshMessage.cs @@ -0,0 +1,49 @@ +using Newtonsoft.Json; +using System; +using System.Runtime.Serialization; + + +namespace GitHub.DistributedTask.WebApi +{ + [DataContract] + public sealed class RunnerRefreshMessage + { + public static readonly String MessageType = "RunnerRefresh"; + + [JsonConstructor] + internal RunnerRefreshMessage() + { + } + + public RunnerRefreshMessage( + Int32 runnerId, + String targetVersion, + int? timeoutInSeconds = null) + { + this.RunnerId = runnerId; + this.TimeoutInSeconds = timeoutInSeconds ?? TimeSpan.FromMinutes(60).Seconds; + this.TargetVersion = targetVersion; + } + + [DataMember] + public Int32 RunnerId + { + get; + private set; + } + + [DataMember] + public int TimeoutInSeconds + { + get; + private set; + } + + [DataMember] + public String TargetVersion + { + get; + private set; + } + } +} diff --git a/src/Test/L0/Listener/MessageListenerL0.cs b/src/Test/L0/Listener/MessageListenerL0.cs index 8ce1dfe0f97..b2db393bf04 100644 --- a/src/Test/L0/Listener/MessageListenerL0.cs +++ b/src/Test/L0/Listener/MessageListenerL0.cs @@ -192,8 +192,8 @@ public async void GetNextMessage() _runnerServer .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny())) - .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, CancellationToken cancellationToken) => + _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny(), It.IsAny())) + .Returns(async (Int32 poolId, Guid sessionId, Int64? lastMessageId, TaskAgentStatus status, string runnerVersion, CancellationToken cancellationToken) => { await Task.Yield(); return messages.Dequeue(); @@ -208,7 +208,7 @@ public async void GetNextMessage() //Assert _runnerServer .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny()), Times.Exactly(arMessages.Length)); + _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny(), It.IsAny()), Times.Exactly(arMessages.Length)); } } @@ -293,7 +293,7 @@ public async void SkipDeleteSession_WhenGetNextMessageGetTaskAgentAccessTokenExp _runnerServer .Setup(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny())) + _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny(), It.IsAny())) .Throws(new TaskAgentAccessTokenExpiredException("test")); try { @@ -311,7 +311,7 @@ public async void SkipDeleteSession_WhenGetNextMessageGetTaskAgentAccessTokenExp //Assert _runnerServer .Verify(x => x.GetAgentMessageAsync( - _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny()), Times.Once); + _settings.PoolId, expectedSession.SessionId, It.IsAny(), TaskAgentStatus.Online, It.IsAny(), It.IsAny()), Times.Once); _runnerServer .Verify(x => x.DeleteAgentSessionAsync( From bf616ec79768c0c63493ce1b152dca961f07c695 Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Tue, 22 Nov 2022 14:46:11 +0100 Subject: [PATCH 054/145] added replace to allow create latest svc to apply --replace flag (#2273) --- scripts/create-latest-svc.sh | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/scripts/create-latest-svc.sh b/scripts/create-latest-svc.sh index b6eca26e7a8..6d4cbd31091 100755 --- a/scripts/create-latest-svc.sh +++ b/scripts/create-latest-svc.sh @@ -13,7 +13,7 @@ set -e flags_found=false -while getopts 's:g:n:r:u:l:d' opt; do +while getopts 's:g:n:r:u:l:df' opt; do flags_found=true case $opt in @@ -35,6 +35,9 @@ while getopts 's:g:n:r:u:l:d' opt; do l) labels=$OPTARG ;; + f) + replace='true' + ;; d) disableupdate='true' ;; @@ -53,7 +56,8 @@ Usage: -r optional name of the runner group to add the runner to, defaults to the Default group -u optional user svc will run as, defaults to current -l optional list of labels (split by comma) applied on the runner - -d optional allow runner to remain on the current version for one month after the release of a newer version" + -d optional allow runner to remain on the current version for one month after the release of a newer version + -f optional replace any existing runner with the same name" exit 0 ;; esac @@ -174,7 +178,7 @@ fi echo echo "Configuring ${runner_name} @ $runner_url" echo "./config.sh --unattended --url $runner_url --token *** --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup \"$runner_group\"} ${disableupdate:+--disableupdate}" -sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup "$runner_group"} ${disableupdate:+--disableupdate} +sudo -E -u ${svc_user} ./config.sh --unattended --url $runner_url --token $RUNNER_TOKEN ${replace:+--replace} --name $runner_name ${labels:+--labels $labels} ${runner_group:+--runnergroup "$runner_group"} ${disableupdate:+--disableupdate} #--------------------------------------- # Configuring as a service From f3cc2b46e1251822a9dc2f833d48e428f733ca8b Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Tue, 22 Nov 2022 14:47:22 +0100 Subject: [PATCH 055/145] (delete.sh) Loggin repaired and made runner_name optional defaulting to hostname (#1871) * Loggin repaired and made runner_name optional defaulting to hostname * Update scripts/delete.sh Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> Co-authored-by: Josh Soref <2119212+jsoref@users.noreply.github.com> --- scripts/delete.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/delete.sh b/scripts/delete.sh index eeb26f89f76..4be949dc920 100755 --- a/scripts/delete.sh +++ b/scripts/delete.sh @@ -1,4 +1,4 @@ -#/bin/bash +#!/bin/bash set -e @@ -12,7 +12,7 @@ set -e # # Usage: # export RUNNER_CFG_PAT= -# ./delete.sh scope name +# ./delete.sh [] # # scope required repo (:owner/:repo) or org (:organization) # name optional defaults to hostname. name to delete @@ -26,17 +26,17 @@ set -e runner_scope=${1} runner_name=${2} -echo "Deleting runner ${runner_name} @ ${runner_scope}" - -function fatal() +function fatal() { echo "error: $1" >&2 exit 1 } if [ -z "${runner_scope}" ]; then fatal "supply scope as argument 1"; fi -if [ -z "${runner_name}" ]; then fatal "supply name as argument 2"; fi if [ -z "${RUNNER_CFG_PAT}" ]; then fatal "RUNNER_CFG_PAT must be set before calling"; fi +if [ -z "${runner_name}" ]; then runner_name=`hostname`; fi + +echo "Deleting runner ${runner_name} @ ${runner_scope}" which curl || fatal "curl required. Please install in PATH with apt-get, brew, etc" which jq || fatal "jq required. Please install in PATH with apt-get, brew, etc" From 3bf19c7c97ac14836d75d30f0ec83120db81392b Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Tue, 22 Nov 2022 15:05:32 +0100 Subject: [PATCH 056/145] added to lowercase on setting github.action_status (#1944) --- src/Runner.Worker/Handlers/CompositeActionHandler.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index c6d52da147a..b22d9399004 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -263,7 +263,7 @@ private async Task RunStepsAsync(List embeddedSteps, ActionRunStage stage // Set action_status to the success of the current composite action var actionResult = ExecutionContext.Result?.ToActionResult() ?? ActionResult.Success; - step.ExecutionContext.SetGitHubContext("action_status", actionResult.ToString()); + step.ExecutionContext.SetGitHubContext("action_status", actionResult.ToString().ToLowerInvariant()); // Initialize env context Trace.Info("Initialize Env context for embedded step"); From 864030d74b6f7d9f0e228751875c74b74fca239c Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Tue, 22 Nov 2022 15:05:47 +0100 Subject: [PATCH 057/145] Backfill notes and version from 'Release 2.299.1 runner.' (#2277) * Release 2.299.1 runner. * Fix typo in releaseNote Co-authored-by: TingluoHuang --- releaseNote.md | 2 +- src/runnerversion | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index e7af4964fc1..9188ec4ff5f 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -2,7 +2,7 @@ - Displays the error logs in dedicated sub-sections of the Initialize containers section (#2182) - Add generateServiceConfig option for configure command (#2226) - Setting debug using GitHub Action variables (#2234) -- run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233) +- run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233, #2240) ## Bugs diff --git a/src/runnerversion b/src/runnerversion index bf5c74669fd..b438eccdad5 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.299.0 +2.299.1 From 1f885077bbaef7710ad333f4335c0b642cd05577 Mon Sep 17 00:00:00 2001 From: Nikola Jokic <97525037+nikola-jokic@users.noreply.github.com> Date: Mon, 28 Nov 2022 17:33:01 +0100 Subject: [PATCH 058/145] Return exit code when MANUALLY_TRAP_SIG is exported (#2285) --- images/Dockerfile | 1 + src/Misc/layoutroot/run.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index b87755b0d74..11820673de8 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -18,6 +18,7 @@ RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-c FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 ENV RUNNER_ALLOW_RUNASROOT=1 +ENV RUNNER_MANUALLY_TRAP_SIG=1 WORKDIR /actions-runner COPY --from=build /actions-runner . diff --git a/src/Misc/layoutroot/run.sh b/src/Misc/layoutroot/run.sh index 89902eb2e7a..a558793d0df 100755 --- a/src/Misc/layoutroot/run.sh +++ b/src/Misc/layoutroot/run.sh @@ -48,7 +48,7 @@ runWithManualTrap() { trap - INT TERM # wait for last parts to be logged wait $PID - exit 0 + exit $returnCode fi done } From 30bbf6c0c213f6cc6d91bd1cf4a433ec0c069c26 Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Tue, 6 Dec 2022 16:16:00 +0100 Subject: [PATCH 059/145] Listener stdout logging (#2291) * Added env variable to control wether the terminal is silent * Log to stdout if PrintLogToStdout is enabled * Extracted console logging to stdouttracelistener * Remove useless usings * Rewrite TraceListener as superclass * Only print to stdout if env is set * Add comment for Console.Out * Format Listener * Revert var name in terminal * Check env in hostcontext instead of Tracing constructor * Remove superclass & dupe logging code * Log hostType * Readonly '_' prefix 'hostType' * Fix test * Revert Terminal change Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> --- src/Runner.Common/Constants.cs | 1 + src/Runner.Common/HostContext.cs | 11 ++- src/Runner.Common/StdoutTraceListener.cs | 90 ++++++++++++++++++++++++ src/Runner.Common/TraceManager.cs | 10 +-- src/Runner.Common/Tracing.cs | 6 +- src/Test/L0/TestHostContext.cs | 2 +- 6 files changed, 112 insertions(+), 8 deletions(-) create mode 100644 src/Runner.Common/StdoutTraceListener.cs diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index f5d489ea99d..f7e04a56506 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -245,6 +245,7 @@ public static class Agent // Set this env var to "node12" to downgrade the node version for internal functions (e.g hashfiles). This does NOT affect the version of node actions. public static readonly string ForcedInternalNodeVersion = "ACTIONS_RUNNER_FORCED_INTERNAL_NODE_VERSION"; public static readonly string ForcedActionsNodeVersion = "ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION"; + public static readonly string PrintLogToStdout = "ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT"; } public static class System diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 9d5aefdf8c6..9e4346489a5 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -94,6 +94,13 @@ public HostContext(string hostType, string logFile = null) this.SecretMasker.AddValueEncoder(ValueEncoders.PowerShellPreAmpersandEscape); this.SecretMasker.AddValueEncoder(ValueEncoders.PowerShellPostAmpersandEscape); + // Create StdoutTraceListener if ENV is set + StdoutTraceListener stdoutTraceListener = null; + if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.PrintLogToStdout))) + { + stdoutTraceListener = new StdoutTraceListener(hostType); + } + // Create the trace manager. if (string.IsNullOrEmpty(logFile)) { @@ -113,11 +120,11 @@ public HostContext(string hostType, string logFile = null) // this should give us _diag folder under runner root directory string diagLogDirectory = Path.Combine(new DirectoryInfo(Path.GetDirectoryName(Assembly.GetEntryAssembly().Location)).Parent.FullName, Constants.Path.DiagDirectory); - _traceManager = new TraceManager(new HostTraceListener(diagLogDirectory, hostType, logPageSize, logRetentionDays), this.SecretMasker); + _traceManager = new TraceManager(new HostTraceListener(diagLogDirectory, hostType, logPageSize, logRetentionDays), stdoutTraceListener, this.SecretMasker); } else { - _traceManager = new TraceManager(new HostTraceListener(logFile), this.SecretMasker); + _traceManager = new TraceManager(new HostTraceListener(logFile), stdoutTraceListener, this.SecretMasker); } _trace = GetTrace(nameof(HostContext)); diff --git a/src/Runner.Common/StdoutTraceListener.cs b/src/Runner.Common/StdoutTraceListener.cs new file mode 100644 index 00000000000..53ebaf5dba3 --- /dev/null +++ b/src/Runner.Common/StdoutTraceListener.cs @@ -0,0 +1,90 @@ +using GitHub.Runner.Sdk; +using System; +using System.Diagnostics; +using System.Globalization; +using System.IO; + +namespace GitHub.Runner.Common +{ + public sealed class StdoutTraceListener : ConsoleTraceListener + { + private readonly string _hostType; + + public StdoutTraceListener(string hostType) + { + this._hostType = hostType; + } + + // Copied and modified slightly from .Net Core source code. Modification was required to make it compile. + // There must be some TraceFilter extension class that is missing in this source code. + public override void TraceEvent(TraceEventCache eventCache, string source, TraceEventType eventType, int id, string message) + { + if (Filter != null && !Filter.ShouldTrace(eventCache, source, eventType, id, message, null, null, null)) + { + return; + } + + WriteHeader(source, eventType, id); + WriteLine(message); + WriteFooter(eventCache); + } + + internal bool IsEnabled(TraceOptions opts) + { + return (opts & TraceOutputOptions) != 0; + } + + // Altered from the original .Net Core implementation. + private void WriteHeader(string source, TraceEventType eventType, int id) + { + string type = null; + switch (eventType) + { + case TraceEventType.Critical: + type = "CRIT"; + break; + case TraceEventType.Error: + type = "ERR "; + break; + case TraceEventType.Warning: + type = "WARN"; + break; + case TraceEventType.Information: + type = "INFO"; + break; + case TraceEventType.Verbose: + type = "VERB"; + break; + default: + type = eventType.ToString(); + break; + } + + Write(StringUtil.Format("[{0} {1:u} {2} {3}] ", _hostType.ToUpperInvariant(), DateTime.UtcNow, type, source)); + } + + // Copied and modified slightly from .Net Core source code to make it compile. The original code + // accesses a private indentLevel field. In this code it has been modified to use the getter/setter. + private void WriteFooter(TraceEventCache eventCache) + { + if (eventCache == null) + return; + + IndentLevel++; + if (IsEnabled(TraceOptions.ProcessId)) + WriteLine("ProcessId=" + eventCache.ProcessId); + + if (IsEnabled(TraceOptions.ThreadId)) + WriteLine("ThreadId=" + eventCache.ThreadId); + + if (IsEnabled(TraceOptions.DateTime)) + WriteLine("DateTime=" + eventCache.DateTime.ToString("o", CultureInfo.InvariantCulture)); + + if (IsEnabled(TraceOptions.Timestamp)) + WriteLine("Timestamp=" + eventCache.Timestamp); + + IndentLevel--; + } + } +} + diff --git a/src/Runner.Common/TraceManager.cs b/src/Runner.Common/TraceManager.cs index f935aaf8bbe..1321d10b20d 100644 --- a/src/Runner.Common/TraceManager.cs +++ b/src/Runner.Common/TraceManager.cs @@ -16,21 +16,23 @@ public sealed class TraceManager : ITraceManager { private readonly ConcurrentDictionary _sources = new(StringComparer.OrdinalIgnoreCase); private readonly HostTraceListener _hostTraceListener; + private readonly StdoutTraceListener _stdoutTraceListener; private TraceSetting _traceSetting; private ISecretMasker _secretMasker; - public TraceManager(HostTraceListener traceListener, ISecretMasker secretMasker) - : this(traceListener, new TraceSetting(), secretMasker) + public TraceManager(HostTraceListener traceListener, StdoutTraceListener stdoutTraceListener, ISecretMasker secretMasker) + : this(traceListener, stdoutTraceListener, new TraceSetting(), secretMasker) { } - public TraceManager(HostTraceListener traceListener, TraceSetting traceSetting, ISecretMasker secretMasker) + public TraceManager(HostTraceListener traceListener, StdoutTraceListener stdoutTraceListener, TraceSetting traceSetting, ISecretMasker secretMasker) { // Validate and store params. ArgUtil.NotNull(traceListener, nameof(traceListener)); ArgUtil.NotNull(traceSetting, nameof(traceSetting)); ArgUtil.NotNull(secretMasker, nameof(secretMasker)); _hostTraceListener = traceListener; + _stdoutTraceListener = stdoutTraceListener; _traceSetting = traceSetting; _secretMasker = secretMasker; @@ -81,7 +83,7 @@ private Tracing CreateTraceSource(string name) Level = sourceTraceLevel.ToSourceLevels() }; } - return new Tracing(name, _secretMasker, sourceSwitch, _hostTraceListener); + return new Tracing(name, _secretMasker, sourceSwitch, _hostTraceListener, _stdoutTraceListener); } } } diff --git a/src/Runner.Common/Tracing.cs b/src/Runner.Common/Tracing.cs index a0d93b658bc..7a5845db61e 100644 --- a/src/Runner.Common/Tracing.cs +++ b/src/Runner.Common/Tracing.cs @@ -12,7 +12,7 @@ public sealed class Tracing : ITraceWriter, IDisposable private ISecretMasker _secretMasker; private TraceSource _traceSource; - public Tracing(string name, ISecretMasker secretMasker, SourceSwitch sourceSwitch, HostTraceListener traceListener) + public Tracing(string name, ISecretMasker secretMasker, SourceSwitch sourceSwitch, HostTraceListener traceListener, StdoutTraceListener stdoutTraceListener = null) { ArgUtil.NotNull(secretMasker, nameof(secretMasker)); _secretMasker = secretMasker; @@ -27,6 +27,10 @@ public Tracing(string name, ISecretMasker secretMasker, SourceSwitch sourceSwitc } _traceSource.Listeners.Add(traceListener); + if (stdoutTraceListener != null) + { + _traceSource.Listeners.Add(stdoutTraceListener); + } } public void Info(string message) diff --git a/src/Test/L0/TestHostContext.cs b/src/Test/L0/TestHostContext.cs index a88b9170ea1..a3e484b14c8 100644 --- a/src/Test/L0/TestHostContext.cs +++ b/src/Test/L0/TestHostContext.cs @@ -59,7 +59,7 @@ public TestHostContext(object testClass, [CallerMemberName] string testName = "" _secretMasker = new SecretMasker(); _secretMasker.AddValueEncoder(ValueEncoders.JsonStringEscape); _secretMasker.AddValueEncoder(ValueEncoders.UriDataEscape); - _traceManager = new TraceManager(traceListener, _secretMasker); + _traceManager = new TraceManager(traceListener, null, _secretMasker); _trace = GetTrace(nameof(TestHostContext)); // inject a terminal in silent mode so all console output From 9de53d297514cecc6aad7d627aaf06a24415746e Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Thu, 8 Dec 2022 22:23:52 +0100 Subject: [PATCH 060/145] Made worker logs available to stdout (#2307) * Made worker logs available to stdout * Log Worker Standard out line by line Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> --- src/Runner.Common/Terminal.cs | 9 ++++++--- src/Runner.Listener/JobDispatcher.cs | 13 +++++++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/Runner.Common/Terminal.cs b/src/Runner.Common/Terminal.cs index bade576a095..75489bc1853 100644 --- a/src/Runner.Common/Terminal.cs +++ b/src/Runner.Common/Terminal.cs @@ -18,7 +18,7 @@ public interface ITerminal : IRunnerService, IDisposable string ReadSecret(); void Write(string message, ConsoleColor? colorCode = null); void WriteLine(); - void WriteLine(string line, ConsoleColor? colorCode = null); + void WriteLine(string line, ConsoleColor? colorCode = null, bool skipTracing = false); void WriteError(Exception ex); void WriteError(string line); void WriteSection(string message); @@ -116,9 +116,12 @@ public void WriteLine() // Do not add a format string overload. Terminal messages are user facing and therefore // should be localized. Use the Loc method in the StringUtil class. - public void WriteLine(string line, ConsoleColor? colorCode = null) + public void WriteLine(string line, ConsoleColor? colorCode = null, bool skipTracing = false) { - Trace.Info($"WRITE LINE: {line}"); + if (!skipTracing) + { + Trace.Info($"WRITE LINE: {line}"); + } if (!Silent) { if (colorCode != null) diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index 1e5306bd0af..aaa83b80c78 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -400,6 +400,7 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc Task workerProcessTask = null; object _outputLock = new(); List workerOutput = new(); + bool printToStdout = StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable(Constants.Variables.Agent.PrintLogToStdout)); using (var processChannel = HostContext.CreateService()) using (var processInvoker = HostContext.CreateService()) { @@ -421,7 +422,15 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc { lock (_outputLock) { - workerOutput.Add(stdout.Data); + if (!stdout.Data.StartsWith("[WORKER")) + { + workerOutput.Add(stdout.Data); + } + + if (printToStdout) + { + term.WriteLine(stdout.Data, skipTracing: true); + } } } }; @@ -658,7 +667,7 @@ await processChannel.SendAsync( finally { Busy = false; - + if (JobStatus != null) { JobStatus(this, new JobStatusEventArgs(TaskAgentStatus.Online)); From 5f507c9557d2c4e27b0281aea0f8de4f8d3b8ea0 Mon Sep 17 00:00:00 2001 From: Brittany Ellich Date: Wed, 14 Dec 2022 00:28:33 -0800 Subject: [PATCH 061/145] Use results for uploading step summaries (#2301) * Use results service for uploading step summaries * Use results summary over generic results naming convention * Apply suggestions from code review Co-authored-by: Tingluo Huang * Addressing feedback * Fix merge issue * Remove empty line * Update Results json objects to use snake case * Adding the reference Co-authored-by: Yang Cao Co-authored-by: Tingluo Huang --- src/Runner.Common/Constants.cs | 1 + src/Runner.Common/JobServer.cs | 21 +++ src/Runner.Common/JobServerQueue.cs | 148 ++++++++++++++++++++- src/Runner.Worker/ExecutionContext.cs | 16 ++- src/Runner.Worker/FileCommandManager.cs | 37 ++++-- src/Sdk/DTWebApi/WebApi/TaskAttachment.cs | 1 - src/Sdk/WebApi/WebApi/Contracts.cs | 60 +++++++++ src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 138 +++++++++++++++++++ 8 files changed, 405 insertions(+), 17 deletions(-) create mode 100644 src/Sdk/WebApi/WebApi/Contracts.cs create mode 100644 src/Sdk/WebApi/WebApi/ResultsHttpClient.cs diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index f7e04a56506..2f5b69d8467 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -164,6 +164,7 @@ public static class Features public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/"; public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`."; public static readonly string UnsupportedSummarySize = "$GITHUB_STEP_SUMMARY upload aborted, supports content up to a size of {0}k, got {1}k. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary"; + public static readonly string SummaryUploadError = "$GITHUB_STEP_SUMMARY upload aborted, an error occurred when uploading the summary. For more information see: https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-markdown-summary"; public static readonly string Node12DetectedAfterEndOfLife = "Node.js 12 actions are deprecated. Please update the following actions to use Node.js 16: {0}. For more information see: https://github.blog/changelog/2022-09-22-github-actions-all-actions-will-begin-running-on-node16-instead-of-node12/."; } diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 4a37311ca03..8a6c4a6a76e 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -13,6 +13,8 @@ using GitHub.Services.Common; using GitHub.Services.WebApi; using GitHub.Services.WebApi.Utilities.Internal; +using GitHub.Services.Results.Client; +using GitHub.Services.OAuth; namespace GitHub.Runner.Common { @@ -22,11 +24,13 @@ public interface IJobServer : IRunnerService, IAsyncDisposable Task ConnectAsync(VssConnection jobConnection); void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint); + void InitializeResultsClient(Uri uri, string token); // logging and console Task AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); + Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); @@ -40,6 +44,7 @@ public sealed class JobServer : RunnerService, IJobServer private bool _hasConnection; private VssConnection _connection; private TaskHttpClient _taskClient; + private ResultsHttpClient _resultsClient; private ClientWebSocket _websocketClient; private ServiceEndpoint _serviceEndpoint; @@ -143,6 +148,12 @@ public void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint) InitializeWebsocketClient(TimeSpan.Zero); } + public void InitializeResultsClient(Uri uri, string token) + { + var httpMessageHandler = HostContext.CreateHttpClientHandler(); + this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true); + } + public ValueTask DisposeAsync() { CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None); @@ -305,6 +316,16 @@ public Task CreateAttachmentAsync(Guid scopeIdentifier, string h return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken); } + public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadStepSummaryAsync(planId, jobId, stepId, file, cancellationToken: cancellationToken); + } + throw new InvalidOperationException("Results client is not initialized."); + } + + public Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) { CheckConnection(); diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index d84614c15c3..fb97ec73220 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -20,6 +20,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter void Start(Pipelines.AgentJobRequestMessage jobRequest); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); + void QueueSummaryUpload(Guid timelineId, Guid timelineRecordId, string stepId, string name, string path, bool deleteSource); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); } @@ -30,6 +31,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000); + private static readonly TimeSpan _delayForSummaryUploadDequeue = TimeSpan.FromMilliseconds(1000); // Job message information private Guid _scopeIdentifier; @@ -44,6 +46,8 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // queue for file upload (log file or attachment) private readonly ConcurrentQueue _fileUploadQueue = new(); + private readonly ConcurrentQueue _summaryFileUploadQueue = new(); + // queue for timeline or timeline record update (one queue per timeline) private readonly ConcurrentDictionary> _timelineUpdateQueue = new(); @@ -56,6 +60,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // Task for each queue's dequeue process private Task _webConsoleLineDequeueTask; private Task _fileUploadDequeueTask; + private Task _summaryUploadDequeueTask; private Task _timelineUpdateDequeueTask; // common @@ -93,6 +98,20 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) _jobServer.InitializeWebsocketClient(serviceEndPoint); + // This code is usually wrapped by an instance of IExecutionContext which isn't available here. + jobRequest.Variables.TryGetValue("system.github.results_endpoint", out VariableValue resultsEndpointVariable); + var resultsReceiverEndpoint = resultsEndpointVariable?.Value; + + if (serviceEndPoint?.Authorization != null && + serviceEndPoint.Authorization.Parameters.TryGetValue("AccessToken", out var accessToken) && + !string.IsNullOrEmpty(accessToken) && + !string.IsNullOrEmpty(resultsReceiverEndpoint)) + { + Trace.Info("Initializing results client"); + _jobServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken); + } + + if (_queueInProcess) { Trace.Info("No-opt, all queue process tasks are running."); @@ -120,10 +139,13 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) Trace.Info("Start process file upload queue."); _fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); + Trace.Info("Start results file upload queue."); + _summaryUploadDequeueTask = ProcessSummaryUploadQueueAsync(); + Trace.Info("Start process timeline update queue."); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); - _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask }; + _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _summaryUploadDequeueTask }; _queueInProcess = true; } @@ -154,6 +176,10 @@ public async Task ShutdownAsync() await ProcessFilesUploadQueueAsync(runOnce: true); Trace.Info("File upload queue drained."); + Trace.Verbose("Draining results summary upload queue."); + await ProcessSummaryUploadQueueAsync(runOnce: true); + Trace.Info("Results summary upload queue drained."); + // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown // if there is any timeline records that failed to update contains output variabls. Trace.Verbose("Draining timeline update queue."); @@ -204,6 +230,28 @@ public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, _fileUploadQueue.Enqueue(newFile); } + public void QueueSummaryUpload(Guid timelineId, Guid timelineRecordId, string stepId, string name, string path, bool deleteSource) + { + ArgUtil.NotEmpty(timelineId, nameof(timelineId)); + ArgUtil.NotEmpty(timelineRecordId, nameof(timelineRecordId)); + + // all parameter not null, file path exist. + var newFile = new SummaryUploadFileInfo() + { + TimelineId = timelineId, + TimelineRecordId = timelineRecordId, + Name = name, + Path = path, + PlanId = _planId.ToString(), + JobId = _jobTimelineRecordId.ToString(), + StepId = stepId, + DeleteSource = deleteSource + }; + + Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to record {1}", newFile.Path, timelineRecordId); + _summaryFileUploadQueue.Enqueue(newFile); + } + public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord) { ArgUtil.NotEmpty(timelineId, nameof(timelineId)); @@ -299,7 +347,7 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) { try { - // Give at most 60s for each request. + // Give at most 60s for each request. using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(60))) { await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token); @@ -394,6 +442,60 @@ private async Task ProcessFilesUploadQueueAsync(bool runOnce = false) } } + private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) + { + Trace.Info("Starting results-based upload queue..."); + + while (!_jobCompletionSource.Task.IsCompleted || runOnce) + { + List filesToUpload = new(); + SummaryUploadFileInfo dequeueFile; + while (_summaryFileUploadQueue.TryDequeue(out dequeueFile)) + { + filesToUpload.Add(dequeueFile); + // process at most 10 file upload. + if (!runOnce && filesToUpload.Count > 10) + { + break; + } + } + + if (filesToUpload.Count > 0) + { + if (runOnce) + { + Trace.Info($"Uploading {filesToUpload.Count} summary files in one shot through results service."); + } + + int errorCount = 0; + foreach (var file in filesToUpload) + { + try + { + await UploadSummaryFile(file); + } + catch (Exception ex) + { + Trace.Info("Catch exception during summary file upload to results, keep going since the process is best effort."); + Trace.Error(ex); + errorCount++; + } + } + + Trace.Info("Tried to upload {0} summary files to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); + } + + if (runOnce) + { + break; + } + else + { + await Task.Delay(_delayForSummaryUploadDequeue); + } + } + } + private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) @@ -665,6 +767,35 @@ private async Task UploadFile(UploadFileInfo file) } } } + + private async Task UploadSummaryFile(SummaryUploadFileInfo file) + { + bool uploadSucceed = false; + try + { + // Upload the step summary + Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); + var cancellationTokenSource = new CancellationTokenSource(); + await _jobServer.CreateStepSymmaryAsync(file.PlanId, file.JobId, file.StepId, file.Path, cancellationTokenSource.Token); + + uploadSucceed = true; + } + finally + { + if (uploadSucceed && file.DeleteSource) + { + try + { + File.Delete(file.Path); + } + catch (Exception ex) + { + Trace.Info("Catch exception during delete success results uploaded summary file."); + Trace.Error(ex); + } + } + } + } } internal class PendingTimelineRecord @@ -683,6 +814,19 @@ internal class UploadFileInfo public bool DeleteSource { get; set; } } + internal class SummaryUploadFileInfo + { + public Guid TimelineId { get; set; } + public Guid TimelineRecordId { get; set; } + public string Name { get; set; } + public string Path { get; set; } + public string PlanId { get; set; } + public string JobId { get; set; } + public string StepId { get; set; } + public bool DeleteSource { get; set; } + } + + internal class ConsoleLineInfo { diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index f4aa14d01d1..c8e6ca41817 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -80,6 +80,7 @@ public interface IExecutionContext : IRunnerService // logging long Write(string tag, string message); void QueueAttachFile(string type, string name, string filePath); + void QueueSummaryFile(string name, string filePath, string stepId); // timeline record update methods void Start(string currentOperation = null); @@ -846,6 +847,19 @@ public void QueueAttachFile(string type, string name, string filePath) _jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false); } + public void QueueSummaryFile(string name, string filePath, string stepId) + { + ArgUtil.NotNullOrEmpty(name, nameof(name)); + ArgUtil.NotNullOrEmpty(filePath, nameof(filePath)); + + if (!File.Exists(filePath)) + { + throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist."); + } + + _jobServerQueue.QueueSummaryUpload(_mainTimelineId, _record.Id, stepId, name, filePath, deleteSource: false); + } + // Add OnMatcherChanged public void Add(OnMatcherChanged handler) { @@ -1085,7 +1099,7 @@ private static void SetDebugUsingVars(IDictionary variabl { if (contextData != null && contextData.TryGetValue(PipelineTemplateConstants.Vars, out var varsPipelineContextData) && - varsPipelineContextData != null && + varsPipelineContextData != null && varsPipelineContextData is DictionaryContextData varsContextData) { // Set debug variables only when StepDebug/RunnerDebug variables are not present. diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index 66362f6812a..600db078a59 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -182,14 +182,6 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container return; } - if (fileSize > AttachmentSizeLimit) - { - context.Error(String.Format(Constants.Runner.UnsupportedSummarySize, AttachmentSizeLimit / 1024, fileSize / 1024)); - Trace.Info($"Step Summary file ({filePath}) is too large ({fileSize} bytes); skipping attachment upload"); - - return; - } - Trace.Verbose($"Step Summary file exists: {filePath} and has a file size of {fileSize} bytes"); var scrubbedFilePath = filePath + "-scrubbed"; @@ -204,13 +196,32 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container } } - var attachmentName = !context.IsEmbedded - ? context.Id.ToString() + var attachmentName = !context.IsEmbedded + ? context.Id.ToString() : context.EmbeddedId.ToString(); - Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); - // Attachments must be added to the parent context (job), not the current context (step) - context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); + context.Global.Variables.TryGetValue("system.github.results_endpoint", out string resultsReceiverEndpoint); + if (resultsReceiverEndpoint != null) + { + Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})"); + var stepId = context.Id.ToString(); + // Attachments must be added to the parent context (job), not the current context (step) + context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId); + } + else + { + if (fileSize > AttachmentSizeLimit) + { + context.Error(String.Format(Constants.Runner.UnsupportedSummarySize, AttachmentSizeLimit / 1024, fileSize / 1024)); + Trace.Info($"Step Summary file ({filePath}) is too large ({fileSize} bytes); skipping attachment upload"); + + return; + } + + Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); + // Attachments must be added to the parent context (job), not the current context (step) + context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); + } } catch (Exception e) { diff --git a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs index fac99a417e7..17027d1261a 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs @@ -27,7 +27,6 @@ public TaskAttachment(String type, String name) this.Type = type; this.Name = name; } - [DataMember] public String Type diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs new file mode 100644 index 00000000000..d240cc1e230 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -0,0 +1,60 @@ +using System.Runtime.Serialization; +using Newtonsoft.Json; +using Newtonsoft.Json.Serialization; + +namespace GitHub.Services.Results.Contracts +{ + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepSummaryURLRequest + { + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string WorkflowRunBackendId; + [DataMember] + public string StepBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepSummaryURLResponse + { + [DataMember] + public string SummaryUrl; + [DataMember] + public long SoftSizeLimit; + [DataMember] + public string BlobStorageType; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class StepSummaryMetadataCreate + { + [DataMember] + public string StepBackendId; + [DataMember] + public string WorkflowRunBackendId; + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public long Size; + [DataMember] + public string UploadedAt; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class CreateStepSummaryMetadataResponse + { + [DataMember] + public bool Ok; + } + + public static class BlobStorageTypes + { + public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE"; + public static readonly string Unspecified = "BLOB_STORAGE_TYPE_UNSPECIFIED"; + } +} diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs new file mode 100644 index 00000000000..674abb20db8 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -0,0 +1,138 @@ +using System; +using System.IO; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.Results.Contracts; +using System.Net.Http.Formatting; +using Sdk.WebApi.WebApi; + +namespace GitHub.Services.Results.Client +{ + public class ResultsHttpClient : RawHttpClientBase + { + public ResultsHttpClient( + Uri baseUrl, + HttpMessageHandler pipeline, + string token, + bool disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + m_token = token; + m_resultsServiceUrl = baseUrl; + m_formatter = new JsonMediaTypeFormatter(); + } + + public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, string stepId, CancellationToken cancellationToken) + { + var request = new GetSignedStepSummaryURLRequest() + { + WorkflowJobRunBackendId= jobId, + WorkflowRunBackendId= planId, + StepBackendId= stepId + }; + + var stepSummaryUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepSummarySignedBlobURL"); + + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepSummaryUploadRequest)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + return await ReadJsonContentAsync(response, cancellationToken); + } + } + } + } + + private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, string stepId, long size, CancellationToken cancellationToken) + { + var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); + var request = new StepSummaryMetadataCreate() + { + WorkflowJobRunBackendId= jobId, + WorkflowRunBackendId= planId, + StepBackendId = stepId, + Size = size, + UploadedAt = timestamp + }; + + var stepSummaryUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepSummaryMetadata"); + + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepSummaryUploadCompleteRequest)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); + if (!jsonResponse.Ok) + { + throw new Exception($"Failed to mark step summary upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, size: {size}, timestamp: {timestamp}"); + } + } + } + } + } + + private async Task UploadFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + { + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("x-ms-blob-type", "BlockBlob"); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } + return response; + } + } + + // Handle file upload for step summary + public async Task UploadStepSummaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) + { + // Get the upload url + var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken); + + // Do we want to throw an exception here or should we just be uploading/truncating the data + var fileSize = new FileInfo(file).Length; + if (fileSize > uploadUrlResponse.SoftSizeLimit) + { + throw new Exception($"File size is larger than the upload url allows, file size: {fileSize}, upload url size: {uploadUrlResponse.SoftSizeLimit}"); + } + + // Upload the file + using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) + { + var response = await UploadFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); + } + + // Send step summary upload complete message + await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); + } + + private MediaTypeFormatter m_formatter; + private Uri m_resultsServiceUrl; + private string m_token; + } +} From f2b463ffa3d57f2547a07fc6a58168649b6618cb Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Wed, 14 Dec 2022 10:33:02 +0200 Subject: [PATCH 062/145] prepare release notes for 2.300.0 (#2312) * Update runner version * Update releaseNote.md * Update releaseNote.md * Update releaseNote.md Co-authored-by: JoannaaKL * Update releaseNote.md * Update releaseNote.md Co-authored-by: JoannaaKL Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> --- releaseNote.md | 19 +++++++++---------- src/runnerversion | 2 +- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 9188ec4ff5f..54fe58ee941 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,18 +1,17 @@ ## Features -- Displays the error logs in dedicated sub-sections of the Initialize containers section (#2182) -- Add generateServiceConfig option for configure command (#2226) -- Setting debug using GitHub Action variables (#2234) -- run.sh installs SIGINT and SIGTERM traps to gracefully stop runner (#2233, #2240) - +- Expose github.actor_id, github.workflow_ref & github.workflow_sha as environment variable (#2249) +- Added worker and listener logs to stdout (#2291, #2307) ## Bugs -- Use Global.Variables instead of JobContext and include action path/ref in the message. (#2214) +- Made github.action_status output lowercase to be consistent with job.status' output (#1944) ## Misc -- Allow '--disableupdate' in create-latest-svc.sh (#2201) -- Fix markup for support link (#2114) -- Add runner devcontainer (#2187) -- Setup linter for Runner (#2211, #2213, #2216) +- Added small size runner image for ARC (#2250) +- Small change to Node.js 12 deprecation message (#2262) +- Added the option to use the --replace argument to the create-latest-svc.sh (#2273) +- Made runner_name optional defaulting to hostname in delete.sh script (#1871) +- Return exit code when MANUALLY_TRAP_SIG is exported (#2285) +- Use results for uploading step summaries (#2301) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index b438eccdad5..82bd2418e57 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.299.1 +2.300.0 From 6e475d8bb20cdbaee036fcfe43ec4d5da5b094c3 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Thu, 15 Dec 2022 15:44:07 +0100 Subject: [PATCH 063/145] Update Dockerfile (#2315) --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index 11820673de8..83ea6b51140 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -2,7 +2,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build ARG RUNNER_VERSION ARG RUNNER_ARCH="x64" -ARG RUNNER_CONTAINER_HOOKS_VERSION=0.1.3 +ARG RUNNER_CONTAINER_HOOKS_VERSION=0.2.0 RUN apt update -y && apt install curl unzip -y From f920de49709284e5261a9036b1e61cc87228b0a3 Mon Sep 17 00:00:00 2001 From: Ava Stancu Date: Thu, 15 Dec 2022 17:17:41 +0200 Subject: [PATCH 064/145] Updated contact links for feature requests (#2314) Users need to use the Github Community feedback page for all feature/enhancement requests. --- .github/ISSUE_TEMPLATE/config.yml | 3 ++ .github/ISSUE_TEMPLATE/enhancement_request.md | 32 ------------------- 2 files changed, 3 insertions(+), 32 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE/enhancement_request.md diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 6dff7198bdf..0822f6a1098 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1,5 +1,8 @@ blank_issues_enabled: false contact_links: + - name: 🛑 Request a feature in the runner application + url: https://github.com/orgs/community/discussions/categories/actions-and-packages + about: If you have feature requests for GitHub Actions, please use the Actions and Packages section on the Github Product Feedback page. - name: ✅ Support for GitHub Actions url: https://github.community/c/code-to-cloud/52 about: If you have questions about GitHub Actions or need support writing workflows, please ask in the GitHub Community Support forum. diff --git a/.github/ISSUE_TEMPLATE/enhancement_request.md b/.github/ISSUE_TEMPLATE/enhancement_request.md deleted file mode 100644 index bc0df757a75..00000000000 --- a/.github/ISSUE_TEMPLATE/enhancement_request.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: 🛑 Request a feature in the runner application -about: If you have feature requests for GitHub Actions, please use the "feedback and suggestions for GitHub Actions" link below. -title: '' -labels: enhancement -assignees: '' - ---- - - - -**Describe the enhancement** -A clear and concise description of what the features or enhancement you need. - -**Code Snippet** -If applicable, add a code snippet. - -**Additional information** -Add any other context about the feature here. - -NOTE: if the feature request has been agreed upon then the assignee will create an ADR. See docs/adrs/README.md From 137dfb2d90c5fa34ffe1afc0e5a3c3ef9b5a9834 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Fri, 16 Dec 2022 15:40:49 +0100 Subject: [PATCH 065/145] Initialize container manager based on whether the ContainerHooksPath is set (#2317) * Added tests around checking if correct manager's Initialize method has been called * repaired missing initialization on container action handler --- .../ContainerOperationProvider.cs | 10 ++++- .../Handlers/ContainerActionHandler.cs | 13 +++++- .../L0/Worker/ContainerOperationProviderL0.cs | 41 ++++++++++++++++++- 3 files changed, 59 insertions(+), 5 deletions(-) diff --git a/src/Runner.Worker/ContainerOperationProvider.cs b/src/Runner.Worker/ContainerOperationProvider.cs index 79cc8cf7643..6e5b120479a 100644 --- a/src/Runner.Worker/ContainerOperationProvider.cs +++ b/src/Runner.Worker/ContainerOperationProvider.cs @@ -33,8 +33,14 @@ public class ContainerOperationProvider : RunnerService, IContainerOperationProv public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); - _dockerManager = HostContext.GetService(); - _containerHookManager = HostContext.GetService(); + if (string.IsNullOrEmpty(Environment.GetEnvironmentVariable(Constants.Hooks.ContainerHooksPath))) + { + _dockerManager = HostContext.GetService(); + } + else + { + _containerHookManager = HostContext.GetService(); + } } public async Task StartContainersAsync(IExecutionContext executionContext, object data) diff --git a/src/Runner.Worker/Handlers/ContainerActionHandler.cs b/src/Runner.Worker/Handlers/ContainerActionHandler.cs index 1fe205f4025..c1d57de116d 100644 --- a/src/Runner.Worker/Handlers/ContainerActionHandler.cs +++ b/src/Runner.Worker/Handlers/ContainerActionHandler.cs @@ -38,8 +38,17 @@ public async Task RunAsync(ActionRunStage stage) // Update the env dictionary. AddInputsToEnvironment(); - var dockerManager = HostContext.GetService(); - var containerHookManager = HostContext.GetService(); + IDockerCommandManager dockerManager = null; + IContainerHookManager containerHookManager = null; + if (FeatureManager.IsContainerHooksEnabled(ExecutionContext.Global.Variables)) + { + containerHookManager = HostContext.GetService(); + } + else + { + dockerManager = HostContext.GetService(); + } + string dockerFile = null; // container image haven't built/pull diff --git a/src/Test/L0/Worker/ContainerOperationProviderL0.cs b/src/Test/L0/Worker/ContainerOperationProviderL0.cs index 1cec0d4daa5..4971baef8b5 100644 --- a/src/Test/L0/Worker/ContainerOperationProviderL0.cs +++ b/src/Test/L0/Worker/ContainerOperationProviderL0.cs @@ -99,6 +99,46 @@ public async void RunServiceContainersHealthcheck_healthyServiceContainerWithout } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void InitializeWithCorrectManager() + { + containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); + _hc = new TestHostContext(this, "Test"); + _ec = new Mock(); + serverQueue = new Mock(); + pagingLogger = new Mock(); + + containerOperationProvider = new ContainerOperationProvider(); + + _hc.SetSingleton(serverQueue.Object); + _hc.SetSingleton(pagingLogger.Object); + + + _ec.Setup(x => x.Global).Returns(new GlobalContext()); + + Environment.SetEnvironmentVariable(Constants.Hooks.ContainerHooksPath, "/tmp/k8s/index.js"); + _dockerManager = new Mock(); + _dockerManager.Setup(x => x.Initialize(_hc)).Throws(new Exception("Docker manager's Initialize should not be called")); + + _containerHookManager = new Mock(); + _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(_containerHookManager.Object); + + containerOperationProvider.Initialize(_hc); + + Environment.SetEnvironmentVariable(Constants.Hooks.ContainerHooksPath, null); + _containerHookManager = new Mock(); + _containerHookManager.Setup(x => x.Initialize(_hc)).Throws(new Exception("Container hook manager's Initialize should not be called")); + + _dockerManager = new Mock(); + _hc.SetSingleton(_dockerManager.Object); + _hc.SetSingleton(_containerHookManager.Object); + + containerOperationProvider.Initialize(_hc); + } + private void Setup([CallerMemberName] string testName = "") { containers.Add(new ContainerInfo() { ContainerImage = "ubuntu:16.04" }); @@ -111,7 +151,6 @@ private void Setup([CallerMemberName] string testName = "") _containerHookManager = new Mock(); containerOperationProvider = new ContainerOperationProvider(); - _hc.SetSingleton(_dockerManager.Object); _hc.SetSingleton(serverQueue.Object); _hc.SetSingleton(pagingLogger.Object); From 8129966c742fe7c60d773be08a556e226c3cae3b Mon Sep 17 00:00:00 2001 From: Stefan Ruvceski <96768603+ruvceskistefan@users.noreply.github.com> Date: Mon, 19 Dec 2022 15:01:53 +0100 Subject: [PATCH 066/145] set env in ProcessInvoker sanitized (#2280) * set env in ProcessInvoker sanitized --- src/Runner.Sdk/ProcessInvoker.cs | 10 +++++ src/Test/L0/ProcessInvokerL0.cs | 69 ++++++++++++++++++++++++++++++++ 2 files changed, 79 insertions(+) diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index e303863b59b..e46b2a057aa 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -264,7 +264,17 @@ public async Task ExecuteAsync( { foreach (KeyValuePair kvp in environment) { +#if OS_WINDOWS + string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : kvp.Key.Split('\0')[0]; + string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : kvp.Value.Split('\0')[0]; + if(!String.IsNullOrWhiteSpace(tempKey)) + { + _proc.StartInfo.Environment[tempKey] = tempValue; + } +#else _proc.StartInfo.Environment[kvp.Key] = kvp.Value; + +#endif } } diff --git a/src/Test/L0/ProcessInvokerL0.cs b/src/Test/L0/ProcessInvokerL0.cs index d3d1b54866a..6d99932e3e2 100644 --- a/src/Test/L0/ProcessInvokerL0.cs +++ b/src/Test/L0/ProcessInvokerL0.cs @@ -128,7 +128,76 @@ public async Task SetCIEnv() } } } +#if OS_WINDOWS + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public async Task SetTestEnvWithNullInKey() + { + using (TestHostContext hc = new(this)) + { + Tracing trace = hc.GetTrace(); + + Int32 exitCode = -1; + var processInvoker = new ProcessInvokerWrapper(); + processInvoker.Initialize(hc); + var stdout = new List(); + var stderr = new List(); + processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stdout.Add(e.Data); + }; + processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stderr.Add(e.Data); + }; + + exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary() { { "TEST\0second", "first" } }, CancellationToken.None); + + + trace.Info("Exit Code: {0}", exitCode); + Assert.Equal(0, exitCode); + Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x))); + + } + } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public async Task SetTestEnvWithNullInValue() + { + using (TestHostContext hc = new(this)) + { + Tracing trace = hc.GetTrace(); + + Int32 exitCode = -1; + var processInvoker = new ProcessInvokerWrapper(); + processInvoker.Initialize(hc); + var stdout = new List(); + var stderr = new List(); + processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stdout.Add(e.Data); + }; + processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stderr.Add(e.Data); + }; + + exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary() { { "TEST", "first\0second" } }, CancellationToken.None); + + trace.Info("Exit Code: {0}", exitCode); + Assert.Equal(0, exitCode); + Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x))); + + } + } +#endif [Fact] [Trait("Level", "L0")] [Trait("Category", "Common")] From 71a27d97a96bc18034502041a34c0182904773d4 Mon Sep 17 00:00:00 2001 From: Bethany Date: Mon, 19 Dec 2022 12:56:28 -0500 Subject: [PATCH 067/145] Re-add file size check prior to reading file (#2321) (#2330) * Re-add file size check prior to reading file * Remove redundant file size check --- src/Runner.Worker/FileCommandManager.cs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index 600db078a59..a4eb2e676ea 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -182,6 +182,14 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container return; } + if (fileSize > AttachmentSizeLimit) + { + context.Error(String.Format(Constants.Runner.UnsupportedSummarySize, AttachmentSizeLimit / 1024, fileSize / 1024)); + Trace.Info($"Step Summary file ({filePath}) is too large ({fileSize} bytes); skipping attachment upload"); + + return; + } + Trace.Verbose($"Step Summary file exists: {filePath} and has a file size of {fileSize} bytes"); var scrubbedFilePath = filePath + "-scrubbed"; @@ -210,14 +218,6 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container } else { - if (fileSize > AttachmentSizeLimit) - { - context.Error(String.Format(Constants.Runner.UnsupportedSummarySize, AttachmentSizeLimit / 1024, fileSize / 1024)); - Trace.Info($"Step Summary file ({filePath}) is too large ({fileSize} bytes); skipping attachment upload"); - - return; - } - Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); // Attachments must be added to the parent context (job), not the current context (step) context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); From dbf660527e04cfa95fa2b25da6d49b77601b9b37 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Mon, 19 Dec 2022 19:08:36 +0100 Subject: [PATCH 068/145] Release notes for 2.300.1 (#2326) * Update runnerversion * Update releaseNote.md --- releaseNote.md | 2 +- src/runnerversion | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 54fe58ee941..60012cc9104 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -11,7 +11,7 @@ - Added the option to use the --replace argument to the create-latest-svc.sh (#2273) - Made runner_name optional defaulting to hostname in delete.sh script (#1871) - Return exit code when MANUALLY_TRAP_SIG is exported (#2285) -- Use results for uploading step summaries (#2301) +- Use results for uploading step summaries (#2301) with limited size (#2321) ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. diff --git a/src/runnerversion b/src/runnerversion index 82bd2418e57..d0985ba3c8f 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.300.0 +2.300.1 From b30f14c3e52d412ed8ef70902c91022bd9fbeeb8 Mon Sep 17 00:00:00 2001 From: TingluoHuang Date: Mon, 19 Dec 2022 14:22:51 -0500 Subject: [PATCH 069/145] Bump runner version to 2.300.2 to match released version. --- src/runnerversion | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/runnerversion b/src/runnerversion index d0985ba3c8f..127ceb9c83d 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.300.1 +2.300.2 From a683911e672438746de9dac0c19a7b478711743d Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 19 Dec 2022 14:46:29 -0500 Subject: [PATCH 070/145] Log GitHub RequestId for better traceability. (#2332) --- .../Configuration/ConfigurationManager.cs | 43 ++++++++++++------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 25217f9bee4..392eb0e3c3b 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -1,10 +1,3 @@ -using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common; -using GitHub.Runner.Common.Util; -using GitHub.Runner.Sdk; -using GitHub.Services.Common; -using GitHub.Services.Common.Internal; -using GitHub.Services.OAuth; using System; using System.Collections.Generic; using System.Linq; @@ -14,6 +7,13 @@ using System.Security.Cryptography; using System.Text; using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; +using GitHub.Services.Common.Internal; +using GitHub.Services.OAuth; namespace GitHub.Runner.Listener.Configuration { @@ -636,7 +636,7 @@ private async Task GetJITRunnerTokenAsync(string gith } int retryCount = 0; - while(retryCount < 3) + while (retryCount < 3) { using (var httpClientHandler = HostContext.CreateHttpClientHandler()) using (var httpClient = new HttpClient(httpClientHandler)) @@ -646,28 +646,29 @@ private async Task GetJITRunnerTokenAsync(string gith httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("basic", base64EncodingToken); httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); httpClient.DefaultRequestHeaders.Accept.ParseAdd("application/vnd.github.v3+json"); - + var responseStatus = System.Net.HttpStatusCode.OK; try { var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty)); responseStatus = response.StatusCode; + var githubRequestId = GetGitHubRequestId(response.Headers); if (response.IsSuccessStatusCode) { - Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'"); + Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' ({githubRequestId})"); var jsonResponse = await response.Content.ReadAsStringAsync(); return StringUtil.ConvertFromJson(jsonResponse); } else { - _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'"); + _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' (Request Id: {githubRequestId})"); var errorResponse = await response.Content.ReadAsStringAsync(); _term.WriteError(errorResponse); response.EnsureSuccessStatusCode(); } } - catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound) + catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound) { retryCount++; Trace.Error($"Failed to get JIT runner token -- Atempt: {retryCount}"); @@ -714,22 +715,23 @@ private async Task GetTenantCredential(string githubUrl, strin { var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json")); responseStatus = response.StatusCode; + var githubRequestId = GetGitHubRequestId(response.Headers); - if(response.IsSuccessStatusCode) + if (response.IsSuccessStatusCode) { - Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'"); + Trace.Info($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' ({githubRequestId})"); var jsonResponse = await response.Content.ReadAsStringAsync(); return StringUtil.ConvertFromJson(jsonResponse); } else { - _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}'"); + _term.WriteError($"Http response code: {response.StatusCode} from 'POST {githubApiUrl}' (Request Id: {githubRequestId})"); var errorResponse = await response.Content.ReadAsStringAsync(); _term.WriteError(errorResponse); response.EnsureSuccessStatusCode(); } } - catch(Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound) + catch (Exception ex) when (retryCount < 2 && responseStatus != System.Net.HttpStatusCode.NotFound) { retryCount++; Trace.Error($"Failed to get tenant credentials -- Atempt: {retryCount}"); @@ -742,5 +744,14 @@ private async Task GetTenantCredential(string githubUrl, strin } return null; } + + private string GetGitHubRequestId(HttpResponseHeaders headers) + { + if (headers.TryGetValues("x-github-request-id", out var headerValues)) + { + return headerValues.FirstOrDefault(); + } + return string.Empty; + } } } From 850e416316d83f0cb2eb5184d738b6da2d2dc35e Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 19 Dec 2022 16:57:16 -0500 Subject: [PATCH 071/145] Make runner image print diag log to STDOUT. (#2331) --- images/Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/images/Dockerfile b/images/Dockerfile index 83ea6b51140..27ba988598a 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -19,6 +19,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 ENV RUNNER_ALLOW_RUNASROOT=1 ENV RUNNER_MANUALLY_TRAP_SIG=1 +ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1 WORKDIR /actions-runner COPY --from=build /actions-runner . From 6a907a04a75f5396269cbdb79b2b4e67ba6998fc Mon Sep 17 00:00:00 2001 From: Stefan Ruvceski <96768603+ruvceskistefan@users.noreply.github.com> Date: Tue, 20 Dec 2022 14:28:29 +0100 Subject: [PATCH 072/145] split by regex (#2333) * split by regex * pr fix * adding tests * test fix --- src/Runner.Sdk/ProcessInvoker.cs | 5 ++- src/Test/L0/ProcessInvokerL0.cs | 68 ++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 2 deletions(-) diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index e46b2a057aa..de980463944 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -11,6 +11,7 @@ using System.Threading.Channels; using System.Threading.Tasks; using GitHub.Runner.Sdk; +using System.Text.RegularExpressions; namespace GitHub.Runner.Sdk { @@ -265,8 +266,8 @@ public async Task ExecuteAsync( foreach (KeyValuePair kvp in environment) { #if OS_WINDOWS - string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : kvp.Key.Split('\0')[0]; - string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : kvp.Value.Split('\0')[0]; + string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : Regex.Split(kvp.Key, @"\p{C}")[0]; + string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : Regex.Split(kvp.Value, @"\p{C}")[0]; if(!String.IsNullOrWhiteSpace(tempKey)) { _proc.StartInfo.Environment[tempKey] = tempValue; diff --git a/src/Test/L0/ProcessInvokerL0.cs b/src/Test/L0/ProcessInvokerL0.cs index 6d99932e3e2..cd958e2f9de 100644 --- a/src/Test/L0/ProcessInvokerL0.cs +++ b/src/Test/L0/ProcessInvokerL0.cs @@ -164,6 +164,40 @@ public async Task SetTestEnvWithNullInKey() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public async Task SetTestEnvWithTabInKey() + { + using (TestHostContext hc = new(this)) + { + Tracing trace = hc.GetTrace(); + + Int32 exitCode = -1; + var processInvoker = new ProcessInvokerWrapper(); + processInvoker.Initialize(hc); + var stdout = new List(); + var stderr = new List(); + processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stdout.Add(e.Data); + }; + processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stderr.Add(e.Data); + }; + + exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary() { { "TEST\u0009second", "first" } }, CancellationToken.None); + + trace.Info("Exit Code: {0}", exitCode); + Assert.Equal(0, exitCode); + Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x))); + + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Common")] @@ -197,6 +231,40 @@ public async Task SetTestEnvWithNullInValue() } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public async Task SetTestEnvWithTabInValue() + { + using (TestHostContext hc = new(this)) + { + Tracing trace = hc.GetTrace(); + + Int32 exitCode = -1; + var processInvoker = new ProcessInvokerWrapper(); + processInvoker.Initialize(hc); + var stdout = new List(); + var stderr = new List(); + processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stdout.Add(e.Data); + }; + processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => + { + trace.Info(e.Data); + stderr.Add(e.Data); + }; + + exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary() { { "TEST", "first\u0009second" } }, CancellationToken.None); + + trace.Info("Exit Code: {0}", exitCode); + Assert.Equal(0, exitCode); + Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x))); + + } + } #endif [Fact] [Trait("Level", "L0")] From b7e09ad08c9f340d7fc3653fdea1fd5501105682 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Wed, 21 Dec 2022 19:30:22 +0100 Subject: [PATCH 073/145] Treat jitconfig as secret. (#2335) Co-authored-by: TingluoHuang --- src/Runner.Common/Constants.cs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 2f5b69d8467..57f8e8e4dd6 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -90,7 +90,6 @@ public static class CommandLine public static class Args { public static readonly string Auth = "auth"; - public static readonly string JitConfig = "jitconfig"; public static readonly string Labels = "labels"; public static readonly string MonitorSocketAddress = "monitorsocketaddress"; public static readonly string Name = "name"; @@ -105,11 +104,13 @@ public static class Args public static readonly string Token = "token"; public static readonly string PAT = "pat"; public static readonly string WindowsLogonPassword = "windowslogonpassword"; + public static readonly string JitConfig = "jitconfig"; public static string[] Secrets => new[] { PAT, Token, WindowsLogonPassword, + JitConfig, }; } From 60585311f596fc81f7e13074724cdf61fa832ea0 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 22 Dec 2022 10:38:29 -0500 Subject: [PATCH 074/145] Add Header/Footer to multi-line message in StdoutTraceListener. (#2336) --- src/Runner.Common/StdoutTraceListener.cs | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/src/Runner.Common/StdoutTraceListener.cs b/src/Runner.Common/StdoutTraceListener.cs index 53ebaf5dba3..26822c54567 100644 --- a/src/Runner.Common/StdoutTraceListener.cs +++ b/src/Runner.Common/StdoutTraceListener.cs @@ -1,8 +1,8 @@ -using GitHub.Runner.Sdk; using System; using System.Diagnostics; using System.Globalization; using System.IO; +using GitHub.Runner.Sdk; namespace GitHub.Runner.Common { @@ -24,9 +24,16 @@ public override void TraceEvent(TraceEventCache eventCache, string source, Trace return; } - WriteHeader(source, eventType, id); - WriteLine(message); - WriteFooter(eventCache); + if (!string.IsNullOrEmpty(message)) + { + var messageLines = message.Split(Environment.NewLine); + foreach (var messageLine in messageLines) + { + WriteHeader(source, eventType, id); + WriteLine(messageLine); + WriteFooter(eventCache); + } + } } internal bool IsEnabled(TraceOptions opts) @@ -86,5 +93,4 @@ private void WriteFooter(TraceEventCache eventCache) IndentLevel--; } } -} - +} \ No newline at end of file From 8a1f32ca9a75ba9427225cd7ee4a35e47e2a57bf Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Wed, 28 Dec 2022 11:56:53 -0500 Subject: [PATCH 075/145] Always upload to avoid issues (#2334) * Remove unnecessary timelineId and timelineRecordId and use Guid stepId * Log upload error to kusto * Remove try-catch * Using a well known telmetry record to avoid replacing issues * fix Guid format --- src/Runner.Common/Constants.cs | 2 ++ src/Runner.Common/JobServerQueue.cs | 28 +++++++++++++--------- src/Runner.Worker/ExecutionContext.cs | 6 ++--- src/Runner.Worker/FileCommandManager.cs | 13 +++++----- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 4 ++++ 5 files changed, 32 insertions(+), 21 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 57f8e8e4dd6..0caee0378a9 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -158,9 +158,11 @@ public static class Features } public static readonly string InternalTelemetryIssueDataKey = "_internal_telemetry"; + public static readonly Guid TelemetryRecordId = new Guid("11111111-1111-1111-1111-111111111111"); public static readonly string WorkerCrash = "WORKER_CRASH"; public static readonly string LowDiskSpace = "LOW_DISK_SPACE"; public static readonly string UnsupportedCommand = "UNSUPPORTED_COMMAND"; + public static readonly string ResultsUploadFailure = "RESULTS_UPLOAD_FAILURE"; public static readonly string UnsupportedCommandMessage = "The `{0}` command is deprecated and will be disabled soon. Please upgrade to using Environment Files. For more information see: https://github.blog/changelog/2022-10-11-github-actions-deprecating-save-state-and-set-output-commands/"; public static readonly string UnsupportedCommandMessageDisabled = "The `{0}` command is disabled. Please upgrade to using Environment Files or opt into unsecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_COMMANDS` environment variable to `true`. For more information see: https://github.blog/changelog/2020-10-01-github-actions-deprecating-set-env-and-add-path-commands/"; public static readonly string UnsupportedStopCommandTokenDisabled = "You cannot use a endToken that is an empty string, the string 'pause-logging', or another workflow command. For more information see: https://docs.github.com/actions/learn-github-actions/workflow-commands-for-github-actions#example-stopping-and-starting-workflow-commands or opt into insecure command execution by setting the `ACTIONS_ALLOW_UNSECURE_STOPCOMMAND_TOKENS` environment variable to `true`."; diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index fb97ec73220..6440da73607 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -20,7 +20,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter void Start(Pipelines.AgentJobRequestMessage jobRequest); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); - void QueueSummaryUpload(Guid timelineId, Guid timelineRecordId, string stepId, string name, string path, bool deleteSource); + void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); } @@ -230,25 +230,20 @@ public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, _fileUploadQueue.Enqueue(newFile); } - public void QueueSummaryUpload(Guid timelineId, Guid timelineRecordId, string stepId, string name, string path, bool deleteSource) + public void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource) { - ArgUtil.NotEmpty(timelineId, nameof(timelineId)); - ArgUtil.NotEmpty(timelineRecordId, nameof(timelineRecordId)); - // all parameter not null, file path exist. var newFile = new SummaryUploadFileInfo() { - TimelineId = timelineId, - TimelineRecordId = timelineRecordId, Name = name, Path = path, PlanId = _planId.ToString(), JobId = _jobTimelineRecordId.ToString(), - StepId = stepId, + StepId = stepRecordId.ToString(), DeleteSource = deleteSource }; - Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to record {1}", newFile.Path, timelineRecordId); + Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, stepRecordId); _summaryFileUploadQueue.Enqueue(newFile); } @@ -476,8 +471,21 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) } catch (Exception ex) { + var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during summary file upload to results. {ex.Message}" }; + issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; + + var telemetryRecord = new TimelineRecord() + { + Id = Constants.Runner.TelemetryRecordId, + }; + telemetryRecord.Issues.Add(issue); + QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord); + Trace.Info("Catch exception during summary file upload to results, keep going since the process is best effort."); Trace.Error(ex); + } + finally + { errorCount++; } } @@ -816,8 +824,6 @@ internal class UploadFileInfo internal class SummaryUploadFileInfo { - public Guid TimelineId { get; set; } - public Guid TimelineRecordId { get; set; } public string Name { get; set; } public string Path { get; set; } public string PlanId { get; set; } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index c8e6ca41817..1b5b0b975a8 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -80,7 +80,7 @@ public interface IExecutionContext : IRunnerService // logging long Write(string tag, string message); void QueueAttachFile(string type, string name, string filePath); - void QueueSummaryFile(string name, string filePath, string stepId); + void QueueSummaryFile(string name, string filePath, Guid stepRecordId); // timeline record update methods void Start(string currentOperation = null); @@ -847,7 +847,7 @@ public void QueueAttachFile(string type, string name, string filePath) _jobServerQueue.QueueFileUpload(_mainTimelineId, _record.Id, type, name, filePath, deleteSource: false); } - public void QueueSummaryFile(string name, string filePath, string stepId) + public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) { ArgUtil.NotNullOrEmpty(name, nameof(name)); ArgUtil.NotNullOrEmpty(filePath, nameof(filePath)); @@ -857,7 +857,7 @@ public void QueueSummaryFile(string name, string filePath, string stepId) throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist."); } - _jobServerQueue.QueueSummaryUpload(_mainTimelineId, _record.Id, stepId, name, filePath, deleteSource: false); + _jobServerQueue.QueueSummaryUpload(stepRecordId, name, filePath, deleteSource: false); } // Add OnMatcherChanged diff --git a/src/Runner.Worker/FileCommandManager.cs b/src/Runner.Worker/FileCommandManager.cs index a4eb2e676ea..646e56548a5 100644 --- a/src/Runner.Worker/FileCommandManager.cs +++ b/src/Runner.Worker/FileCommandManager.cs @@ -208,20 +208,19 @@ public void ProcessCommand(IExecutionContext context, string filePath, Container ? context.Id.ToString() : context.EmbeddedId.ToString(); + Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); + // Attachments must be added to the parent context (job), not the current context (step) + context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); + + // Dual upload the same files to Results Service context.Global.Variables.TryGetValue("system.github.results_endpoint", out string resultsReceiverEndpoint); if (resultsReceiverEndpoint != null) { Trace.Info($"Queueing results file ({filePath}) for attachment upload ({attachmentName})"); - var stepId = context.Id.ToString(); + var stepId = context.Id; // Attachments must be added to the parent context (job), not the current context (step) context.Root.QueueSummaryFile(attachmentName, scrubbedFilePath, stepId); } - else - { - Trace.Info($"Queueing file ({filePath}) for attachment upload ({attachmentName})"); - // Attachments must be added to the parent context (job), not the current context (step) - context.Root.QueueAttachFile(ChecksAttachmentType.StepSummary, attachmentName, scrubbedFilePath); - } } catch (Exception e) { diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 674abb20db8..77a733eaeba 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -113,6 +113,10 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, string ste { // Get the upload url var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken); + if (uploadUrlResponse == null) + { + throw new Exception("Failed to get step summary upload url"); + } // Do we want to throw an exception here or should we just be uploading/truncating the data var fileSize = new FileInfo(file).Length; From 90e0dff69bd7a8972b87f6550f9fbc657c9f44de Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 16 Jan 2023 10:18:55 -0500 Subject: [PATCH 076/145] Allow provide extra User-Agent for better correlation. (#2370) --- src/Runner.Common/HostContext.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 9e4346489a5..4a78da83fd2 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -226,6 +226,20 @@ public HostContext(string hostType, string logFile = null) } _userAgents.Add(new ProductInfoHeaderValue("CommitSHA", BuildConstants.Source.CommitHash)); + + var extraUserAgent = Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_EXTRA_USER_AGENT"); + if (!string.IsNullOrEmpty(extraUserAgent)) + { + var extraUserAgentSplit = extraUserAgent.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (extraUserAgentSplit.Length != 2) + { + _trace.Error($"GITHUB_ACTIONS_RUNNER_EXTRA_USER_AGENT is not in the format of 'name/version'."); + } + + var extraUserAgentHeader = new ProductInfoHeaderValue(extraUserAgentSplit[0], extraUserAgentSplit[1]); + _trace.Info($"Adding extra user agent '{extraUserAgentHeader}' to all HTTP requests."); + _userAgents.Add(extraUserAgentHeader); + } } public string GetDirectory(WellKnownDirectory directory) From 4c7e7862a8d187b240dd78ecba08c525bb4340a8 Mon Sep 17 00:00:00 2001 From: yujincat <55138573+yujincat@users.noreply.github.com> Date: Thu, 19 Jan 2023 00:40:35 +0900 Subject: [PATCH 077/145] Show more information in the runner log (#2377) * fix typo * add workflow ref in the log * show job name for all jobs * update ref * reflect the feedback * fix a small bug --- docs/contribute/vscode.md | 2 +- src/Runner.Worker/JobExtension.cs | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/docs/contribute/vscode.md b/docs/contribute/vscode.md index 65ec207c083..f8968295124 100644 --- a/docs/contribute/vscode.md +++ b/docs/contribute/vscode.md @@ -35,7 +35,7 @@ All the configs below can be found in `.vscode/launch.json`. If you launch `Run` or `Run [build]`, it starts a process called `Runner.Listener`. This process will receive any job queued on this repository if the job runs on matching labels (e.g `runs-on: self-hosted`). Once a job is received, a `Runner.Listener` starts a new process of `Runner.Worker`. -Since this is a diferent process, you can't use the same debugger session debug it. +Since this is a different process, you can't use the same debugger session debug it. Instead, a parallel debugging session has to be started, using a different launch config. Luckily, VS Code supports multiple parallel debugging sessions. diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 246274734d9..770db66435d 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -321,7 +321,10 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel if (message.Variables.TryGetValue("system.workflowFileFullPath", out VariableValue workflowFileFullPath)) { - context.Output($"Uses: {workflowFileFullPath.Value}"); + var usesLogText = $"Uses: {workflowFileFullPath.Value}"; + var reference = GetWorkflowReference(message.Variables); + context.Output(usesLogText + reference); + if (message.ContextData.TryGetValue("inputs", out var pipelineContextData)) { var inputs = pipelineContextData.AssertDictionary("inputs"); @@ -335,11 +338,11 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.Output("##[endgroup]"); } } + } - if (!string.IsNullOrWhiteSpace(message.JobDisplayName)) - { - context.Output($"Complete job name: {message.JobDisplayName}"); - } + if (!string.IsNullOrWhiteSpace(message.JobDisplayName)) + { + context.Output($"Complete job name: {message.JobDisplayName}"); } var intraActionStates = new Dictionary>(); @@ -452,6 +455,24 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } + private string GetWorkflowReference(IDictionary variables) + { + var reference = ""; + if (variables.TryGetValue("system.workflowFileSha", out VariableValue workflowFileSha)) + { + if (variables.TryGetValue("system.workflowFileRef", out VariableValue workflowFileRef) + && !string.IsNullOrEmpty(workflowFileRef.Value)) + { + reference += $"@{workflowFileRef.Value} ({workflowFileSha.Value})"; + } + else + { + reference += $"@{workflowFileSha.Value}"; + } + } + return reference; + } + public void FinalizeJob(IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, DateTime jobStartTimeUtc) { Trace.Entering(); From 9bb450436042a4ddde3dcbcece9e939e457db9aa Mon Sep 17 00:00:00 2001 From: John Sudol <24583161+johnsudol@users.noreply.github.com> Date: Wed, 18 Jan 2023 10:41:15 -0500 Subject: [PATCH 078/145] update node to 16.16.0 (#2371) --- src/Misc/contentHash/externals/linux-arm | 2 +- src/Misc/contentHash/externals/linux-arm64 | 2 +- src/Misc/contentHash/externals/linux-x64 | 2 +- src/Misc/contentHash/externals/osx-arm64 | 2 +- src/Misc/contentHash/externals/osx-x64 | 2 +- src/Misc/contentHash/externals/win-arm64 | 2 +- src/Misc/contentHash/externals/win-x64 | 2 +- src/Misc/externals.sh | 2 +- 8 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/Misc/contentHash/externals/linux-arm b/src/Misc/contentHash/externals/linux-arm index 742fcf719b2..778d0975d4a 100644 --- a/src/Misc/contentHash/externals/linux-arm +++ b/src/Misc/contentHash/externals/linux-arm @@ -1 +1 @@ -6ed30a2c1ee403a610d63e82bb230b9ba846a9c25cec9e4ea8672fb6ed4e1a51 \ No newline at end of file +3807dcbf947e840c33535fb466b096d76bf09e5c0254af8fc8cbbb24c6388222 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/linux-arm64 b/src/Misc/contentHash/externals/linux-arm64 index d7fdd3308cd..49f41a1da9e 100644 --- a/src/Misc/contentHash/externals/linux-arm64 +++ b/src/Misc/contentHash/externals/linux-arm64 @@ -1 +1 @@ -711c30c51ec52c9b7a9a2eb399d6ab2ab5ee1dc72de11879f2f36f919f163d78 \ No newline at end of file +ee01eee80cd8a460a4b9780ee13fdd20f25c59e754b4ccd99df55fbba2a85634 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/linux-x64 b/src/Misc/contentHash/externals/linux-x64 index 9722fe1e88c..29c47502111 100644 --- a/src/Misc/contentHash/externals/linux-x64 +++ b/src/Misc/contentHash/externals/linux-x64 @@ -1 +1 @@ -a49479ca4b4988a06c097e8d22c51fd08a11c13f40807366236213d0e008cf6a \ No newline at end of file +a9fb9c14e24e79aec97d4da197dd7bfc6364297d6fce573afb2df48cc9a931f8 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/osx-arm64 b/src/Misc/contentHash/externals/osx-arm64 index 5f2d6f38c5f..7418d71c3ea 100644 --- a/src/Misc/contentHash/externals/osx-arm64 +++ b/src/Misc/contentHash/externals/osx-arm64 @@ -1 +1 @@ -cc4708962a80325de0baa5ae8484e0cb9ae976ac6a4178c1c0d448b8c52bd7f7 \ No newline at end of file +a4e0e8fc62eba0967a39c7d693dcd0aeb8b2bed0765f9c38df80d42884f65341 \ No newline at end of file diff --git a/src/Misc/contentHash/externals/osx-x64 b/src/Misc/contentHash/externals/osx-x64 index b21bee85912..f387ba76ede 100644 --- a/src/Misc/contentHash/externals/osx-x64 +++ b/src/Misc/contentHash/externals/osx-x64 @@ -1 +1 @@ -8e97df75230b843462a9b4c578ccec604ee4b4a1066120c85b04374317fa372b \ No newline at end of file +17ac17fbe785b3d6fa2868d8d17185ebfe0c90b4b0ddf6b67eac70e42bcd989b \ No newline at end of file diff --git a/src/Misc/contentHash/externals/win-arm64 b/src/Misc/contentHash/externals/win-arm64 index bd10bc5ea31..04f80554432 100644 --- a/src/Misc/contentHash/externals/win-arm64 +++ b/src/Misc/contentHash/externals/win-arm64 @@ -1 +1 @@ -e5dace2d41cc0682d096dcce4970079ad48ec7107e46195970eecfdb3df2acef +89f24657a550f1e818b0e9975e5b80edcf4dd22b7d4bccbb9e48e37f45d30fb1 diff --git a/src/Misc/contentHash/externals/win-x64 b/src/Misc/contentHash/externals/win-x64 index 5ed33fa4998..46422113b85 100644 --- a/src/Misc/contentHash/externals/win-x64 +++ b/src/Misc/contentHash/externals/win-x64 @@ -1 +1 @@ -f75a671e5a188c76680739689aa75331a2c09d483dce9c80023518c48fd67a18 \ No newline at end of file +24fd131b5dce33ef16038b771407bc0507da8682a72fb3b7780607235f76db0b \ No newline at end of file diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 123c46ce5bd..f9c9c95df39 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -5,7 +5,7 @@ PRECACHE=$2 NODE_URL=https://nodejs.org/dist UNOFFICIAL_NODE_URL=https://unofficial-builds.nodejs.org/download/release NODE12_VERSION="12.22.7" -NODE16_VERSION="16.13.0" +NODE16_VERSION="16.16.0" get_abs_path() { # exploits the fact that pwd will print abs path when no args From f6b01101d2c63c15c16c3bfa125391d52ac29fce Mon Sep 17 00:00:00 2001 From: Cory Miller <13227161+cory-miller@users.noreply.github.com> Date: Wed, 18 Jan 2023 10:41:53 -0500 Subject: [PATCH 079/145] Add a disclaimer for which runner version is available to a given tenant (#2362) * Add a disclaimer for which runner version is available to a given tenant * Update releaseNote.md --- releaseNote.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/releaseNote.md b/releaseNote.md index 60012cc9104..42999201d96 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -13,6 +13,10 @@ - Return exit code when MANUALLY_TRAP_SIG is exported (#2285) - Use results for uploading step summaries (#2301) with limited size (#2321) +_Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. +To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. +See https://docs.github.com/en/enterprise-cloud@latest/actions/hosting-your-own-runners/adding-self-hosted-runners_ + ## Windows x64 We recommend configuring the runner in a root folder of the Windows drive (e.g. "C:\actions-runner"). This will help avoid issues related to service identity folder permissions and long file path restrictions on Windows. From 9a80f6fd6c9542fe3b90040cf52fb7d85daf12ad Mon Sep 17 00:00:00 2001 From: John Sudol <24583161+johnsudol@users.noreply.github.com> Date: Wed, 18 Jan 2023 11:28:43 -0500 Subject: [PATCH 080/145] new option to remove local config files (#2367) --- src/Runner.Common/Constants.cs | 1 + src/Runner.Listener/CommandSettings.cs | 4 +++- src/Runner.Listener/Runner.cs | 7 ++++++ src/Test/L0/Listener/CommandSettingsL0.cs | 1 + src/Test/L0/Listener/RunnerL0.cs | 29 +++++++++++++++++++++++ 5 files changed, 41 insertions(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 0caee0378a9..86599ada0fa 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -131,6 +131,7 @@ public static class Flags public static readonly string Ephemeral = "ephemeral"; public static readonly string GenerateServiceConfig = "generateServiceConfig"; public static readonly string Help = "help"; + public static readonly string Local = "local"; public static readonly string Replace = "replace"; public static readonly string DisableUpdate = "disableupdate"; public static readonly string Once = "once"; // Keep this around since customers still relies on it diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index 4d00f1d4592..cce711696cb 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -56,7 +56,8 @@ public sealed class CommandSettings new string[] { Constants.Runner.CommandLine.Args.Token, - Constants.Runner.CommandLine.Args.PAT + Constants.Runner.CommandLine.Args.PAT, + Constants.Runner.CommandLine.Flags.Local }, // Valid run flags and args [Constants.Runner.CommandLine.Commands.Run] = @@ -86,6 +87,7 @@ public sealed class CommandSettings public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help); public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended); public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version); + public bool RemoveLocalConfig => TestFlag(Constants.Runner.CommandLine.Flags.Local); // Keep this around since customers still relies on it public bool RunOnce => TestFlag(Constants.Runner.CommandLine.Flags.Once); diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 853e0791207..c28e5a9f7c4 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -135,6 +135,12 @@ public async Task ExecuteCommand(CommandSettings command) // remove config files, remove service, and exit if (command.Remove) { + // only remove local config files and exit + if(command.RemoveLocalConfig) + { + configManager.DeleteLocalRunnerConfig(); + return Constants.Runner.ReturnCode.Success; + } try { await configManager.UnconfigureAsync(command); @@ -647,6 +653,7 @@ private void PrintUsage(CommandSettings command) --name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"}) --runnergroup string Name of the runner group to add this runner to (defaults to the default runner group) --labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}' + --local Removes the runner config files from your local machine. Used as an option to the remove command --work string Relative runner work directory (default {Constants.Path.WorkDirectory}) --replace Replace any existing runner with the same name (default false) --pat GitHub personal access token with repo scope. Used for checking network connectivity when executing `.{separator}run.{ext} --check` diff --git a/src/Test/L0/Listener/CommandSettingsL0.cs b/src/Test/L0/Listener/CommandSettingsL0.cs index 249d050895e..c407ce54ac8 100644 --- a/src/Test/L0/Listener/CommandSettingsL0.cs +++ b/src/Test/L0/Listener/CommandSettingsL0.cs @@ -824,6 +824,7 @@ public void ValidateGoodCommandline() [InlineData("remove", "version")] [InlineData("remove", "commit")] [InlineData("remove", "check")] + [InlineData("remove", "local")] [InlineData("run", "help")] [InlineData("run", "version")] [InlineData("run", "commit")] diff --git a/src/Test/L0/Listener/RunnerL0.cs b/src/Test/L0/Listener/RunnerL0.cs index f35f7e766fe..f64cd8b9e5a 100644 --- a/src/Test/L0/Listener/RunnerL0.cs +++ b/src/Test/L0/Listener/RunnerL0.cs @@ -502,5 +502,34 @@ public async void TestRunOnceHandleUpdateMessage() _messageListener.Verify(x => x.DeleteMessageAsync(It.IsAny()), Times.Once()); } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void TestRemoveLocalRunnerConfig() + { + using (var hc = new TestHostContext(this)) + { + hc.SetSingleton(_configurationManager.Object); + hc.SetSingleton(_configStore.Object); + hc.SetSingleton(_promptManager.Object); + + var command = new CommandSettings(hc, new[] { "remove", "--local" }); + + _configStore.Setup(x => x.IsConfigured()) + .Returns(true); + + _configStore.Setup(x => x.HasCredentials()) + .Returns(true); + + + var runner = new Runner.Listener.Runner(); + runner.Initialize(hc); + await runner.ExecuteCommand(command); + + // verify that we delete the local runner config with the correct remove parameter + _configurationManager.Verify(x => x.DeleteLocalRunnerConfig(), Times.Once()); + } + } } } From 54f998e902b6e4636e9d47c8e513a309db3ce8d9 Mon Sep 17 00:00:00 2001 From: John Sudol <24583161+johnsudol@users.noreply.github.com> Date: Wed, 18 Jan 2023 14:19:28 -0500 Subject: [PATCH 081/145] Update Node dependencies (#2381) Co-authored-by: Tingluo Huang --- .devcontainer/devcontainer.json | 3 + .../hashFiles/package-lock.json | 43 +- .../expressionFunc/hashFiles/package.json | 2 +- src/Misc/layoutbin/hashFiles/index.js | 7500 +++++++++-------- 4 files changed, 3795 insertions(+), 3753 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 2981e97e275..ec8b7d2831a 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -6,6 +6,9 @@ "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { "version": "6.0.300" + }, + "ghcr.io/devcontainers/features/node:1": { + "version": "16" } }, "customizations": { diff --git a/src/Misc/expressionFunc/hashFiles/package-lock.json b/src/Misc/expressionFunc/hashFiles/package-lock.json index c584d01b315..da51bd4bccf 100644 --- a/src/Misc/expressionFunc/hashFiles/package-lock.json +++ b/src/Misc/expressionFunc/hashFiles/package-lock.json @@ -14,7 +14,7 @@ "devDependencies": { "@types/node": "^12.7.12", "@typescript-eslint/parser": "^5.15.0", - "@zeit/ncc": "^0.20.5", + "@vercel/ncc": "^0.36.0", "eslint": "^8.11.0", "eslint-plugin-github": "^4.3.5", "prettier": "^1.19.1", @@ -346,11 +346,10 @@ "url": "https://opencollective.com/typescript-eslint" } }, - "node_modules/@zeit/ncc": { - "version": "0.20.5", - "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.20.5.tgz", - "integrity": "sha512-XU6uzwvv95DqxciQx+aOLhbyBx/13ky+RK1y88Age9Du3BlA4mMPCy13BGjayOrrumOzlq1XV3SD/BWiZENXlw==", - "deprecated": "@zeit/ncc is no longer maintained. Please use @vercel/ncc instead.", + "node_modules/@vercel/ncc": { + "version": "0.36.0", + "resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.36.0.tgz", + "integrity": "sha512-/ZTUJ/ZkRt694k7KJNimgmHjtQcRuVwsST2Z6XfYveQIuBbHR+EqkTc1jfgPkQmMyk/vtpxo3nVxe8CNuau86A==", "dev": true, "bin": { "ncc": "dist/ncc/cli.js" @@ -1722,9 +1721,9 @@ "dev": true }, "node_modules/json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "dev": true, "dependencies": { "minimist": "^1.2.0" @@ -1824,9 +1823,9 @@ } }, "node_modules/minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "dependencies": { "brace-expansion": "^1.1.7" }, @@ -2747,10 +2746,10 @@ "eslint-visitor-keys": "^3.0.0" } }, - "@zeit/ncc": { - "version": "0.20.5", - "resolved": "https://registry.npmjs.org/@zeit/ncc/-/ncc-0.20.5.tgz", - "integrity": "sha512-XU6uzwvv95DqxciQx+aOLhbyBx/13ky+RK1y88Age9Du3BlA4mMPCy13BGjayOrrumOzlq1XV3SD/BWiZENXlw==", + "@vercel/ncc": { + "version": "0.36.0", + "resolved": "https://registry.npmjs.org/@vercel/ncc/-/ncc-0.36.0.tgz", + "integrity": "sha512-/ZTUJ/ZkRt694k7KJNimgmHjtQcRuVwsST2Z6XfYveQIuBbHR+EqkTc1jfgPkQmMyk/vtpxo3nVxe8CNuau86A==", "dev": true }, "acorn": { @@ -3756,9 +3755,9 @@ "dev": true }, "json5": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.1.tgz", - "integrity": "sha512-aKS4WQjPenRxiQsC93MNfjx+nbF4PAdYzmd/1JIj8HYzqfbu86beTuNgXDzPknWk0n0uARlyewZo4s++ES36Ow==", + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/json5/-/json5-1.0.2.tgz", + "integrity": "sha512-g1MWMLBiz8FKi1e4w0UyVL3w+iJceWAFBAaBnnGKOpNa5f8TLktkbre1+s6oICydWAm+HRUGTmI+//xv2hvXYA==", "dev": true, "requires": { "minimist": "^1.2.0" @@ -3840,9 +3839,9 @@ } }, "minimatch": { - "version": "3.0.4", - "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", - "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz", + "integrity": "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw==", "requires": { "brace-expansion": "^1.1.7" } diff --git a/src/Misc/expressionFunc/hashFiles/package.json b/src/Misc/expressionFunc/hashFiles/package.json index acff9e3dab7..48dfe4281f3 100644 --- a/src/Misc/expressionFunc/hashFiles/package.json +++ b/src/Misc/expressionFunc/hashFiles/package.json @@ -26,7 +26,7 @@ "devDependencies": { "@types/node": "^12.7.12", "@typescript-eslint/parser": "^5.15.0", - "@zeit/ncc": "^0.20.5", + "@vercel/ncc": "^0.36.0", "eslint": "^8.11.0", "eslint-plugin-github": "^4.3.5", "prettier": "^1.19.1", diff --git a/src/Misc/layoutbin/hashFiles/index.js b/src/Misc/layoutbin/hashFiles/index.js index e0bf2f96b47..6a0f30e7049 100644 --- a/src/Misc/layoutbin/hashFiles/index.js +++ b/src/Misc/layoutbin/hashFiles/index.js @@ -1,1731 +1,1092 @@ -module.exports = -/******/ (function(modules, runtime) { // webpackBootstrap -/******/ "use strict"; -/******/ // The module cache -/******/ var installedModules = {}; -/******/ -/******/ // The require function -/******/ function __webpack_require__(moduleId) { -/******/ -/******/ // Check if module is in cache -/******/ if(installedModules[moduleId]) { -/******/ return installedModules[moduleId].exports; -/******/ } -/******/ // Create a new module (and put it into the cache) -/******/ var module = installedModules[moduleId] = { -/******/ i: moduleId, -/******/ l: false, -/******/ exports: {} -/******/ }; -/******/ -/******/ // Execute the module function -/******/ modules[moduleId].call(module.exports, module, module.exports, __webpack_require__); -/******/ -/******/ // Flag the module as loaded -/******/ module.l = true; -/******/ -/******/ // Return the exports of the module -/******/ return module.exports; -/******/ } -/******/ -/******/ -/******/ __webpack_require__.ab = __dirname + "/"; -/******/ -/******/ // the startup function -/******/ function startup() { -/******/ // Load entry module and return exports -/******/ return __webpack_require__(349); -/******/ }; -/******/ -/******/ // run startup -/******/ return startup(); -/******/ }) -/************************************************************************/ -/******/ ({ - -/***/ 16: -/***/ (function(module) { - -module.exports = require("tls"); - -/***/ }), +/******/ (() => { // webpackBootstrap +/******/ var __webpack_modules__ = ({ -/***/ 22: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 2627: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; - -var _validate = _interopRequireDefault(__webpack_require__(78)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function parse(uuid) { - if (!(0, _validate.default)(uuid)) { - throw TypeError('Invalid UUID'); - } - - let v; - const arr = new Uint8Array(16); // Parse ########-....-....-....-............ - - arr[0] = (v = parseInt(uuid.slice(0, 8), 16)) >>> 24; - arr[1] = v >>> 16 & 0xff; - arr[2] = v >>> 8 & 0xff; - arr[3] = v & 0xff; // Parse ........-####-....-....-............ - - arr[4] = (v = parseInt(uuid.slice(9, 13), 16)) >>> 8; - arr[5] = v & 0xff; // Parse ........-....-####-....-............ - - arr[6] = (v = parseInt(uuid.slice(14, 18), 16)) >>> 8; - arr[7] = v & 0xff; // Parse ........-....-....-####-............ - - arr[8] = (v = parseInt(uuid.slice(19, 23), 16)) >>> 8; - arr[9] = v & 0xff; // Parse ........-....-....-....-############ - // (Use "/" to avoid 32-bit truncation when bit-shifting high-order bytes) - - arr[10] = (v = parseInt(uuid.slice(24, 36), 16)) / 0x10000000000 & 0xff; - arr[11] = v / 0x100000000 & 0xff; - arr[12] = v >>> 24 & 0xff; - arr[13] = v >>> 16 & 0xff; - arr[14] = v >>> 8 & 0xff; - arr[15] = v & 0xff; - return arr; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +var __asyncValues = (this && this.__asyncValues) || function (o) { + if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); + var m = o[Symbol.asyncIterator], i; + return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); + function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } + function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } +}; +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; + result["default"] = mod; + return result; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +const crypto = __importStar(__nccwpck_require__(6113)); +const fs = __importStar(__nccwpck_require__(7147)); +const glob = __importStar(__nccwpck_require__(8090)); +const path = __importStar(__nccwpck_require__(1017)); +const stream = __importStar(__nccwpck_require__(2781)); +const util = __importStar(__nccwpck_require__(3837)); +function run() { + var e_1, _a; + return __awaiter(this, void 0, void 0, function* () { + // arg0 -> node + // arg1 -> hashFiles.js + // env[followSymbolicLinks] = true/null + // env[patterns] -> glob patterns + let followSymbolicLinks = false; + const matchPatterns = process.env.patterns || ''; + if (process.env.followSymbolicLinks === 'true') { + console.log('Follow symbolic links'); + followSymbolicLinks = true; + } + console.log(`Match Pattern: ${matchPatterns}`); + let hasMatch = false; + const githubWorkspace = process.cwd(); + const result = crypto.createHash('sha256'); + let count = 0; + const globber = yield glob.create(matchPatterns, { followSymbolicLinks }); + try { + for (var _b = __asyncValues(globber.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { + const file = _c.value; + console.log(file); + if (!file.startsWith(`${githubWorkspace}${path.sep}`)) { + console.log(`Ignore '${file}' since it is not under GITHUB_WORKSPACE.`); + continue; + } + if (fs.statSync(file).isDirectory()) { + console.log(`Skip directory '${file}'.`); + continue; + } + const hash = crypto.createHash('sha256'); + const pipeline = util.promisify(stream.pipeline); + yield pipeline(fs.createReadStream(file), hash); + result.write(hash.digest()); + count++; + if (!hasMatch) { + hasMatch = true; + } + } + } + catch (e_1_1) { e_1 = { error: e_1_1 }; } + finally { + try { + if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b); + } + finally { if (e_1) throw e_1.error; } + } + result.end(); + if (hasMatch) { + console.log(`Found ${count} files to hash.`); + console.error(`__OUTPUT__${result.digest('hex')}__OUTPUT__`); + } + else { + console.error(`__OUTPUT____OUTPUT__`); + } + }); } +run() + .then(out => { + console.log(out); + process.exit(0); +}) + .catch(err => { + console.error(err); + process.exit(1); +}); -var _default = parse; -exports.default = _default; /***/ }), -/***/ 36: -/***/ (function(__unusedmodule, exports) { +/***/ 7351: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.issue = exports.issueCommand = void 0; +const os = __importStar(__nccwpck_require__(2037)); +const utils_1 = __nccwpck_require__(5278); /** - * Indicates whether a pattern matches a path + * Commands + * + * Command Format: + * ::name key=value,key=value::message + * + * Examples: + * ::warning::This is the message + * ::set-env name=MY_VAR::some value */ -var MatchKind; -(function (MatchKind) { - /** Not matched */ - MatchKind[MatchKind["None"] = 0] = "None"; - /** Matched if the path is a directory */ - MatchKind[MatchKind["Directory"] = 1] = "Directory"; - /** Matched if the path is a regular file */ - MatchKind[MatchKind["File"] = 2] = "File"; - /** Matched */ - MatchKind[MatchKind["All"] = 3] = "All"; -})(MatchKind = exports.MatchKind || (exports.MatchKind = {})); -//# sourceMappingURL=internal-match-kind.js.map +function issueCommand(command, properties, message) { + const cmd = new Command(command, properties, message); + process.stdout.write(cmd.toString() + os.EOL); +} +exports.issueCommand = issueCommand; +function issue(name, message = '') { + issueCommand(name, {}, message); +} +exports.issue = issue; +const CMD_STRING = '::'; +class Command { + constructor(command, properties, message) { + if (!command) { + command = 'missing.command'; + } + this.command = command; + this.properties = properties; + this.message = message; + } + toString() { + let cmdStr = CMD_STRING + this.command; + if (this.properties && Object.keys(this.properties).length > 0) { + cmdStr += ' '; + let first = true; + for (const key in this.properties) { + if (this.properties.hasOwnProperty(key)) { + const val = this.properties[key]; + if (val) { + if (first) { + first = false; + } + else { + cmdStr += ','; + } + cmdStr += `${key}=${escapeProperty(val)}`; + } + } + } + } + cmdStr += `${CMD_STRING}${escapeData(this.message)}`; + return cmdStr; + } +} +function escapeData(s) { + return utils_1.toCommandValue(s) + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A'); +} +function escapeProperty(s) { + return utils_1.toCommandValue(s) + .replace(/%/g, '%25') + .replace(/\r/g, '%0D') + .replace(/\n/g, '%0A') + .replace(/:/g, '%3A') + .replace(/,/g, '%2C'); +} +//# sourceMappingURL=command.js.map /***/ }), -/***/ 62: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 2186: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; - -Object.defineProperty(exports, "__esModule", { - value: true +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; }); -Object.defineProperty(exports, "v1", { - enumerable: true, - get: function () { - return _v.default; - } -}); -Object.defineProperty(exports, "v3", { - enumerable: true, - get: function () { - return _v2.default; - } -}); -Object.defineProperty(exports, "v4", { - enumerable: true, - get: function () { - return _v3.default; - } -}); -Object.defineProperty(exports, "v5", { - enumerable: true, - get: function () { - return _v4.default; - } -}); -Object.defineProperty(exports, "NIL", { - enumerable: true, - get: function () { - return _nil.default; - } -}); -Object.defineProperty(exports, "version", { - enumerable: true, - get: function () { - return _version.default; - } -}); -Object.defineProperty(exports, "validate", { - enumerable: true, - get: function () { - return _validate.default; - } -}); -Object.defineProperty(exports, "stringify", { - enumerable: true, - get: function () { - return _stringify.default; - } -}); -Object.defineProperty(exports, "parse", { - enumerable: true, - get: function () { - return _parse.default; - } -}); - -var _v = _interopRequireDefault(__webpack_require__(893)); - -var _v2 = _interopRequireDefault(__webpack_require__(209)); - -var _v3 = _interopRequireDefault(__webpack_require__(733)); - -var _v4 = _interopRequireDefault(__webpack_require__(384)); - -var _nil = _interopRequireDefault(__webpack_require__(327)); - -var _version = _interopRequireDefault(__webpack_require__(695)); - -var _validate = _interopRequireDefault(__webpack_require__(78)); - -var _stringify = _interopRequireDefault(__webpack_require__(411)); - -var _parse = _interopRequireDefault(__webpack_require__(22)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -/***/ }), - -/***/ 78: -/***/ (function(__unusedmodule, exports, __webpack_require__) { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; - -var _regex = _interopRequireDefault(__webpack_require__(456)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function validate(uuid) { - return typeof uuid === 'string' && _regex.default.test(uuid); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.getIDToken = exports.getState = exports.saveState = exports.group = exports.endGroup = exports.startGroup = exports.info = exports.notice = exports.warning = exports.error = exports.debug = exports.isDebug = exports.setFailed = exports.setCommandEcho = exports.setOutput = exports.getBooleanInput = exports.getMultilineInput = exports.getInput = exports.addPath = exports.setSecret = exports.exportVariable = exports.ExitCode = void 0; +const command_1 = __nccwpck_require__(7351); +const file_command_1 = __nccwpck_require__(717); +const utils_1 = __nccwpck_require__(5278); +const os = __importStar(__nccwpck_require__(2037)); +const path = __importStar(__nccwpck_require__(1017)); +const uuid_1 = __nccwpck_require__(5840); +const oidc_utils_1 = __nccwpck_require__(8041); +/** + * The code to exit an action + */ +var ExitCode; +(function (ExitCode) { + /** + * A code indicating that the action was successful + */ + ExitCode[ExitCode["Success"] = 0] = "Success"; + /** + * A code indicating that the action was a failure + */ + ExitCode[ExitCode["Failure"] = 1] = "Failure"; +})(ExitCode = exports.ExitCode || (exports.ExitCode = {})); +//----------------------------------------------------------------------- +// Variables +//----------------------------------------------------------------------- +/** + * Sets env variable for this action and future actions in the job + * @param name the name of the variable to set + * @param val the value of the variable. Non-string values will be converted to a string via JSON.stringify + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function exportVariable(name, val) { + const convertedVal = utils_1.toCommandValue(val); + process.env[name] = convertedVal; + const filePath = process.env['GITHUB_ENV'] || ''; + if (filePath) { + const delimiter = `ghadelimiter_${uuid_1.v4()}`; + // These should realistically never happen, but just in case someone finds a way to exploit uuid generation let's not allow keys or values that contain the delimiter. + if (name.includes(delimiter)) { + throw new Error(`Unexpected input: name should not contain the delimiter "${delimiter}"`); + } + if (convertedVal.includes(delimiter)) { + throw new Error(`Unexpected input: value should not contain the delimiter "${delimiter}"`); + } + const commandValue = `${name}<<${delimiter}${os.EOL}${convertedVal}${os.EOL}${delimiter}`; + file_command_1.issueCommand('ENV', commandValue); + } + else { + command_1.issueCommand('set-env', { name }, convertedVal); + } } - -var _default = validate; -exports.default = _default; - -/***/ }), - -/***/ 82: -/***/ (function(__unusedmodule, exports) { - -"use strict"; - -// We use any as a valid input type -/* eslint-disable @typescript-eslint/no-explicit-any */ -Object.defineProperty(exports, "__esModule", { value: true }); -exports.toCommandProperties = exports.toCommandValue = void 0; +exports.exportVariable = exportVariable; /** - * Sanitizes an input into a string so it can be passed into issueCommand safely - * @param input input to sanitize into a string + * Registers a secret which will get masked from logs + * @param secret value of the secret */ -function toCommandValue(input) { - if (input === null || input === undefined) { - return ''; +function setSecret(secret) { + command_1.issueCommand('add-mask', {}, secret); +} +exports.setSecret = setSecret; +/** + * Prepends inputPath to the PATH (for this action and future actions) + * @param inputPath + */ +function addPath(inputPath) { + const filePath = process.env['GITHUB_PATH'] || ''; + if (filePath) { + file_command_1.issueCommand('PATH', inputPath); } - else if (typeof input === 'string' || input instanceof String) { - return input; + else { + command_1.issueCommand('add-path', {}, inputPath); } - return JSON.stringify(input); + process.env['PATH'] = `${inputPath}${path.delimiter}${process.env['PATH']}`; } -exports.toCommandValue = toCommandValue; +exports.addPath = addPath; /** + * Gets the value of an input. + * Unless trimWhitespace is set to false in InputOptions, the value is also trimmed. + * Returns an empty string if the value is not defined. * - * @param annotationProperties - * @returns The command properties to send with the actual annotation command - * See IssueCommandProperties: https://github.com/actions/runner/blob/main/src/Runner.Worker/ActionCommandManager.cs#L646 + * @param name name of the input to get + * @param options optional. See InputOptions. + * @returns string */ -function toCommandProperties(annotationProperties) { - if (!Object.keys(annotationProperties).length) { - return {}; +function getInput(name, options) { + const val = process.env[`INPUT_${name.replace(/ /g, '_').toUpperCase()}`] || ''; + if (options && options.required && !val) { + throw new Error(`Input required and not supplied: ${name}`); } - return { - title: annotationProperties.title, - file: annotationProperties.file, - line: annotationProperties.startLine, - endLine: annotationProperties.endLine, - col: annotationProperties.startColumn, - endColumn: annotationProperties.endColumn - }; + if (options && options.trimWhitespace === false) { + return val; + } + return val.trim(); } -exports.toCommandProperties = toCommandProperties; -//# sourceMappingURL=utils.js.map - -/***/ }), - -/***/ 87: -/***/ (function(module) { - -module.exports = require("os"); - -/***/ }), - -/***/ 93: -/***/ (function(module, __unusedexports, __webpack_require__) { - -module.exports = minimatch -minimatch.Minimatch = Minimatch - -var path = { sep: '/' } -try { - path = __webpack_require__(622) -} catch (er) {} - -var GLOBSTAR = minimatch.GLOBSTAR = Minimatch.GLOBSTAR = {} -var expand = __webpack_require__(306) - -var plTypes = { - '!': { open: '(?:(?!(?:', close: '))[^/]*?)'}, - '?': { open: '(?:', close: ')?' }, - '+': { open: '(?:', close: ')+' }, - '*': { open: '(?:', close: ')*' }, - '@': { open: '(?:', close: ')' } +exports.getInput = getInput; +/** + * Gets the values of an multiline input. Each value is also trimmed. + * + * @param name name of the input to get + * @param options optional. See InputOptions. + * @returns string[] + * + */ +function getMultilineInput(name, options) { + const inputs = getInput(name, options) + .split('\n') + .filter(x => x !== ''); + return inputs; } - -// any single thing other than / -// don't need to escape / when using new RegExp() -var qmark = '[^/]' - -// * => any number of characters -var star = qmark + '*?' - -// ** when dots are allowed. Anything goes, except .. and . -// not (^ or / followed by one or two dots followed by $ or /), -// followed by anything, any number of times. -var twoStarDot = '(?:(?!(?:\\\/|^)(?:\\.{1,2})($|\\\/)).)*?' - -// not a ^ or / followed by a dot, -// followed by anything, any number of times. -var twoStarNoDot = '(?:(?!(?:\\\/|^)\\.).)*?' - -// characters that need to be escaped in RegExp. -var reSpecials = charSet('().*{}+?[]^$\\!') - -// "abc" -> { a:true, b:true, c:true } -function charSet (s) { - return s.split('').reduce(function (set, c) { - set[c] = true - return set - }, {}) +exports.getMultilineInput = getMultilineInput; +/** + * Gets the input value of the boolean type in the YAML 1.2 "core schema" specification. + * Support boolean input list: `true | True | TRUE | false | False | FALSE` . + * The return value is also in boolean type. + * ref: https://yaml.org/spec/1.2/spec.html#id2804923 + * + * @param name name of the input to get + * @param options optional. See InputOptions. + * @returns boolean + */ +function getBooleanInput(name, options) { + const trueValue = ['true', 'True', 'TRUE']; + const falseValue = ['false', 'False', 'FALSE']; + const val = getInput(name, options); + if (trueValue.includes(val)) + return true; + if (falseValue.includes(val)) + return false; + throw new TypeError(`Input does not meet YAML 1.2 "Core Schema" specification: ${name}\n` + + `Support boolean input list: \`true | True | TRUE | false | False | FALSE\``); } - -// normalizes slashes. -var slashSplit = /\/+/ - -minimatch.filter = filter -function filter (pattern, options) { - options = options || {} - return function (p, i, list) { - return minimatch(p, pattern, options) - } +exports.getBooleanInput = getBooleanInput; +/** + * Sets the value of an output. + * + * @param name name of the output to set + * @param value value to store. Non-string values will be converted to a string via JSON.stringify + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function setOutput(name, value) { + process.stdout.write(os.EOL); + command_1.issueCommand('set-output', { name }, value); } - -function ext (a, b) { - a = a || {} - b = b || {} - var t = {} - Object.keys(b).forEach(function (k) { - t[k] = b[k] - }) - Object.keys(a).forEach(function (k) { - t[k] = a[k] - }) - return t +exports.setOutput = setOutput; +/** + * Enables or disables the echoing of commands into stdout for the rest of the step. + * Echoing is disabled by default if ACTIONS_STEP_DEBUG is not set. + * + */ +function setCommandEcho(enabled) { + command_1.issue('echo', enabled ? 'on' : 'off'); } - -minimatch.defaults = function (def) { - if (!def || !Object.keys(def).length) return minimatch - - var orig = minimatch - - var m = function minimatch (p, pattern, options) { - return orig.minimatch(p, pattern, ext(def, options)) - } - - m.Minimatch = function Minimatch (pattern, options) { - return new orig.Minimatch(pattern, ext(def, options)) - } - - return m +exports.setCommandEcho = setCommandEcho; +//----------------------------------------------------------------------- +// Results +//----------------------------------------------------------------------- +/** + * Sets the action status to failed. + * When the action exits it will be with an exit code of 1 + * @param message add error issue message + */ +function setFailed(message) { + process.exitCode = ExitCode.Failure; + error(message); } - -Minimatch.defaults = function (def) { - if (!def || !Object.keys(def).length) return Minimatch - return minimatch.defaults(def).Minimatch +exports.setFailed = setFailed; +//----------------------------------------------------------------------- +// Logging Commands +//----------------------------------------------------------------------- +/** + * Gets whether Actions Step Debug is on or not + */ +function isDebug() { + return process.env['RUNNER_DEBUG'] === '1'; } - -function minimatch (p, pattern, options) { - if (typeof pattern !== 'string') { - throw new TypeError('glob pattern string required') - } - - if (!options) options = {} - - // shortcut: comments match nothing. - if (!options.nocomment && pattern.charAt(0) === '#') { - return false - } - - // "" only matches "" - if (pattern.trim() === '') return p === '' - - return new Minimatch(pattern, options).match(p) +exports.isDebug = isDebug; +/** + * Writes debug message to user log + * @param message debug message + */ +function debug(message) { + command_1.issueCommand('debug', {}, message); } +exports.debug = debug; +/** + * Adds an error issue + * @param message error issue message. Errors will be converted to string via toString() + * @param properties optional properties to add to the annotation. + */ +function error(message, properties = {}) { + command_1.issueCommand('error', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); +} +exports.error = error; +/** + * Adds a warning issue + * @param message warning issue message. Errors will be converted to string via toString() + * @param properties optional properties to add to the annotation. + */ +function warning(message, properties = {}) { + command_1.issueCommand('warning', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); +} +exports.warning = warning; +/** + * Adds a notice issue + * @param message notice issue message. Errors will be converted to string via toString() + * @param properties optional properties to add to the annotation. + */ +function notice(message, properties = {}) { + command_1.issueCommand('notice', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); +} +exports.notice = notice; +/** + * Writes info to log with console.log. + * @param message info message + */ +function info(message) { + process.stdout.write(message + os.EOL); +} +exports.info = info; +/** + * Begin an output group. + * + * Output until the next `groupEnd` will be foldable in this group + * + * @param name The name of the output group + */ +function startGroup(name) { + command_1.issue('group', name); +} +exports.startGroup = startGroup; +/** + * End an output group. + */ +function endGroup() { + command_1.issue('endgroup'); +} +exports.endGroup = endGroup; +/** + * Wrap an asynchronous function call in a group. + * + * Returns the same type as the function itself. + * + * @param name The name of the group + * @param fn The function to wrap in the group + */ +function group(name, fn) { + return __awaiter(this, void 0, void 0, function* () { + startGroup(name); + let result; + try { + result = yield fn(); + } + finally { + endGroup(); + } + return result; + }); +} +exports.group = group; +//----------------------------------------------------------------------- +// Wrapper action state +//----------------------------------------------------------------------- +/** + * Saves state for current action, the state can only be retrieved by this action's post job execution. + * + * @param name name of the state to store + * @param value value to store. Non-string values will be converted to a string via JSON.stringify + */ +// eslint-disable-next-line @typescript-eslint/no-explicit-any +function saveState(name, value) { + command_1.issueCommand('save-state', { name }, value); +} +exports.saveState = saveState; +/** + * Gets the value of an state set by this action's main execution. + * + * @param name name of the state to get + * @returns string + */ +function getState(name) { + return process.env[`STATE_${name}`] || ''; +} +exports.getState = getState; +function getIDToken(aud) { + return __awaiter(this, void 0, void 0, function* () { + return yield oidc_utils_1.OidcClient.getIDToken(aud); + }); +} +exports.getIDToken = getIDToken; +/** + * Summary exports + */ +var summary_1 = __nccwpck_require__(1327); +Object.defineProperty(exports, "summary", ({ enumerable: true, get: function () { return summary_1.summary; } })); +/** + * @deprecated use core.summary + */ +var summary_2 = __nccwpck_require__(1327); +Object.defineProperty(exports, "markdownSummary", ({ enumerable: true, get: function () { return summary_2.markdownSummary; } })); +/** + * Path exports + */ +var path_utils_1 = __nccwpck_require__(2981); +Object.defineProperty(exports, "toPosixPath", ({ enumerable: true, get: function () { return path_utils_1.toPosixPath; } })); +Object.defineProperty(exports, "toWin32Path", ({ enumerable: true, get: function () { return path_utils_1.toWin32Path; } })); +Object.defineProperty(exports, "toPlatformPath", ({ enumerable: true, get: function () { return path_utils_1.toPlatformPath; } })); +//# sourceMappingURL=core.js.map -function Minimatch (pattern, options) { - if (!(this instanceof Minimatch)) { - return new Minimatch(pattern, options) - } - - if (typeof pattern !== 'string') { - throw new TypeError('glob pattern string required') - } +/***/ }), - if (!options) options = {} - pattern = pattern.trim() +/***/ 717: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - // windows support: need to use /, not \ - if (path.sep !== '/') { - pattern = pattern.split(path.sep).join('/') - } +"use strict"; - this.options = options - this.set = [] - this.pattern = pattern - this.regexp = null - this.negate = false - this.comment = false - this.empty = false - - // make the set of regexps etc. - this.make() +// For internal use, subject to change. +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.issueCommand = void 0; +// We use any as a valid input type +/* eslint-disable @typescript-eslint/no-explicit-any */ +const fs = __importStar(__nccwpck_require__(7147)); +const os = __importStar(__nccwpck_require__(2037)); +const utils_1 = __nccwpck_require__(5278); +function issueCommand(command, message) { + const filePath = process.env[`GITHUB_${command}`]; + if (!filePath) { + throw new Error(`Unable to find environment variable for file command ${command}`); + } + if (!fs.existsSync(filePath)) { + throw new Error(`Missing file at path: ${filePath}`); + } + fs.appendFileSync(filePath, `${utils_1.toCommandValue(message)}${os.EOL}`, { + encoding: 'utf8' + }); } +exports.issueCommand = issueCommand; +//# sourceMappingURL=file-command.js.map -Minimatch.prototype.debug = function () {} - -Minimatch.prototype.make = make -function make () { - // don't do it more than once. - if (this._made) return - - var pattern = this.pattern - var options = this.options - - // empty patterns and comments match nothing. - if (!options.nocomment && pattern.charAt(0) === '#') { - this.comment = true - return - } - if (!pattern) { - this.empty = true - return - } - - // step 1: figure out negation, etc. - this.parseNegate() - - // step 2: expand braces - var set = this.globSet = this.braceExpand() - - if (options.debug) this.debug = console.error - - this.debug(this.pattern, set) - - // step 3: now we have a set, so turn each one into a series of path-portion - // matching patterns. - // These will be regexps, except in the case of "**", which is - // set to the GLOBSTAR object for globstar behavior, - // and will not contain any / characters - set = this.globParts = set.map(function (s) { - return s.split(slashSplit) - }) - - this.debug(this.pattern, set) - - // glob --> regexps - set = set.map(function (s, si, set) { - return s.map(this.parse, this) - }, this) - - this.debug(this.pattern, set) +/***/ }), - // filter out everything that didn't compile properly. - set = set.filter(function (s) { - return s.indexOf(false) === -1 - }) +/***/ 8041: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - this.debug(this.pattern, set) +"use strict"; - this.set = set +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.OidcClient = void 0; +const http_client_1 = __nccwpck_require__(6255); +const auth_1 = __nccwpck_require__(5526); +const core_1 = __nccwpck_require__(2186); +class OidcClient { + static createHttpClient(allowRetry = true, maxRetry = 10) { + const requestOptions = { + allowRetries: allowRetry, + maxRetries: maxRetry + }; + return new http_client_1.HttpClient('actions/oidc-client', [new auth_1.BearerCredentialHandler(OidcClient.getRequestToken())], requestOptions); + } + static getRequestToken() { + const token = process.env['ACTIONS_ID_TOKEN_REQUEST_TOKEN']; + if (!token) { + throw new Error('Unable to get ACTIONS_ID_TOKEN_REQUEST_TOKEN env variable'); + } + return token; + } + static getIDTokenUrl() { + const runtimeUrl = process.env['ACTIONS_ID_TOKEN_REQUEST_URL']; + if (!runtimeUrl) { + throw new Error('Unable to get ACTIONS_ID_TOKEN_REQUEST_URL env variable'); + } + return runtimeUrl; + } + static getCall(id_token_url) { + var _a; + return __awaiter(this, void 0, void 0, function* () { + const httpclient = OidcClient.createHttpClient(); + const res = yield httpclient + .getJson(id_token_url) + .catch(error => { + throw new Error(`Failed to get ID Token. \n + Error Code : ${error.statusCode}\n + Error Message: ${error.result.message}`); + }); + const id_token = (_a = res.result) === null || _a === void 0 ? void 0 : _a.value; + if (!id_token) { + throw new Error('Response json body do not have ID Token field'); + } + return id_token; + }); + } + static getIDToken(audience) { + return __awaiter(this, void 0, void 0, function* () { + try { + // New ID Token is requested from action service + let id_token_url = OidcClient.getIDTokenUrl(); + if (audience) { + const encodedAudience = encodeURIComponent(audience); + id_token_url = `${id_token_url}&audience=${encodedAudience}`; + } + core_1.debug(`ID token url is ${id_token_url}`); + const id_token = yield OidcClient.getCall(id_token_url); + core_1.setSecret(id_token); + return id_token; + } + catch (error) { + throw new Error(`Error message: ${error.message}`); + } + }); + } } +exports.OidcClient = OidcClient; +//# sourceMappingURL=oidc-utils.js.map -Minimatch.prototype.parseNegate = parseNegate -function parseNegate () { - var pattern = this.pattern - var negate = false - var options = this.options - var negateOffset = 0 +/***/ }), - if (options.nonegate) return +/***/ 2981: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - for (var i = 0, l = pattern.length - ; i < l && pattern.charAt(i) === '!' - ; i++) { - negate = !negate - negateOffset++ - } +"use strict"; - if (negateOffset) this.pattern = pattern.substr(negateOffset) - this.negate = negate +var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); +}) : (function(o, m, k, k2) { + if (k2 === undefined) k2 = k; + o[k2] = m[k]; +})); +var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { + Object.defineProperty(o, "default", { enumerable: true, value: v }); +}) : function(o, v) { + o["default"] = v; +}); +var __importStar = (this && this.__importStar) || function (mod) { + if (mod && mod.__esModule) return mod; + var result = {}; + if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); + __setModuleDefault(result, mod); + return result; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.toPlatformPath = exports.toWin32Path = exports.toPosixPath = void 0; +const path = __importStar(__nccwpck_require__(1017)); +/** + * toPosixPath converts the given path to the posix form. On Windows, \\ will be + * replaced with /. + * + * @param pth. Path to transform. + * @return string Posix path. + */ +function toPosixPath(pth) { + return pth.replace(/[\\]/g, '/'); } - -// Brace expansion: -// a{b,c}d -> abd acd -// a{b,}c -> abc ac -// a{0..3}d -> a0d a1d a2d a3d -// a{b,c{d,e}f}g -> abg acdfg acefg -// a{b,c}d{e,f}g -> abdeg acdeg abdeg abdfg -// -// Invalid sets are not expanded. -// a{2..}b -> a{2..}b -// a{b}c -> a{b}c -minimatch.braceExpand = function (pattern, options) { - return braceExpand(pattern, options) +exports.toPosixPath = toPosixPath; +/** + * toWin32Path converts the given path to the win32 form. On Linux, / will be + * replaced with \\. + * + * @param pth. Path to transform. + * @return string Win32 path. + */ +function toWin32Path(pth) { + return pth.replace(/[/]/g, '\\'); +} +exports.toWin32Path = toWin32Path; +/** + * toPlatformPath converts the given path to a platform-specific path. It does + * this by replacing instances of / and \ with the platform-specific path + * separator. + * + * @param pth The path to platformize. + * @return string The platform-specific path. + */ +function toPlatformPath(pth) { + return pth.replace(/[/\\]/g, path.sep); } +exports.toPlatformPath = toPlatformPath; +//# sourceMappingURL=path-utils.js.map -Minimatch.prototype.braceExpand = braceExpand +/***/ }), -function braceExpand (pattern, options) { - if (!options) { - if (this instanceof Minimatch) { - options = this.options - } else { - options = {} - } - } - - pattern = typeof pattern === 'undefined' - ? this.pattern : pattern - - if (typeof pattern === 'undefined') { - throw new TypeError('undefined pattern') - } - - if (options.nobrace || - !pattern.match(/\{.*\}/)) { - // shortcut. no need to expand. - return [pattern] - } - - return expand(pattern) -} - -// parse a component of the expanded set. -// At this point, no pattern may contain "/" in it -// so we're going to return a 2d array, where each entry is the full -// pattern, split on '/', and then turned into a regular expression. -// A regexp is made at the end which joins each array with an -// escaped /, and another full one which joins each regexp with |. -// -// Following the lead of Bash 4.1, note that "**" only has special meaning -// when it is the *only* thing in a path portion. Otherwise, any series -// of * is equivalent to a single *. Globstar behavior is enabled by -// default, and can be disabled by setting options.noglobstar. -Minimatch.prototype.parse = parse -var SUBPARSE = {} -function parse (pattern, isSub) { - if (pattern.length > 1024 * 64) { - throw new TypeError('pattern is too long') - } - - var options = this.options - - // shortcuts - if (!options.noglobstar && pattern === '**') return GLOBSTAR - if (pattern === '') return '' - - var re = '' - var hasMagic = !!options.nocase - var escaping = false - // ? => one single character - var patternListStack = [] - var negativeLists = [] - var stateChar - var inClass = false - var reClassStart = -1 - var classStart = -1 - // . and .. never match anything that doesn't start with ., - // even when options.dot is set. - var patternStart = pattern.charAt(0) === '.' ? '' // anything - // not (start or / followed by . or .. followed by / or end) - : options.dot ? '(?!(?:^|\\\/)\\.{1,2}(?:$|\\\/))' - : '(?!\\.)' - var self = this - - function clearStateChar () { - if (stateChar) { - // we had some state-tracking character - // that wasn't consumed by this pass. - switch (stateChar) { - case '*': - re += star - hasMagic = true - break - case '?': - re += qmark - hasMagic = true - break - default: - re += '\\' + stateChar - break - } - self.debug('clearStateChar %j %j', stateChar, re) - stateChar = false - } - } - - for (var i = 0, len = pattern.length, c - ; (i < len) && (c = pattern.charAt(i)) - ; i++) { - this.debug('%s\t%s %s %j', pattern, i, re, c) - - // skip over any that are escaped. - if (escaping && reSpecials[c]) { - re += '\\' + c - escaping = false - continue - } - - switch (c) { - case '/': - // completely not allowed, even escaped. - // Should already be path-split by now. - return false - - case '\\': - clearStateChar() - escaping = true - continue - - // the various stateChar values - // for the "extglob" stuff. - case '?': - case '*': - case '+': - case '@': - case '!': - this.debug('%s\t%s %s %j <-- stateChar', pattern, i, re, c) - - // all of those are literals inside a class, except that - // the glob [!a] means [^a] in regexp - if (inClass) { - this.debug(' in class') - if (c === '!' && i === classStart + 1) c = '^' - re += c - continue - } - - // if we already have a stateChar, then it means - // that there was something like ** or +? in there. - // Handle the stateChar, then proceed with this one. - self.debug('call clearStateChar %j', stateChar) - clearStateChar() - stateChar = c - // if extglob is disabled, then +(asdf|foo) isn't a thing. - // just clear the statechar *now*, rather than even diving into - // the patternList stuff. - if (options.noext) clearStateChar() - continue - - case '(': - if (inClass) { - re += '(' - continue - } - - if (!stateChar) { - re += '\\(' - continue - } - - patternListStack.push({ - type: stateChar, - start: i - 1, - reStart: re.length, - open: plTypes[stateChar].open, - close: plTypes[stateChar].close - }) - // negation is (?:(?!js)[^/]*) - re += stateChar === '!' ? '(?:(?!(?:' : '(?:' - this.debug('plType %j %j', stateChar, re) - stateChar = false - continue - - case ')': - if (inClass || !patternListStack.length) { - re += '\\)' - continue - } - - clearStateChar() - hasMagic = true - var pl = patternListStack.pop() - // negation is (?:(?!js)[^/]*) - // The others are (?:) - re += pl.close - if (pl.type === '!') { - negativeLists.push(pl) - } - pl.reEnd = re.length - continue - - case '|': - if (inClass || !patternListStack.length || escaping) { - re += '\\|' - escaping = false - continue - } - - clearStateChar() - re += '|' - continue - - // these are mostly the same in regexp and glob - case '[': - // swallow any state-tracking char before the [ - clearStateChar() - - if (inClass) { - re += '\\' + c - continue - } - - inClass = true - classStart = i - reClassStart = re.length - re += c - continue - - case ']': - // a right bracket shall lose its special - // meaning and represent itself in - // a bracket expression if it occurs - // first in the list. -- POSIX.2 2.8.3.2 - if (i === classStart + 1 || !inClass) { - re += '\\' + c - escaping = false - continue - } - - // handle the case where we left a class open. - // "[z-a]" is valid, equivalent to "\[z-a\]" - if (inClass) { - // split where the last [ was, make sure we don't have - // an invalid re. if so, re-walk the contents of the - // would-be class to re-translate any characters that - // were passed through as-is - // TODO: It would probably be faster to determine this - // without a try/catch and a new RegExp, but it's tricky - // to do safely. For now, this is safe and works. - var cs = pattern.substring(classStart + 1, i) - try { - RegExp('[' + cs + ']') - } catch (er) { - // not a valid class! - var sp = this.parse(cs, SUBPARSE) - re = re.substr(0, reClassStart) + '\\[' + sp[0] + '\\]' - hasMagic = hasMagic || sp[1] - inClass = false - continue - } - } - - // finish up the class. - hasMagic = true - inClass = false - re += c - continue - - default: - // swallow any state char that wasn't consumed - clearStateChar() - - if (escaping) { - // no need - escaping = false - } else if (reSpecials[c] - && !(c === '^' && inClass)) { - re += '\\' - } - - re += c - - } // switch - } // for - - // handle the case where we left a class open. - // "[abc" is valid, equivalent to "\[abc" - if (inClass) { - // split where the last [ was, and escape it - // this is a huge pita. We now have to re-walk - // the contents of the would-be class to re-translate - // any characters that were passed through as-is - cs = pattern.substr(classStart + 1) - sp = this.parse(cs, SUBPARSE) - re = re.substr(0, reClassStart) + '\\[' + sp[0] - hasMagic = hasMagic || sp[1] - } - - // handle the case where we had a +( thing at the *end* - // of the pattern. - // each pattern list stack adds 3 chars, and we need to go through - // and escape any | chars that were passed through as-is for the regexp. - // Go through and escape them, taking care not to double-escape any - // | chars that were already escaped. - for (pl = patternListStack.pop(); pl; pl = patternListStack.pop()) { - var tail = re.slice(pl.reStart + pl.open.length) - this.debug('setting tail', re, pl) - // maybe some even number of \, then maybe 1 \, followed by a | - tail = tail.replace(/((?:\\{2}){0,64})(\\?)\|/g, function (_, $1, $2) { - if (!$2) { - // the | isn't already escaped, so escape it. - $2 = '\\' - } - - // need to escape all those slashes *again*, without escaping the - // one that we need for escaping the | character. As it works out, - // escaping an even number of slashes can be done by simply repeating - // it exactly after itself. That's why this trick works. - // - // I am sorry that you have to see this. - return $1 + $1 + $2 + '|' - }) - - this.debug('tail=%j\n %s', tail, tail, pl, re) - var t = pl.type === '*' ? star - : pl.type === '?' ? qmark - : '\\' + pl.type - - hasMagic = true - re = re.slice(0, pl.reStart) + t + '\\(' + tail - } - - // handle trailing things that only matter at the very end. - clearStateChar() - if (escaping) { - // trailing \\ - re += '\\\\' - } - - // only need to apply the nodot start if the re starts with - // something that could conceivably capture a dot - var addPatternStart = false - switch (re.charAt(0)) { - case '.': - case '[': - case '(': addPatternStart = true - } - - // Hack to work around lack of negative lookbehind in JS - // A pattern like: *.!(x).!(y|z) needs to ensure that a name - // like 'a.xyz.yz' doesn't match. So, the first negative - // lookahead, has to look ALL the way ahead, to the end of - // the pattern. - for (var n = negativeLists.length - 1; n > -1; n--) { - var nl = negativeLists[n] - - var nlBefore = re.slice(0, nl.reStart) - var nlFirst = re.slice(nl.reStart, nl.reEnd - 8) - var nlLast = re.slice(nl.reEnd - 8, nl.reEnd) - var nlAfter = re.slice(nl.reEnd) - - nlLast += nlAfter - - // Handle nested stuff like *(*.js|!(*.json)), where open parens - // mean that we should *not* include the ) in the bit that is considered - // "after" the negated section. - var openParensBefore = nlBefore.split('(').length - 1 - var cleanAfter = nlAfter - for (i = 0; i < openParensBefore; i++) { - cleanAfter = cleanAfter.replace(/\)[+*?]?/, '') - } - nlAfter = cleanAfter - - var dollar = '' - if (nlAfter === '' && isSub !== SUBPARSE) { - dollar = '$' - } - var newRe = nlBefore + nlFirst + nlAfter + dollar + nlLast - re = newRe - } - - // if the re is not "" at this point, then we need to make sure - // it doesn't match against an empty path part. - // Otherwise a/* will match a/, which it should not. - if (re !== '' && hasMagic) { - re = '(?=.)' + re - } - - if (addPatternStart) { - re = patternStart + re - } - - // parsing just a piece of a larger pattern. - if (isSub === SUBPARSE) { - return [re, hasMagic] - } - - // skip the regexp for non-magical patterns - // unescape anything in it, though, so that it'll be - // an exact match against a file etc. - if (!hasMagic) { - return globUnescape(pattern) - } - - var flags = options.nocase ? 'i' : '' - try { - var regExp = new RegExp('^' + re + '$', flags) - } catch (er) { - // If it was an invalid regular expression, then it can't match - // anything. This trick looks for a character after the end of - // the string, which is of course impossible, except in multi-line - // mode, but it's not a /m regex. - return new RegExp('$.') - } - - regExp._glob = pattern - regExp._src = re - - return regExp -} - -minimatch.makeRe = function (pattern, options) { - return new Minimatch(pattern, options || {}).makeRe() -} - -Minimatch.prototype.makeRe = makeRe -function makeRe () { - if (this.regexp || this.regexp === false) return this.regexp - - // at this point, this.set is a 2d array of partial - // pattern strings, or "**". - // - // It's better to use .match(). This function shouldn't - // be used, really, but it's pretty convenient sometimes, - // when you just want to work with a regex. - var set = this.set - - if (!set.length) { - this.regexp = false - return this.regexp - } - var options = this.options - - var twoStar = options.noglobstar ? star - : options.dot ? twoStarDot - : twoStarNoDot - var flags = options.nocase ? 'i' : '' - - var re = set.map(function (pattern) { - return pattern.map(function (p) { - return (p === GLOBSTAR) ? twoStar - : (typeof p === 'string') ? regExpEscape(p) - : p._src - }).join('\\\/') - }).join('|') - - // must match entire pattern - // ending in a * or ** will make it less strict. - re = '^(?:' + re + ')$' - - // can match anything, as long as it's not this. - if (this.negate) re = '^(?!' + re + ').*$' - - try { - this.regexp = new RegExp(re, flags) - } catch (ex) { - this.regexp = false - } - return this.regexp -} - -minimatch.match = function (list, pattern, options) { - options = options || {} - var mm = new Minimatch(pattern, options) - list = list.filter(function (f) { - return mm.match(f) - }) - if (mm.options.nonull && !list.length) { - list.push(pattern) - } - return list -} - -Minimatch.prototype.match = match -function match (f, partial) { - this.debug('match', f, this.pattern) - // short-circuit in the case of busted things. - // comments, etc. - if (this.comment) return false - if (this.empty) return f === '' - - if (f === '/' && partial) return true - - var options = this.options - - // windows: need to use /, not \ - if (path.sep !== '/') { - f = f.split(path.sep).join('/') - } - - // treat the test path as a set of pathparts. - f = f.split(slashSplit) - this.debug(this.pattern, 'split', f) - - // just ONE of the pattern sets in this.set needs to match - // in order for it to be valid. If negating, then just one - // match means that we have failed. - // Either way, return on the first hit. - - var set = this.set - this.debug(this.pattern, 'set', set) - - // Find the basename of the path by looking for the last non-empty segment - var filename - var i - for (i = f.length - 1; i >= 0; i--) { - filename = f[i] - if (filename) break - } - - for (i = 0; i < set.length; i++) { - var pattern = set[i] - var file = f - if (options.matchBase && pattern.length === 1) { - file = [filename] - } - var hit = this.matchOne(file, pattern, partial) - if (hit) { - if (options.flipNegate) return true - return !this.negate - } - } - - // didn't get any hits. this is success if it's a negative - // pattern, failure otherwise. - if (options.flipNegate) return false - return this.negate -} - -// set partial to true to test if, for example, -// "/a/b" matches the start of "/*/b/*/d" -// Partial means, if you run out of file before you run -// out of pattern, then that's fine, as long as all -// the parts match. -Minimatch.prototype.matchOne = function (file, pattern, partial) { - var options = this.options - - this.debug('matchOne', - { 'this': this, file: file, pattern: pattern }) - - this.debug('matchOne', file.length, pattern.length) - - for (var fi = 0, - pi = 0, - fl = file.length, - pl = pattern.length - ; (fi < fl) && (pi < pl) - ; fi++, pi++) { - this.debug('matchOne loop') - var p = pattern[pi] - var f = file[fi] - - this.debug(pattern, p, f) - - // should be impossible. - // some invalid regexp stuff in the set. - if (p === false) return false - - if (p === GLOBSTAR) { - this.debug('GLOBSTAR', [pattern, p, f]) - - // "**" - // a/**/b/**/c would match the following: - // a/b/x/y/z/c - // a/x/y/z/b/c - // a/b/x/b/x/c - // a/b/c - // To do this, take the rest of the pattern after - // the **, and see if it would match the file remainder. - // If so, return success. - // If not, the ** "swallows" a segment, and try again. - // This is recursively awful. - // - // a/**/b/**/c matching a/b/x/y/z/c - // - a matches a - // - doublestar - // - matchOne(b/x/y/z/c, b/**/c) - // - b matches b - // - doublestar - // - matchOne(x/y/z/c, c) -> no - // - matchOne(y/z/c, c) -> no - // - matchOne(z/c, c) -> no - // - matchOne(c, c) yes, hit - var fr = fi - var pr = pi + 1 - if (pr === pl) { - this.debug('** at the end') - // a ** at the end will just swallow the rest. - // We have found a match. - // however, it will not swallow /.x, unless - // options.dot is set. - // . and .. are *never* matched by **, for explosively - // exponential reasons. - for (; fi < fl; fi++) { - if (file[fi] === '.' || file[fi] === '..' || - (!options.dot && file[fi].charAt(0) === '.')) return false - } - return true - } - - // ok, let's see if we can swallow whatever we can. - while (fr < fl) { - var swallowee = file[fr] - - this.debug('\nglobstar while', file, fr, pattern, pr, swallowee) - - // XXX remove this slice. Just pass the start index. - if (this.matchOne(file.slice(fr), pattern.slice(pr), partial)) { - this.debug('globstar found match!', fr, fl, swallowee) - // found a match. - return true - } else { - // can't swallow "." or ".." ever. - // can only swallow ".foo" when explicitly asked. - if (swallowee === '.' || swallowee === '..' || - (!options.dot && swallowee.charAt(0) === '.')) { - this.debug('dot detected!', file, fr, pattern, pr) - break - } - - // ** swallows a segment, and continue. - this.debug('globstar swallow a segment, and continue') - fr++ - } - } - - // no match was found. - // However, in partial mode, we can't say this is necessarily over. - // If there's more *pattern* left, then - if (partial) { - // ran out of file - this.debug('\n>>> no match, partial?', file, fr, pattern, pr) - if (fr === fl) return true - } - return false - } - - // something other than ** - // non-magic patterns just have to match exactly - // patterns with magic have been turned into regexps. - var hit - if (typeof p === 'string') { - if (options.nocase) { - hit = f.toLowerCase() === p.toLowerCase() - } else { - hit = f === p - } - this.debug('string match', p, f, hit) - } else { - hit = f.match(p) - this.debug('pattern match', p, f, hit) - } - - if (!hit) return false - } - - // Note: ending in / means that we'll get a final "" - // at the end of the pattern. This can only match a - // corresponding "" at the end of the file. - // If the file ends in /, then it can only match a - // a pattern that ends in /, unless the pattern just - // doesn't have any more for it. But, a/b/ should *not* - // match "a/b/*", even though "" matches against the - // [^/]*? pattern, except in partial mode, where it might - // simply not be reached yet. - // However, a/b/ should still satisfy a/* - - // now either we fell off the end of the pattern, or we're done. - if (fi === fl && pi === pl) { - // ran out of pattern and filename at the same time. - // an exact hit! - return true - } else if (fi === fl) { - // ran out of file, but still had pattern left. - // this is ok if we're doing the match as part of - // a glob fs traversal. - return partial - } else if (pi === pl) { - // ran out of pattern, still have file left. - // this is only acceptable if we're on the very last - // empty segment of a file with a trailing slash. - // a/* should match a/b/ - var emptyFileEnd = (fi === fl - 1) && (file[fi] === '') - return emptyFileEnd - } - - // should be unreachable. - throw new Error('wtf?') -} - -// replace stuff like \* with * -function globUnescape (s) { - return s.replace(/\\(.)/g, '$1') -} - -function regExpEscape (s) { - return s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') -} - - -/***/ }), - -/***/ 102: -/***/ (function(__unusedmodule, exports, __webpack_require__) { - -"use strict"; - -// For internal use, subject to change. -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.issueCommand = void 0; -// We use any as a valid input type -/* eslint-disable @typescript-eslint/no-explicit-any */ -const fs = __importStar(__webpack_require__(747)); -const os = __importStar(__webpack_require__(87)); -const utils_1 = __webpack_require__(82); -function issueCommand(command, message) { - const filePath = process.env[`GITHUB_${command}`]; - if (!filePath) { - throw new Error(`Unable to find environment variable for file command ${command}`); - } - if (!fs.existsSync(filePath)) { - throw new Error(`Missing file at path: ${filePath}`); - } - fs.appendFileSync(filePath, `${utils_1.toCommandValue(message)}${os.EOL}`, { - encoding: 'utf8' - }); -} -exports.issueCommand = issueCommand; -//# sourceMappingURL=file-command.js.map - -/***/ }), - -/***/ 141: -/***/ (function(__unusedmodule, exports, __webpack_require__) { - -"use strict"; - - -var net = __webpack_require__(631); -var tls = __webpack_require__(16); -var http = __webpack_require__(605); -var https = __webpack_require__(211); -var events = __webpack_require__(614); -var assert = __webpack_require__(357); -var util = __webpack_require__(669); - - -exports.httpOverHttp = httpOverHttp; -exports.httpsOverHttp = httpsOverHttp; -exports.httpOverHttps = httpOverHttps; -exports.httpsOverHttps = httpsOverHttps; - - -function httpOverHttp(options) { - var agent = new TunnelingAgent(options); - agent.request = http.request; - return agent; -} - -function httpsOverHttp(options) { - var agent = new TunnelingAgent(options); - agent.request = http.request; - agent.createSocket = createSecureSocket; - agent.defaultPort = 443; - return agent; -} - -function httpOverHttps(options) { - var agent = new TunnelingAgent(options); - agent.request = https.request; - return agent; -} - -function httpsOverHttps(options) { - var agent = new TunnelingAgent(options); - agent.request = https.request; - agent.createSocket = createSecureSocket; - agent.defaultPort = 443; - return agent; -} - - -function TunnelingAgent(options) { - var self = this; - self.options = options || {}; - self.proxyOptions = self.options.proxy || {}; - self.maxSockets = self.options.maxSockets || http.Agent.defaultMaxSockets; - self.requests = []; - self.sockets = []; - - self.on('free', function onFree(socket, host, port, localAddress) { - var options = toOptions(host, port, localAddress); - for (var i = 0, len = self.requests.length; i < len; ++i) { - var pending = self.requests[i]; - if (pending.host === options.host && pending.port === options.port) { - // Detect the request to connect same origin server, - // reuse the connection. - self.requests.splice(i, 1); - pending.request.onSocket(socket); - return; - } - } - socket.destroy(); - self.removeSocket(socket); - }); -} -util.inherits(TunnelingAgent, events.EventEmitter); - -TunnelingAgent.prototype.addRequest = function addRequest(req, host, port, localAddress) { - var self = this; - var options = mergeOptions({request: req}, self.options, toOptions(host, port, localAddress)); - - if (self.sockets.length >= this.maxSockets) { - // We are over limit so we'll add it to the queue. - self.requests.push(options); - return; - } - - // If we are under maxSockets create a new one. - self.createSocket(options, function(socket) { - socket.on('free', onFree); - socket.on('close', onCloseOrRemove); - socket.on('agentRemove', onCloseOrRemove); - req.onSocket(socket); - - function onFree() { - self.emit('free', socket, options); - } - - function onCloseOrRemove(err) { - self.removeSocket(socket); - socket.removeListener('free', onFree); - socket.removeListener('close', onCloseOrRemove); - socket.removeListener('agentRemove', onCloseOrRemove); - } - }); -}; - -TunnelingAgent.prototype.createSocket = function createSocket(options, cb) { - var self = this; - var placeholder = {}; - self.sockets.push(placeholder); - - var connectOptions = mergeOptions({}, self.proxyOptions, { - method: 'CONNECT', - path: options.host + ':' + options.port, - agent: false, - headers: { - host: options.host + ':' + options.port - } - }); - if (options.localAddress) { - connectOptions.localAddress = options.localAddress; - } - if (connectOptions.proxyAuth) { - connectOptions.headers = connectOptions.headers || {}; - connectOptions.headers['Proxy-Authorization'] = 'Basic ' + - new Buffer(connectOptions.proxyAuth).toString('base64'); - } - - debug('making CONNECT request'); - var connectReq = self.request(connectOptions); - connectReq.useChunkedEncodingByDefault = false; // for v0.6 - connectReq.once('response', onResponse); // for v0.6 - connectReq.once('upgrade', onUpgrade); // for v0.6 - connectReq.once('connect', onConnect); // for v0.7 or later - connectReq.once('error', onError); - connectReq.end(); - - function onResponse(res) { - // Very hacky. This is necessary to avoid http-parser leaks. - res.upgrade = true; - } - - function onUpgrade(res, socket, head) { - // Hacky. - process.nextTick(function() { - onConnect(res, socket, head); - }); - } - - function onConnect(res, socket, head) { - connectReq.removeAllListeners(); - socket.removeAllListeners(); - - if (res.statusCode !== 200) { - debug('tunneling socket could not be established, statusCode=%d', - res.statusCode); - socket.destroy(); - var error = new Error('tunneling socket could not be established, ' + - 'statusCode=' + res.statusCode); - error.code = 'ECONNRESET'; - options.request.emit('error', error); - self.removeSocket(placeholder); - return; - } - if (head.length > 0) { - debug('got illegal response body from proxy'); - socket.destroy(); - var error = new Error('got illegal response body from proxy'); - error.code = 'ECONNRESET'; - options.request.emit('error', error); - self.removeSocket(placeholder); - return; - } - debug('tunneling connection has established'); - self.sockets[self.sockets.indexOf(placeholder)] = socket; - return cb(socket); - } - - function onError(cause) { - connectReq.removeAllListeners(); - - debug('tunneling socket could not be established, cause=%s\n', - cause.message, cause.stack); - var error = new Error('tunneling socket could not be established, ' + - 'cause=' + cause.message); - error.code = 'ECONNRESET'; - options.request.emit('error', error); - self.removeSocket(placeholder); - } -}; - -TunnelingAgent.prototype.removeSocket = function removeSocket(socket) { - var pos = this.sockets.indexOf(socket) - if (pos === -1) { - return; - } - this.sockets.splice(pos, 1); +/***/ 1327: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { - var pending = this.requests.shift(); - if (pending) { - // If we have pending requests and a socket gets closed a new one - // needs to be created to take over in the pool for the one that closed. - this.createSocket(pending, function(socket) { - pending.request.onSocket(socket); - }); - } -}; +"use strict"; -function createSecureSocket(options, cb) { - var self = this; - TunnelingAgent.prototype.createSocket.call(self, options, function(socket) { - var hostHeader = options.request.getHeader('host'); - var tlsOptions = mergeOptions({}, self.options, { - socket: socket, - servername: hostHeader ? hostHeader.replace(/:.*$/, '') : options.host +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); }); - - // 0 is dummy port for v0.6 - var secureSocket = tls.connect(0, tlsOptions); - self.sockets[self.sockets.indexOf(socket)] = secureSocket; - cb(secureSocket); - }); -} - - -function toOptions(host, port, localAddress) { - if (typeof host === 'string') { // since v0.10 - return { - host: host, - port: port, - localAddress: localAddress - }; - } - return host; // for v0.11 or later -} - -function mergeOptions(target) { - for (var i = 1, len = arguments.length; i < len; ++i) { - var overrides = arguments[i]; - if (typeof overrides === 'object') { - var keys = Object.keys(overrides); - for (var j = 0, keyLen = keys.length; j < keyLen; ++j) { - var k = keys[j]; - if (overrides[k] !== undefined) { - target[k] = overrides[k]; +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.summary = exports.markdownSummary = exports.SUMMARY_DOCS_URL = exports.SUMMARY_ENV_VAR = void 0; +const os_1 = __nccwpck_require__(2037); +const fs_1 = __nccwpck_require__(7147); +const { access, appendFile, writeFile } = fs_1.promises; +exports.SUMMARY_ENV_VAR = 'GITHUB_STEP_SUMMARY'; +exports.SUMMARY_DOCS_URL = 'https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary'; +class Summary { + constructor() { + this._buffer = ''; + } + /** + * Finds the summary file path from the environment, rejects if env var is not found or file does not exist + * Also checks r/w permissions. + * + * @returns step summary file path + */ + filePath() { + return __awaiter(this, void 0, void 0, function* () { + if (this._filePath) { + return this._filePath; + } + const pathFromEnv = process.env[exports.SUMMARY_ENV_VAR]; + if (!pathFromEnv) { + throw new Error(`Unable to find environment variable for $${exports.SUMMARY_ENV_VAR}. Check if your runtime environment supports job summaries.`); + } + try { + yield access(pathFromEnv, fs_1.constants.R_OK | fs_1.constants.W_OK); + } + catch (_a) { + throw new Error(`Unable to access summary file: '${pathFromEnv}'. Check if the file has correct read/write permissions.`); + } + this._filePath = pathFromEnv; + return this._filePath; + }); + } + /** + * Wraps content in an HTML tag, adding any HTML attributes + * + * @param {string} tag HTML tag to wrap + * @param {string | null} content content within the tag + * @param {[attribute: string]: string} attrs key-value list of HTML attributes to add + * + * @returns {string} content wrapped in HTML element + */ + wrap(tag, content, attrs = {}) { + const htmlAttrs = Object.entries(attrs) + .map(([key, value]) => ` ${key}="${value}"`) + .join(''); + if (!content) { + return `<${tag}${htmlAttrs}>`; } - } + return `<${tag}${htmlAttrs}>${content}`; } - } - return target; -} - - -var debug; -if (process.env.NODE_DEBUG && /\btunnel\b/.test(process.env.NODE_DEBUG)) { - debug = function() { - var args = Array.prototype.slice.call(arguments); - if (typeof args[0] === 'string') { - args[0] = 'TUNNEL: ' + args[0]; - } else { - args.unshift('TUNNEL:'); + /** + * Writes text in the buffer to the summary buffer file and empties buffer. Will append by default. + * + * @param {SummaryWriteOptions} [options] (optional) options for write operation + * + * @returns {Promise} summary instance + */ + write(options) { + return __awaiter(this, void 0, void 0, function* () { + const overwrite = !!(options === null || options === void 0 ? void 0 : options.overwrite); + const filePath = yield this.filePath(); + const writeFunc = overwrite ? writeFile : appendFile; + yield writeFunc(filePath, this._buffer, { encoding: 'utf8' }); + return this.emptyBuffer(); + }); } - console.error.apply(console, args); - } -} else { - debug = function() {}; -} -exports.debug = debug; // for test - - -/***/ }), - -/***/ 177: -/***/ (function(__unusedmodule, exports) { - -"use strict"; - -Object.defineProperty(exports, "__esModule", { value: true }); -exports.checkBypass = exports.getProxyUrl = void 0; -function getProxyUrl(reqUrl) { - const usingSsl = reqUrl.protocol === 'https:'; - if (checkBypass(reqUrl)) { - return undefined; + /** + * Clears the summary buffer and wipes the summary file + * + * @returns {Summary} summary instance + */ + clear() { + return __awaiter(this, void 0, void 0, function* () { + return this.emptyBuffer().write({ overwrite: true }); + }); } - const proxyVar = (() => { - if (usingSsl) { - return process.env['https_proxy'] || process.env['HTTPS_PROXY']; - } - else { - return process.env['http_proxy'] || process.env['HTTP_PROXY']; - } - })(); - if (proxyVar) { - return new URL(proxyVar); + /** + * Returns the current summary buffer as a string + * + * @returns {string} string of summary buffer + */ + stringify() { + return this._buffer; } - else { - return undefined; + /** + * If the summary buffer is empty + * + * @returns {boolen} true if the buffer is empty + */ + isEmptyBuffer() { + return this._buffer.length === 0; + } + /** + * Resets the summary buffer without writing to summary file + * + * @returns {Summary} summary instance + */ + emptyBuffer() { + this._buffer = ''; + return this; + } + /** + * Adds raw text to the summary buffer + * + * @param {string} text content to add + * @param {boolean} [addEOL=false] (optional) append an EOL to the raw text (default: false) + * + * @returns {Summary} summary instance + */ + addRaw(text, addEOL = false) { + this._buffer += text; + return addEOL ? this.addEOL() : this; + } + /** + * Adds the operating system-specific end-of-line marker to the buffer + * + * @returns {Summary} summary instance + */ + addEOL() { + return this.addRaw(os_1.EOL); + } + /** + * Adds an HTML codeblock to the summary buffer + * + * @param {string} code content to render within fenced code block + * @param {string} lang (optional) language to syntax highlight code + * + * @returns {Summary} summary instance + */ + addCodeBlock(code, lang) { + const attrs = Object.assign({}, (lang && { lang })); + const element = this.wrap('pre', this.wrap('code', code), attrs); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML list to the summary buffer + * + * @param {string[]} items list of items to render + * @param {boolean} [ordered=false] (optional) if the rendered list should be ordered or not (default: false) + * + * @returns {Summary} summary instance + */ + addList(items, ordered = false) { + const tag = ordered ? 'ol' : 'ul'; + const listItems = items.map(item => this.wrap('li', item)).join(''); + const element = this.wrap(tag, listItems); + return this.addRaw(element).addEOL(); + } + /** + * Adds an HTML table to the summary buffer + * + * @param {SummaryTableCell[]} rows table rows + * + * @returns {Summary} summary instance + */ + addTable(rows) { + const tableBody = rows + .map(row => { + const cells = row + .map(cell => { + if (typeof cell === 'string') { + return this.wrap('td', cell); + } + const { header, data, colspan, rowspan } = cell; + const tag = header ? 'th' : 'td'; + const attrs = Object.assign(Object.assign({}, (colspan && { colspan })), (rowspan && { rowspan })); + return this.wrap(tag, data, attrs); + }) + .join(''); + return this.wrap('tr', cells); + }) + .join(''); + const element = this.wrap('table', tableBody); + return this.addRaw(element).addEOL(); } -} -exports.getProxyUrl = getProxyUrl; -function checkBypass(reqUrl) { - if (!reqUrl.hostname) { - return false; + /** + * Adds a collapsable HTML details element to the summary buffer + * + * @param {string} label text for the closed state + * @param {string} content collapsable content + * + * @returns {Summary} summary instance + */ + addDetails(label, content) { + const element = this.wrap('details', this.wrap('summary', label) + content); + return this.addRaw(element).addEOL(); } - const noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; - if (!noProxy) { - return false; + /** + * Adds an HTML image tag to the summary buffer + * + * @param {string} src path to the image you to embed + * @param {string} alt text description of the image + * @param {SummaryImageOptions} options (optional) addition image attributes + * + * @returns {Summary} summary instance + */ + addImage(src, alt, options) { + const { width, height } = options || {}; + const attrs = Object.assign(Object.assign({}, (width && { width })), (height && { height })); + const element = this.wrap('img', null, Object.assign({ src, alt }, attrs)); + return this.addRaw(element).addEOL(); } - // Determine the request port - let reqPort; - if (reqUrl.port) { - reqPort = Number(reqUrl.port); + /** + * Adds an HTML section heading element + * + * @param {string} text heading text + * @param {number | string} [level=1] (optional) the heading level, default: 1 + * + * @returns {Summary} summary instance + */ + addHeading(text, level) { + const tag = `h${level}`; + const allowedTag = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tag) + ? tag + : 'h1'; + const element = this.wrap(allowedTag, text); + return this.addRaw(element).addEOL(); } - else if (reqUrl.protocol === 'http:') { - reqPort = 80; + /** + * Adds an HTML thematic break (
) to the summary buffer + * + * @returns {Summary} summary instance + */ + addSeparator() { + const element = this.wrap('hr', null); + return this.addRaw(element).addEOL(); } - else if (reqUrl.protocol === 'https:') { - reqPort = 443; + /** + * Adds an HTML line break (
) to the summary buffer + * + * @returns {Summary} summary instance + */ + addBreak() { + const element = this.wrap('br', null); + return this.addRaw(element).addEOL(); } - // Format the request hostname and hostname with port - const upperReqHosts = [reqUrl.hostname.toUpperCase()]; - if (typeof reqPort === 'number') { - upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); + /** + * Adds an HTML blockquote to the summary buffer + * + * @param {string} text quote text + * @param {string} cite (optional) citation url + * + * @returns {Summary} summary instance + */ + addQuote(text, cite) { + const attrs = Object.assign({}, (cite && { cite })); + const element = this.wrap('blockquote', text, attrs); + return this.addRaw(element).addEOL(); } - // Compare request host against noproxy - for (const upperNoProxyItem of noProxy - .split(',') - .map(x => x.trim().toUpperCase()) - .filter(x => x)) { - if (upperReqHosts.some(x => x === upperNoProxyItem)) { - return true; - } + /** + * Adds an HTML anchor tag to the summary buffer + * + * @param {string} text link text/content + * @param {string} href hyperlink + * + * @returns {Summary} summary instance + */ + addLink(text, href) { + const element = this.wrap('a', text, { href }); + return this.addRaw(element).addEOL(); } - return false; } -exports.checkBypass = checkBypass; -//# sourceMappingURL=proxy.js.map - -/***/ }), - -/***/ 209: -/***/ (function(__unusedmodule, exports, __webpack_require__) { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; - -var _v = _interopRequireDefault(__webpack_require__(212)); - -var _md = _interopRequireDefault(__webpack_require__(803)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -const v3 = (0, _v.default)('v3', 0x30, _md.default); -var _default = v3; -exports.default = _default; - -/***/ }), - -/***/ 211: -/***/ (function(module) { - -module.exports = require("https"); +const _summary = new Summary(); +/** + * @deprecated use `core.summary` + */ +exports.markdownSummary = _summary; +exports.summary = _summary; +//# sourceMappingURL=summary.js.map /***/ }), -/***/ 212: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 5278: +/***/ ((__unused_webpack_module, exports) => { "use strict"; - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = _default; -exports.URL = exports.DNS = void 0; - -var _stringify = _interopRequireDefault(__webpack_require__(411)); - -var _parse = _interopRequireDefault(__webpack_require__(22)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -function stringToBytes(str) { - str = unescape(encodeURIComponent(str)); // UTF8 escape - - const bytes = []; - - for (let i = 0; i < str.length; ++i) { - bytes.push(str.charCodeAt(i)); - } - - return bytes; -} - -const DNS = '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; -exports.DNS = DNS; -const URL = '6ba7b811-9dad-11d1-80b4-00c04fd430c8'; -exports.URL = URL; - -function _default(name, version, hashfunc) { - function generateUUID(value, namespace, buf, offset) { - if (typeof value === 'string') { - value = stringToBytes(value); +// We use any as a valid input type +/* eslint-disable @typescript-eslint/no-explicit-any */ +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.toCommandProperties = exports.toCommandValue = void 0; +/** + * Sanitizes an input into a string so it can be passed into issueCommand safely + * @param input input to sanitize into a string + */ +function toCommandValue(input) { + if (input === null || input === undefined) { + return ''; } - - if (typeof namespace === 'string') { - namespace = (0, _parse.default)(namespace); + else if (typeof input === 'string' || input instanceof String) { + return input; } - - if (namespace.length !== 16) { - throw TypeError('Namespace must be array-like (16 iterable integer values, 0-255)'); - } // Compute hash of namespace and value, Per 4.3 - // Future: Use spread syntax when supported on all platforms, e.g. `bytes = - // hashfunc([...namespace, ... value])` - - - let bytes = new Uint8Array(16 + value.length); - bytes.set(namespace); - bytes.set(value, namespace.length); - bytes = hashfunc(bytes); - bytes[6] = bytes[6] & 0x0f | version; - bytes[8] = bytes[8] & 0x3f | 0x80; - - if (buf) { - offset = offset || 0; - - for (let i = 0; i < 16; ++i) { - buf[offset + i] = bytes[i]; - } - - return buf; + return JSON.stringify(input); +} +exports.toCommandValue = toCommandValue; +/** + * + * @param annotationProperties + * @returns The command properties to send with the actual annotation command + * See IssueCommandProperties: https://github.com/actions/runner/blob/main/src/Runner.Worker/ActionCommandManager.cs#L646 + */ +function toCommandProperties(annotationProperties) { + if (!Object.keys(annotationProperties).length) { + return {}; } - - return (0, _stringify.default)(bytes); - } // Function#name is not settable on some platforms (#270) - - - try { - generateUUID.name = name; // eslint-disable-next-line no-empty - } catch (err) {} // For CommonJS default export support - - - generateUUID.DNS = DNS; - generateUUID.URL = URL; - return generateUUID; + return { + title: annotationProperties.title, + file: annotationProperties.file, + line: annotationProperties.startLine, + endLine: annotationProperties.endLine, + col: annotationProperties.startColumn, + endColumn: annotationProperties.endColumn + }; } +exports.toCommandProperties = toCommandProperties; +//# sourceMappingURL=utils.js.map /***/ }), -/***/ 281: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 8090: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; @@ -1738,8 +1099,8 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; -Object.defineProperty(exports, "__esModule", { value: true }); -const internal_globber_1 = __webpack_require__(297); +Object.defineProperty(exports, "__esModule", ({ value: true })); +const internal_globber_1 = __nccwpck_require__(8298); /** * Constructs a globber * @@ -1756,8 +1117,45 @@ exports.create = create; /***/ }), -/***/ 297: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 1026: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { + +"use strict"; + +Object.defineProperty(exports, "__esModule", ({ value: true })); +const core = __nccwpck_require__(2186); +/** + * Returns a copy with defaults filled in. + */ +function getOptions(copy) { + const result = { + followSymbolicLinks: true, + implicitDescendants: true, + omitBrokenSymbolicLinks: true + }; + if (copy) { + if (typeof copy.followSymbolicLinks === 'boolean') { + result.followSymbolicLinks = copy.followSymbolicLinks; + core.debug(`followSymbolicLinks '${result.followSymbolicLinks}'`); + } + if (typeof copy.implicitDescendants === 'boolean') { + result.implicitDescendants = copy.implicitDescendants; + core.debug(`implicitDescendants '${result.implicitDescendants}'`); + } + if (typeof copy.omitBrokenSymbolicLinks === 'boolean') { + result.omitBrokenSymbolicLinks = copy.omitBrokenSymbolicLinks; + core.debug(`omitBrokenSymbolicLinks '${result.omitBrokenSymbolicLinks}'`); + } + } + return result; +} +exports.getOptions = getOptions; +//# sourceMappingURL=internal-glob-options-helper.js.map + +/***/ }), + +/***/ 8298: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; @@ -1789,15 +1187,15 @@ var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _ar function reject(value) { resume("throw", value); } function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); } }; -Object.defineProperty(exports, "__esModule", { value: true }); -const core = __webpack_require__(470); -const fs = __webpack_require__(747); -const globOptionsHelper = __webpack_require__(601); -const path = __webpack_require__(622); -const patternHelper = __webpack_require__(597); -const internal_match_kind_1 = __webpack_require__(36); -const internal_pattern_1 = __webpack_require__(923); -const internal_search_state_1 = __webpack_require__(728); +Object.defineProperty(exports, "__esModule", ({ value: true })); +const core = __nccwpck_require__(2186); +const fs = __nccwpck_require__(7147); +const globOptionsHelper = __nccwpck_require__(1026); +const path = __nccwpck_require__(1017); +const patternHelper = __nccwpck_require__(9005); +const internal_match_kind_1 = __nccwpck_require__(1063); +const internal_pattern_1 = __nccwpck_require__(4536); +const internal_search_state_1 = __nccwpck_require__(9117); const IS_WINDOWS = process.platform === 'win32'; class DefaultGlobber { constructor(options) { @@ -1978,525 +1376,735 @@ exports.DefaultGlobber = DefaultGlobber; /***/ }), -/***/ 306: -/***/ (function(module, __unusedexports, __webpack_require__) { +/***/ 1063: +/***/ ((__unused_webpack_module, exports) => { -var concatMap = __webpack_require__(896); -var balanced = __webpack_require__(621); +"use strict"; -module.exports = expandTop; +Object.defineProperty(exports, "__esModule", ({ value: true })); +/** + * Indicates whether a pattern matches a path + */ +var MatchKind; +(function (MatchKind) { + /** Not matched */ + MatchKind[MatchKind["None"] = 0] = "None"; + /** Matched if the path is a directory */ + MatchKind[MatchKind["Directory"] = 1] = "Directory"; + /** Matched if the path is a regular file */ + MatchKind[MatchKind["File"] = 2] = "File"; + /** Matched */ + MatchKind[MatchKind["All"] = 3] = "All"; +})(MatchKind = exports.MatchKind || (exports.MatchKind = {})); +//# sourceMappingURL=internal-match-kind.js.map -var escSlash = '\0SLASH'+Math.random()+'\0'; -var escOpen = '\0OPEN'+Math.random()+'\0'; -var escClose = '\0CLOSE'+Math.random()+'\0'; -var escComma = '\0COMMA'+Math.random()+'\0'; -var escPeriod = '\0PERIOD'+Math.random()+'\0'; +/***/ }), -function numeric(str) { - return parseInt(str, 10) == str - ? parseInt(str, 10) - : str.charCodeAt(0); -} +/***/ 1849: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { -function escapeBraces(str) { - return str.split('\\\\').join(escSlash) - .split('\\{').join(escOpen) - .split('\\}').join(escClose) - .split('\\,').join(escComma) - .split('\\.').join(escPeriod); -} +"use strict"; -function unescapeBraces(str) { - return str.split(escSlash).join('\\') - .split(escOpen).join('{') - .split(escClose).join('}') - .split(escComma).join(',') - .split(escPeriod).join('.'); +Object.defineProperty(exports, "__esModule", ({ value: true })); +const assert = __nccwpck_require__(9491); +const path = __nccwpck_require__(1017); +const IS_WINDOWS = process.platform === 'win32'; +/** + * Similar to path.dirname except normalizes the path separators and slightly better handling for Windows UNC paths. + * + * For example, on Linux/macOS: + * - `/ => /` + * - `/hello => /` + * + * For example, on Windows: + * - `C:\ => C:\` + * - `C:\hello => C:\` + * - `C: => C:` + * - `C:hello => C:` + * - `\ => \` + * - `\hello => \` + * - `\\hello => \\hello` + * - `\\hello\world => \\hello\world` + */ +function dirname(p) { + // Normalize slashes and trim unnecessary trailing slash + p = safeTrimTrailingSeparator(p); + // Windows UNC root, e.g. \\hello or \\hello\world + if (IS_WINDOWS && /^\\\\[^\\]+(\\[^\\]+)?$/.test(p)) { + return p; + } + // Get dirname + let result = path.dirname(p); + // Trim trailing slash for Windows UNC root, e.g. \\hello\world\ + if (IS_WINDOWS && /^\\\\[^\\]+\\[^\\]+\\$/.test(result)) { + result = safeTrimTrailingSeparator(result); + } + return result; } - - -// Basically just str.split(","), but handling cases -// where we have nested braced sections, which should be -// treated as individual members, like {a,{b,c},d} -function parseCommaParts(str) { - if (!str) - return ['']; - - var parts = []; - var m = balanced('{', '}', str); - - if (!m) - return str.split(','); - - var pre = m.pre; - var body = m.body; - var post = m.post; - var p = pre.split(','); - - p[p.length-1] += '{' + body + '}'; - var postParts = parseCommaParts(post); - if (post.length) { - p[p.length-1] += postParts.shift(); - p.push.apply(p, postParts); - } - - parts.push.apply(parts, p); - - return parts; +exports.dirname = dirname; +/** + * Roots the path if not already rooted. On Windows, relative roots like `\` + * or `C:` are expanded based on the current working directory. + */ +function ensureAbsoluteRoot(root, itemPath) { + assert(root, `ensureAbsoluteRoot parameter 'root' must not be empty`); + assert(itemPath, `ensureAbsoluteRoot parameter 'itemPath' must not be empty`); + // Already rooted + if (hasAbsoluteRoot(itemPath)) { + return itemPath; + } + // Windows + if (IS_WINDOWS) { + // Check for itemPath like C: or C:foo + if (itemPath.match(/^[A-Z]:[^\\/]|^[A-Z]:$/i)) { + let cwd = process.cwd(); + assert(cwd.match(/^[A-Z]:\\/i), `Expected current directory to start with an absolute drive root. Actual '${cwd}'`); + // Drive letter matches cwd? Expand to cwd + if (itemPath[0].toUpperCase() === cwd[0].toUpperCase()) { + // Drive only, e.g. C: + if (itemPath.length === 2) { + // Preserve specified drive letter case (upper or lower) + return `${itemPath[0]}:\\${cwd.substr(3)}`; + } + // Drive + path, e.g. C:foo + else { + if (!cwd.endsWith('\\')) { + cwd += '\\'; + } + // Preserve specified drive letter case (upper or lower) + return `${itemPath[0]}:\\${cwd.substr(3)}${itemPath.substr(2)}`; + } + } + // Different drive + else { + return `${itemPath[0]}:\\${itemPath.substr(2)}`; + } + } + // Check for itemPath like \ or \foo + else if (normalizeSeparators(itemPath).match(/^\\$|^\\[^\\]/)) { + const cwd = process.cwd(); + assert(cwd.match(/^[A-Z]:\\/i), `Expected current directory to start with an absolute drive root. Actual '${cwd}'`); + return `${cwd[0]}:\\${itemPath.substr(1)}`; + } + } + assert(hasAbsoluteRoot(root), `ensureAbsoluteRoot parameter 'root' must have an absolute root`); + // Otherwise ensure root ends with a separator + if (root.endsWith('/') || (IS_WINDOWS && root.endsWith('\\'))) { + // Intentionally empty + } + else { + // Append separator + root += path.sep; + } + return root + itemPath; } - -function expandTop(str) { - if (!str) - return []; - - // I don't know why Bash 4.3 does this, but it does. - // Anything starting with {} will have the first two bytes preserved - // but *only* at the top level, so {},a}b will not expand to anything, - // but a{},b}c will be expanded to [a}c,abc]. - // One could argue that this is a bug in Bash, but since the goal of - // this module is to match Bash's rules, we escape a leading {} - if (str.substr(0, 2) === '{}') { - str = '\\{\\}' + str.substr(2); - } - - return expand(escapeBraces(str), true).map(unescapeBraces); +exports.ensureAbsoluteRoot = ensureAbsoluteRoot; +/** + * On Linux/macOS, true if path starts with `/`. On Windows, true for paths like: + * `\\hello\share` and `C:\hello` (and using alternate separator). + */ +function hasAbsoluteRoot(itemPath) { + assert(itemPath, `hasAbsoluteRoot parameter 'itemPath' must not be empty`); + // Normalize separators + itemPath = normalizeSeparators(itemPath); + // Windows + if (IS_WINDOWS) { + // E.g. \\hello\share or C:\hello + return itemPath.startsWith('\\\\') || /^[A-Z]:\\/i.test(itemPath); + } + // E.g. /hello + return itemPath.startsWith('/'); } - -function identity(e) { - return e; +exports.hasAbsoluteRoot = hasAbsoluteRoot; +/** + * On Linux/macOS, true if path starts with `/`. On Windows, true for paths like: + * `\`, `\hello`, `\\hello\share`, `C:`, and `C:\hello` (and using alternate separator). + */ +function hasRoot(itemPath) { + assert(itemPath, `isRooted parameter 'itemPath' must not be empty`); + // Normalize separators + itemPath = normalizeSeparators(itemPath); + // Windows + if (IS_WINDOWS) { + // E.g. \ or \hello or \\hello + // E.g. C: or C:\hello + return itemPath.startsWith('\\') || /^[A-Z]:/i.test(itemPath); + } + // E.g. /hello + return itemPath.startsWith('/'); } - -function embrace(str) { - return '{' + str + '}'; +exports.hasRoot = hasRoot; +/** + * Removes redundant slashes and converts `/` to `\` on Windows + */ +function normalizeSeparators(p) { + p = p || ''; + // Windows + if (IS_WINDOWS) { + // Convert slashes on Windows + p = p.replace(/\//g, '\\'); + // Remove redundant slashes + const isUnc = /^\\\\+[^\\]/.test(p); // e.g. \\hello + return (isUnc ? '\\' : '') + p.replace(/\\\\+/g, '\\'); // preserve leading \\ for UNC + } + // Remove redundant slashes + return p.replace(/\/\/+/g, '/'); } -function isPadded(el) { - return /^-?0\d/.test(el); +exports.normalizeSeparators = normalizeSeparators; +/** + * Normalizes the path separators and trims the trailing separator (when safe). + * For example, `/foo/ => /foo` but `/ => /` + */ +function safeTrimTrailingSeparator(p) { + // Short-circuit if empty + if (!p) { + return ''; + } + // Normalize separators + p = normalizeSeparators(p); + // No trailing slash + if (!p.endsWith(path.sep)) { + return p; + } + // Check '/' on Linux/macOS and '\' on Windows + if (p === path.sep) { + return p; + } + // On Windows check if drive root. E.g. C:\ + if (IS_WINDOWS && /^[A-Z]:\\$/i.test(p)) { + return p; + } + // Otherwise trim trailing slash + return p.substr(0, p.length - 1); } +exports.safeTrimTrailingSeparator = safeTrimTrailingSeparator; +//# sourceMappingURL=internal-path-helper.js.map -function lte(i, y) { - return i <= y; -} -function gte(i, y) { - return i >= y; -} +/***/ }), -function expand(str, isTop) { - var expansions = []; +/***/ 6836: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - var m = balanced('{', '}', str); - if (!m || /\$$/.test(m.pre)) return [str]; +"use strict"; - var isNumericSequence = /^-?\d+\.\.-?\d+(?:\.\.-?\d+)?$/.test(m.body); - var isAlphaSequence = /^[a-zA-Z]\.\.[a-zA-Z](?:\.\.-?\d+)?$/.test(m.body); - var isSequence = isNumericSequence || isAlphaSequence; - var isOptions = m.body.indexOf(',') >= 0; - if (!isSequence && !isOptions) { - // {a},b} - if (m.post.match(/,.*\}/)) { - str = m.pre + '{' + m.body + escClose + m.post; - return expand(str); +Object.defineProperty(exports, "__esModule", ({ value: true })); +const assert = __nccwpck_require__(9491); +const path = __nccwpck_require__(1017); +const pathHelper = __nccwpck_require__(1849); +const IS_WINDOWS = process.platform === 'win32'; +/** + * Helper class for parsing paths into segments + */ +class Path { + /** + * Constructs a Path + * @param itemPath Path or array of segments + */ + constructor(itemPath) { + this.segments = []; + // String + if (typeof itemPath === 'string') { + assert(itemPath, `Parameter 'itemPath' must not be empty`); + // Normalize slashes and trim unnecessary trailing slash + itemPath = pathHelper.safeTrimTrailingSeparator(itemPath); + // Not rooted + if (!pathHelper.hasRoot(itemPath)) { + this.segments = itemPath.split(path.sep); + } + // Rooted + else { + // Add all segments, while not at the root + let remaining = itemPath; + let dir = pathHelper.dirname(remaining); + while (dir !== remaining) { + // Add the segment + const basename = path.basename(remaining); + this.segments.unshift(basename); + // Truncate the last segment + remaining = dir; + dir = pathHelper.dirname(remaining); + } + // Remainder is the root + this.segments.unshift(remaining); + } + } + // Array + else { + // Must not be empty + assert(itemPath.length > 0, `Parameter 'itemPath' must not be an empty array`); + // Each segment + for (let i = 0; i < itemPath.length; i++) { + let segment = itemPath[i]; + // Must not be empty + assert(segment, `Parameter 'itemPath' must not contain any empty segments`); + // Normalize slashes + segment = pathHelper.normalizeSeparators(itemPath[i]); + // Root segment + if (i === 0 && pathHelper.hasRoot(segment)) { + segment = pathHelper.safeTrimTrailingSeparator(segment); + assert(segment === pathHelper.dirname(segment), `Parameter 'itemPath' root segment contains information for multiple segments`); + this.segments.push(segment); + } + // All other segments + else { + // Must not contain slash + assert(!segment.includes(path.sep), `Parameter 'itemPath' contains unexpected path separators`); + this.segments.push(segment); + } + } + } } - return [str]; - } - - var n; - if (isSequence) { - n = m.body.split(/\.\./); - } else { - n = parseCommaParts(m.body); - if (n.length === 1) { - // x{{a,b}}y ==> x{a}y x{b}y - n = expand(n[0], false).map(embrace); - if (n.length === 1) { - var post = m.post.length - ? expand(m.post, false) - : ['']; - return post.map(function(p) { - return m.pre + n[0] + p; - }); - } + /** + * Converts the path to it's string representation + */ + toString() { + // First segment + let result = this.segments[0]; + // All others + let skipSlash = result.endsWith(path.sep) || (IS_WINDOWS && /^[A-Z]:$/i.test(result)); + for (let i = 1; i < this.segments.length; i++) { + if (skipSlash) { + skipSlash = false; + } + else { + result += path.sep; + } + result += this.segments[i]; + } + return result; } - } +} +exports.Path = Path; +//# sourceMappingURL=internal-path.js.map - // at this point, n is the parts, and we know it's not a comma set - // with a single entry. +/***/ }), - // no need to expand pre, since it is guaranteed to be free of brace-sets - var pre = m.pre; - var post = m.post.length - ? expand(m.post, false) - : ['']; +/***/ 9005: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { - var N; +"use strict"; - if (isSequence) { - var x = numeric(n[0]); - var y = numeric(n[1]); - var width = Math.max(n[0].length, n[1].length) - var incr = n.length == 3 - ? Math.abs(numeric(n[2])) - : 1; - var test = lte; - var reverse = y < x; - if (reverse) { - incr *= -1; - test = gte; +Object.defineProperty(exports, "__esModule", ({ value: true })); +const pathHelper = __nccwpck_require__(1849); +const internal_match_kind_1 = __nccwpck_require__(1063); +const IS_WINDOWS = process.platform === 'win32'; +/** + * Given an array of patterns, returns an array of paths to search. + * Duplicates and paths under other included paths are filtered out. + */ +function getSearchPaths(patterns) { + // Ignore negate patterns + patterns = patterns.filter(x => !x.negate); + // Create a map of all search paths + const searchPathMap = {}; + for (const pattern of patterns) { + const key = IS_WINDOWS + ? pattern.searchPath.toUpperCase() + : pattern.searchPath; + searchPathMap[key] = 'candidate'; } - var pad = n.some(isPadded); - - N = []; - - for (var i = x; test(i, y); i += incr) { - var c; - if (isAlphaSequence) { - c = String.fromCharCode(i); - if (c === '\\') - c = ''; - } else { - c = String(i); - if (pad) { - var need = width - c.length; - if (need > 0) { - var z = new Array(need + 1).join('0'); - if (i < 0) - c = '-' + z + c.slice(1); - else - c = z + c; - } + const result = []; + for (const pattern of patterns) { + // Check if already included + const key = IS_WINDOWS + ? pattern.searchPath.toUpperCase() + : pattern.searchPath; + if (searchPathMap[key] === 'included') { + continue; + } + // Check for an ancestor search path + let foundAncestor = false; + let tempKey = key; + let parent = pathHelper.dirname(tempKey); + while (parent !== tempKey) { + if (searchPathMap[parent]) { + foundAncestor = true; + break; + } + tempKey = parent; + parent = pathHelper.dirname(tempKey); + } + // Include the search pattern in the result + if (!foundAncestor) { + result.push(pattern.searchPath); + searchPathMap[key] = 'included'; + } + } + return result; +} +exports.getSearchPaths = getSearchPaths; +/** + * Matches the patterns against the path + */ +function match(patterns, itemPath) { + let result = internal_match_kind_1.MatchKind.None; + for (const pattern of patterns) { + if (pattern.negate) { + result &= ~pattern.match(itemPath); + } + else { + result |= pattern.match(itemPath); } - } - N.push(c); - } - } else { - N = concatMap(n, function(el) { return expand(el, false) }); - } - - for (var j = 0; j < N.length; j++) { - for (var k = 0; k < post.length; k++) { - var expansion = pre + N[j] + post[k]; - if (!isTop || isSequence || expansion) - expansions.push(expansion); } - } - - return expansions; + return result; } - - - -/***/ }), - -/***/ 327: -/***/ (function(__unusedmodule, exports) { - -"use strict"; - - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; -var _default = '00000000-0000-0000-0000-000000000000'; -exports.default = _default; +exports.match = match; +/** + * Checks whether to descend further into the directory + */ +function partialMatch(patterns, itemPath) { + return patterns.some(x => !x.negate && x.partialMatch(itemPath)); +} +exports.partialMatch = partialMatch; +//# sourceMappingURL=internal-pattern-helper.js.map /***/ }), -/***/ 349: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 4536: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -var __asyncValues = (this && this.__asyncValues) || function (o) { - if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined."); - var m = o[Symbol.asyncIterator], i; - return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i); - function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; } - function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); } -}; -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (Object.hasOwnProperty.call(mod, k)) result[k] = mod[k]; - result["default"] = mod; - return result; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -const crypto = __importStar(__webpack_require__(417)); -const fs = __importStar(__webpack_require__(747)); -const glob = __importStar(__webpack_require__(281)); -const path = __importStar(__webpack_require__(622)); -const stream = __importStar(__webpack_require__(794)); -const util = __importStar(__webpack_require__(669)); -function run() { - var e_1, _a; - return __awaiter(this, void 0, void 0, function* () { - // arg0 -> node - // arg1 -> hashFiles.js - // env[followSymbolicLinks] = true/null - // env[patterns] -> glob patterns - let followSymbolicLinks = false; - const matchPatterns = process.env.patterns || ''; - if (process.env.followSymbolicLinks === 'true') { - console.log('Follow symbolic links'); - followSymbolicLinks = true; +Object.defineProperty(exports, "__esModule", ({ value: true })); +const assert = __nccwpck_require__(9491); +const os = __nccwpck_require__(2037); +const path = __nccwpck_require__(1017); +const pathHelper = __nccwpck_require__(1849); +const minimatch_1 = __nccwpck_require__(3973); +const internal_match_kind_1 = __nccwpck_require__(1063); +const internal_path_1 = __nccwpck_require__(6836); +const IS_WINDOWS = process.platform === 'win32'; +class Pattern { + constructor(patternOrNegate, segments) { + /** + * Indicates whether matches should be excluded from the result set + */ + this.negate = false; + // Pattern overload + let pattern; + if (typeof patternOrNegate === 'string') { + pattern = patternOrNegate.trim(); } - console.log(`Match Pattern: ${matchPatterns}`); - let hasMatch = false; - const githubWorkspace = process.cwd(); - const result = crypto.createHash('sha256'); - let count = 0; - const globber = yield glob.create(matchPatterns, { followSymbolicLinks }); - try { - for (var _b = __asyncValues(globber.globGenerator()), _c; _c = yield _b.next(), !_c.done;) { - const file = _c.value; - console.log(file); - if (!file.startsWith(`${githubWorkspace}${path.sep}`)) { - console.log(`Ignore '${file}' since it is not under GITHUB_WORKSPACE.`); - continue; - } - if (fs.statSync(file).isDirectory()) { - console.log(`Skip directory '${file}'.`); - continue; - } - const hash = crypto.createHash('sha256'); - const pipeline = util.promisify(stream.pipeline); - yield pipeline(fs.createReadStream(file), hash); - result.write(hash.digest()); - count++; - if (!hasMatch) { - hasMatch = true; - } + // Segments overload + else { + // Convert to pattern + segments = segments || []; + assert(segments.length, `Parameter 'segments' must not empty`); + const root = Pattern.getLiteral(segments[0]); + assert(root && pathHelper.hasAbsoluteRoot(root), `Parameter 'segments' first element must be a root path`); + pattern = new internal_path_1.Path(segments).toString().trim(); + if (patternOrNegate) { + pattern = `!${pattern}`; } } - catch (e_1_1) { e_1 = { error: e_1_1 }; } - finally { - try { - if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b); + // Negate + while (pattern.startsWith('!')) { + this.negate = !this.negate; + pattern = pattern.substr(1).trim(); + } + // Normalize slashes and ensures absolute root + pattern = Pattern.fixupPattern(pattern); + // Segments + this.segments = new internal_path_1.Path(pattern).segments; + // Trailing slash indicates the pattern should only match directories, not regular files + this.trailingSeparator = pathHelper + .normalizeSeparators(pattern) + .endsWith(path.sep); + pattern = pathHelper.safeTrimTrailingSeparator(pattern); + // Search path (literal path prior to the first glob segment) + let foundGlob = false; + const searchSegments = this.segments + .map(x => Pattern.getLiteral(x)) + .filter(x => !foundGlob && !(foundGlob = x === '')); + this.searchPath = new internal_path_1.Path(searchSegments).toString(); + // Root RegExp (required when determining partial match) + this.rootRegExp = new RegExp(Pattern.regExpEscape(searchSegments[0]), IS_WINDOWS ? 'i' : ''); + // Create minimatch + const minimatchOptions = { + dot: true, + nobrace: true, + nocase: IS_WINDOWS, + nocomment: true, + noext: true, + nonegate: true + }; + pattern = IS_WINDOWS ? pattern.replace(/\\/g, '/') : pattern; + this.minimatch = new minimatch_1.Minimatch(pattern, minimatchOptions); + } + /** + * Matches the pattern against the specified path + */ + match(itemPath) { + // Last segment is globstar? + if (this.segments[this.segments.length - 1] === '**') { + // Normalize slashes + itemPath = pathHelper.normalizeSeparators(itemPath); + // Append a trailing slash. Otherwise Minimatch will not match the directory immediately + // preceeding the globstar. For example, given the pattern `/foo/**`, Minimatch returns + // false for `/foo` but returns true for `/foo/`. Append a trailing slash to handle that quirk. + if (!itemPath.endsWith(path.sep)) { + // Note, this is safe because the constructor ensures the pattern has an absolute root. + // For example, formats like C: and C:foo on Windows are resolved to an aboslute root. + itemPath = `${itemPath}${path.sep}`; } - finally { if (e_1) throw e_1.error; } } - result.end(); - if (hasMatch) { - console.log(`Found ${count} files to hash.`); - console.error(`__OUTPUT__${result.digest('hex')}__OUTPUT__`); + else { + // Normalize slashes and trim unnecessary trailing slash + itemPath = pathHelper.safeTrimTrailingSeparator(itemPath); + } + // Match + if (this.minimatch.match(itemPath)) { + return this.trailingSeparator ? internal_match_kind_1.MatchKind.Directory : internal_match_kind_1.MatchKind.All; + } + return internal_match_kind_1.MatchKind.None; + } + /** + * Indicates whether the pattern may match descendants of the specified path + */ + partialMatch(itemPath) { + // Normalize slashes and trim unnecessary trailing slash + itemPath = pathHelper.safeTrimTrailingSeparator(itemPath); + // matchOne does not handle root path correctly + if (pathHelper.dirname(itemPath) === itemPath) { + return this.rootRegExp.test(itemPath); + } + return this.minimatch.matchOne(itemPath.split(IS_WINDOWS ? /\\+/ : /\/+/), this.minimatch.set[0], true); + } + /** + * Escapes glob patterns within a path + */ + static globEscape(s) { + return (IS_WINDOWS ? s : s.replace(/\\/g, '\\\\')) // escape '\' on Linux/macOS + .replace(/(\[)(?=[^/]+\])/g, '[[]') // escape '[' when ']' follows within the path segment + .replace(/\?/g, '[?]') // escape '?' + .replace(/\*/g, '[*]'); // escape '*' + } + /** + * Normalizes slashes and ensures absolute root + */ + static fixupPattern(pattern) { + // Empty + assert(pattern, 'pattern cannot be empty'); + // Must not contain `.` segment, unless first segment + // Must not contain `..` segment + const literalSegments = new internal_path_1.Path(pattern).segments.map(x => Pattern.getLiteral(x)); + assert(literalSegments.every((x, i) => (x !== '.' || i === 0) && x !== '..'), `Invalid pattern '${pattern}'. Relative pathing '.' and '..' is not allowed.`); + // Must not contain globs in root, e.g. Windows UNC path \\foo\b*r + assert(!pathHelper.hasRoot(pattern) || literalSegments[0], `Invalid pattern '${pattern}'. Root segment must not contain globs.`); + // Normalize slashes + pattern = pathHelper.normalizeSeparators(pattern); + // Replace leading `.` segment + if (pattern === '.' || pattern.startsWith(`.${path.sep}`)) { + pattern = Pattern.globEscape(process.cwd()) + pattern.substr(1); + } + // Replace leading `~` segment + else if (pattern === '~' || pattern.startsWith(`~${path.sep}`)) { + const homedir = os.homedir(); + assert(homedir, 'Unable to determine HOME directory'); + assert(pathHelper.hasAbsoluteRoot(homedir), `Expected HOME directory to be a rooted path. Actual '${homedir}'`); + pattern = Pattern.globEscape(homedir) + pattern.substr(1); + } + // Replace relative drive root, e.g. pattern is C: or C:foo + else if (IS_WINDOWS && + (pattern.match(/^[A-Z]:$/i) || pattern.match(/^[A-Z]:[^\\]/i))) { + let root = pathHelper.ensureAbsoluteRoot('C:\\dummy-root', pattern.substr(0, 2)); + if (pattern.length > 2 && !root.endsWith('\\')) { + root += '\\'; + } + pattern = Pattern.globEscape(root) + pattern.substr(2); + } + // Replace relative root, e.g. pattern is \ or \foo + else if (IS_WINDOWS && (pattern === '\\' || pattern.match(/^\\[^\\]/))) { + let root = pathHelper.ensureAbsoluteRoot('C:\\dummy-root', '\\'); + if (!root.endsWith('\\')) { + root += '\\'; + } + pattern = Pattern.globEscape(root) + pattern.substr(1); } + // Otherwise ensure absolute root else { - console.error(`__OUTPUT____OUTPUT__`); + pattern = pathHelper.ensureAbsoluteRoot(Pattern.globEscape(process.cwd()), pattern); } - }); -} -run() - .then(out => { - console.log(out); - process.exit(0); -}) - .catch(err => { - console.error(err); - process.exit(1); -}); - - -/***/ }), - -/***/ 357: -/***/ (function(module) { - -module.exports = require("assert"); - -/***/ }), - -/***/ 383: -/***/ (function(__unusedmodule, exports, __webpack_require__) { - -"use strict"; - -Object.defineProperty(exports, "__esModule", { value: true }); -const assert = __webpack_require__(357); -const path = __webpack_require__(622); -const pathHelper = __webpack_require__(972); -const IS_WINDOWS = process.platform === 'win32'; -/** - * Helper class for parsing paths into segments - */ -class Path { + return pathHelper.normalizeSeparators(pattern); + } /** - * Constructs a Path - * @param itemPath Path or array of segments + * Attempts to unescape a pattern segment to create a literal path segment. + * Otherwise returns empty string. */ - constructor(itemPath) { - this.segments = []; - // String - if (typeof itemPath === 'string') { - assert(itemPath, `Parameter 'itemPath' must not be empty`); - // Normalize slashes and trim unnecessary trailing slash - itemPath = pathHelper.safeTrimTrailingSeparator(itemPath); - // Not rooted - if (!pathHelper.hasRoot(itemPath)) { - this.segments = itemPath.split(path.sep); + static getLiteral(segment) { + let literal = ''; + for (let i = 0; i < segment.length; i++) { + const c = segment[i]; + // Escape + if (c === '\\' && !IS_WINDOWS && i + 1 < segment.length) { + literal += segment[++i]; + continue; } - // Rooted - else { - // Add all segments, while not at the root - let remaining = itemPath; - let dir = pathHelper.dirname(remaining); - while (dir !== remaining) { - // Add the segment - const basename = path.basename(remaining); - this.segments.unshift(basename); - // Truncate the last segment - remaining = dir; - dir = pathHelper.dirname(remaining); - } - // Remainder is the root - this.segments.unshift(remaining); + // Wildcard + else if (c === '*' || c === '?') { + return ''; } - } - // Array - else { - // Must not be empty - assert(itemPath.length > 0, `Parameter 'itemPath' must not be an empty array`); - // Each segment - for (let i = 0; i < itemPath.length; i++) { - let segment = itemPath[i]; - // Must not be empty - assert(segment, `Parameter 'itemPath' must not contain any empty segments`); - // Normalize slashes - segment = pathHelper.normalizeSeparators(itemPath[i]); - // Root segment - if (i === 0 && pathHelper.hasRoot(segment)) { - segment = pathHelper.safeTrimTrailingSeparator(segment); - assert(segment === pathHelper.dirname(segment), `Parameter 'itemPath' root segment contains information for multiple segments`); - this.segments.push(segment); + // Character set + else if (c === '[' && i + 1 < segment.length) { + let set = ''; + let closed = -1; + for (let i2 = i + 1; i2 < segment.length; i2++) { + const c2 = segment[i2]; + // Escape + if (c2 === '\\' && !IS_WINDOWS && i2 + 1 < segment.length) { + set += segment[++i2]; + continue; + } + // Closed + else if (c2 === ']') { + closed = i2; + break; + } + // Otherwise + else { + set += c2; + } } - // All other segments - else { - // Must not contain slash - assert(!segment.includes(path.sep), `Parameter 'itemPath' contains unexpected path separators`); - this.segments.push(segment); + // Closed? + if (closed >= 0) { + // Cannot convert + if (set.length > 1) { + return ''; + } + // Convert to literal + if (set) { + literal += set; + i = closed; + continue; + } } + // Otherwise fall thru } + // Append + literal += c; } + return literal; } /** - * Converts the path to it's string representation + * Escapes regexp special characters + * https://javascript.info/regexp-escaping */ - toString() { - // First segment - let result = this.segments[0]; - // All others - let skipSlash = result.endsWith(path.sep) || (IS_WINDOWS && /^[A-Z]:$/i.test(result)); - for (let i = 1; i < this.segments.length; i++) { - if (skipSlash) { - skipSlash = false; - } - else { - result += path.sep; - } - result += this.segments[i]; - } - return result; + static regExpEscape(s) { + return s.replace(/[[\\^$.|?*+()]/g, '\\$&'); } } -exports.Path = Path; -//# sourceMappingURL=internal-path.js.map +exports.Pattern = Pattern; +//# sourceMappingURL=internal-pattern.js.map /***/ }), -/***/ 384: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 9117: +/***/ ((__unused_webpack_module, exports) => { "use strict"; - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; - -var _v = _interopRequireDefault(__webpack_require__(212)); - -var _sha = _interopRequireDefault(__webpack_require__(498)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -const v5 = (0, _v.default)('v5', 0x50, _sha.default); -var _default = v5; -exports.default = _default; +Object.defineProperty(exports, "__esModule", ({ value: true })); +class SearchState { + constructor(path, level) { + this.path = path; + this.level = level; + } +} +exports.SearchState = SearchState; +//# sourceMappingURL=internal-search-state.js.map /***/ }), -/***/ 411: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 5526: +/***/ (function(__unused_webpack_module, exports) { "use strict"; - -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; - -var _validate = _interopRequireDefault(__webpack_require__(78)); - -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - -/** - * Convert array of 16 byte values to UUID string format of the form: - * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX - */ -const byteToHex = []; - -for (let i = 0; i < 256; ++i) { - byteToHex.push((i + 0x100).toString(16).substr(1)); +var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { + function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } + return new (P || (P = Promise))(function (resolve, reject) { + function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } + function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } + function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } + step((generator = generator.apply(thisArg, _arguments || [])).next()); + }); +}; +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.PersonalAccessTokenCredentialHandler = exports.BearerCredentialHandler = exports.BasicCredentialHandler = void 0; +class BasicCredentialHandler { + constructor(username, password) { + this.username = username; + this.password = password; + } + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString('base64')}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } } - -function stringify(arr, offset = 0) { - // Note: Be careful editing this code! It's been tuned for performance - // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 - const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + '-' + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + '-' + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + '-' + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + '-' + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase(); // Consistency check for valid UUID. If this throws, it's likely due to one - // of the following: - // - One or more input array values don't map to a hex octet (leading to - // "undefined" in the uuid) - // - Invalid input values for the RFC `version` or `variant` fields - - if (!(0, _validate.default)(uuid)) { - throw TypeError('Stringified UUID is invalid'); - } - - return uuid; +exports.BasicCredentialHandler = BasicCredentialHandler; +class BearerCredentialHandler { + constructor(token) { + this.token = token; + } + // currently implements pre-authorization + // TODO: support preAuth = false where it hooks on 401 + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Bearer ${this.token}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } +} +exports.BearerCredentialHandler = BearerCredentialHandler; +class PersonalAccessTokenCredentialHandler { + constructor(token) { + this.token = token; + } + // currently implements pre-authorization + // TODO: support preAuth = false where it hooks on 401 + prepareRequest(options) { + if (!options.headers) { + throw Error('The request has no headers'); + } + options.headers['Authorization'] = `Basic ${Buffer.from(`PAT:${this.token}`).toString('base64')}`; + } + // This handler cannot handle 401 + canHandleAuthentication() { + return false; + } + handleAuthentication() { + return __awaiter(this, void 0, void 0, function* () { + throw new Error('not implemented'); + }); + } } - -var _default = stringify; -exports.default = _default; - -/***/ }), - -/***/ 413: -/***/ (function(module, __unusedexports, __webpack_require__) { - -module.exports = __webpack_require__(141); - - -/***/ }), - -/***/ 417: -/***/ (function(module) { - -module.exports = require("crypto"); +exports.PersonalAccessTokenCredentialHandler = PersonalAccessTokenCredentialHandler; +//# sourceMappingURL=auth.js.map /***/ }), -/***/ 425: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 6255: +/***/ (function(__unused_webpack_module, exports, __nccwpck_require__) { "use strict"; @@ -2529,12 +2137,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge step((generator = generator.apply(thisArg, _arguments || [])).next()); }); }; -Object.defineProperty(exports, "__esModule", { value: true }); +Object.defineProperty(exports, "__esModule", ({ value: true })); exports.HttpClient = exports.isHttps = exports.HttpClientResponse = exports.HttpClientError = exports.getProxyUrl = exports.MediaTypes = exports.Headers = exports.HttpCodes = void 0; -const http = __importStar(__webpack_require__(605)); -const https = __importStar(__webpack_require__(211)); -const pm = __importStar(__webpack_require__(177)); -const tunnel = __importStar(__webpack_require__(413)); +const http = __importStar(__nccwpck_require__(3685)); +const https = __importStar(__nccwpck_require__(5687)); +const pm = __importStar(__nccwpck_require__(9835)); +const tunnel = __importStar(__nccwpck_require__(4294)); var HttpCodes; (function (HttpCodes) { HttpCodes[HttpCodes["OK"] = 200] = "OK"; @@ -3107,1419 +2715,1922 @@ const lowercaseKeys = (obj) => Object.keys(obj).reduce((c, k) => ((c[k.toLowerCa /***/ }), -/***/ 431: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 9835: +/***/ ((__unused_webpack_module, exports) => { "use strict"; -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.issue = exports.issueCommand = void 0; -const os = __importStar(__webpack_require__(87)); -const utils_1 = __webpack_require__(82); -/** - * Commands - * - * Command Format: - * ::name key=value,key=value::message - * - * Examples: - * ::warning::This is the message - * ::set-env name=MY_VAR::some value - */ -function issueCommand(command, properties, message) { - const cmd = new Command(command, properties, message); - process.stdout.write(cmd.toString() + os.EOL); -} -exports.issueCommand = issueCommand; -function issue(name, message = '') { - issueCommand(name, {}, message); +Object.defineProperty(exports, "__esModule", ({ value: true })); +exports.checkBypass = exports.getProxyUrl = void 0; +function getProxyUrl(reqUrl) { + const usingSsl = reqUrl.protocol === 'https:'; + if (checkBypass(reqUrl)) { + return undefined; + } + const proxyVar = (() => { + if (usingSsl) { + return process.env['https_proxy'] || process.env['HTTPS_PROXY']; + } + else { + return process.env['http_proxy'] || process.env['HTTP_PROXY']; + } + })(); + if (proxyVar) { + return new URL(proxyVar); + } + else { + return undefined; + } } -exports.issue = issue; -const CMD_STRING = '::'; -class Command { - constructor(command, properties, message) { - if (!command) { - command = 'missing.command'; +exports.getProxyUrl = getProxyUrl; +function checkBypass(reqUrl) { + if (!reqUrl.hostname) { + return false; + } + const noProxy = process.env['no_proxy'] || process.env['NO_PROXY'] || ''; + if (!noProxy) { + return false; + } + // Determine the request port + let reqPort; + if (reqUrl.port) { + reqPort = Number(reqUrl.port); + } + else if (reqUrl.protocol === 'http:') { + reqPort = 80; + } + else if (reqUrl.protocol === 'https:') { + reqPort = 443; + } + // Format the request hostname and hostname with port + const upperReqHosts = [reqUrl.hostname.toUpperCase()]; + if (typeof reqPort === 'number') { + upperReqHosts.push(`${upperReqHosts[0]}:${reqPort}`); + } + // Compare request host against noproxy + for (const upperNoProxyItem of noProxy + .split(',') + .map(x => x.trim().toUpperCase()) + .filter(x => x)) { + if (upperReqHosts.some(x => x === upperNoProxyItem)) { + return true; } - this.command = command; - this.properties = properties; - this.message = message; } - toString() { - let cmdStr = CMD_STRING + this.command; - if (this.properties && Object.keys(this.properties).length > 0) { - cmdStr += ' '; - let first = true; - for (const key in this.properties) { - if (this.properties.hasOwnProperty(key)) { - const val = this.properties[key]; - if (val) { - if (first) { - first = false; - } - else { - cmdStr += ','; - } - cmdStr += `${key}=${escapeProperty(val)}`; - } - } - } + return false; +} +exports.checkBypass = checkBypass; +//# sourceMappingURL=proxy.js.map + +/***/ }), + +/***/ 9417: +/***/ ((module) => { + +"use strict"; + +module.exports = balanced; +function balanced(a, b, str) { + if (a instanceof RegExp) a = maybeMatch(a, str); + if (b instanceof RegExp) b = maybeMatch(b, str); + + var r = range(a, b, str); + + return r && { + start: r[0], + end: r[1], + pre: str.slice(0, r[0]), + body: str.slice(r[0] + a.length, r[1]), + post: str.slice(r[1] + b.length) + }; +} + +function maybeMatch(reg, str) { + var m = str.match(reg); + return m ? m[0] : null; +} + +balanced.range = range; +function range(a, b, str) { + var begs, beg, left, right, result; + var ai = str.indexOf(a); + var bi = str.indexOf(b, ai + 1); + var i = ai; + + if (ai >= 0 && bi > 0) { + begs = []; + left = str.length; + + while (i >= 0 && !result) { + if (i == ai) { + begs.push(i); + ai = str.indexOf(a, i + 1); + } else if (begs.length == 1) { + result = [ begs.pop(), bi ]; + } else { + beg = begs.pop(); + if (beg < left) { + left = beg; + right = bi; } - cmdStr += `${CMD_STRING}${escapeData(this.message)}`; - return cmdStr; + + bi = str.indexOf(b, i + 1); + } + + i = ai < bi && ai >= 0 ? ai : bi; + } + + if (begs.length) { + result = [ left, right ]; } + } + + return result; } -function escapeData(s) { - return utils_1.toCommandValue(s) - .replace(/%/g, '%25') - .replace(/\r/g, '%0D') - .replace(/\n/g, '%0A'); + + +/***/ }), + +/***/ 3717: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +var concatMap = __nccwpck_require__(6891); +var balanced = __nccwpck_require__(9417); + +module.exports = expandTop; + +var escSlash = '\0SLASH'+Math.random()+'\0'; +var escOpen = '\0OPEN'+Math.random()+'\0'; +var escClose = '\0CLOSE'+Math.random()+'\0'; +var escComma = '\0COMMA'+Math.random()+'\0'; +var escPeriod = '\0PERIOD'+Math.random()+'\0'; + +function numeric(str) { + return parseInt(str, 10) == str + ? parseInt(str, 10) + : str.charCodeAt(0); } -function escapeProperty(s) { - return utils_1.toCommandValue(s) - .replace(/%/g, '%25') - .replace(/\r/g, '%0D') - .replace(/\n/g, '%0A') - .replace(/:/g, '%3A') - .replace(/,/g, '%2C'); + +function escapeBraces(str) { + return str.split('\\\\').join(escSlash) + .split('\\{').join(escOpen) + .split('\\}').join(escClose) + .split('\\,').join(escComma) + .split('\\.').join(escPeriod); +} + +function unescapeBraces(str) { + return str.split(escSlash).join('\\') + .split(escOpen).join('{') + .split(escClose).join('}') + .split(escComma).join(',') + .split(escPeriod).join('.'); } -//# sourceMappingURL=command.js.map -/***/ }), -/***/ 456: -/***/ (function(__unusedmodule, exports) { +// Basically just str.split(","), but handling cases +// where we have nested braced sections, which should be +// treated as individual members, like {a,{b,c},d} +function parseCommaParts(str) { + if (!str) + return ['']; -"use strict"; + var parts = []; + var m = balanced('{', '}', str); + if (!m) + return str.split(','); -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; -var _default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i; -exports.default = _default; + var pre = m.pre; + var body = m.body; + var post = m.post; + var p = pre.split(','); -/***/ }), + p[p.length-1] += '{' + body + '}'; + var postParts = parseCommaParts(post); + if (post.length) { + p[p.length-1] += postParts.shift(); + p.push.apply(p, postParts); + } -/***/ 470: -/***/ (function(__unusedmodule, exports, __webpack_require__) { + parts.push.apply(parts, p); -"use strict"; + return parts; +} -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.getIDToken = exports.getState = exports.saveState = exports.group = exports.endGroup = exports.startGroup = exports.info = exports.notice = exports.warning = exports.error = exports.debug = exports.isDebug = exports.setFailed = exports.setCommandEcho = exports.setOutput = exports.getBooleanInput = exports.getMultilineInput = exports.getInput = exports.addPath = exports.setSecret = exports.exportVariable = exports.ExitCode = void 0; -const command_1 = __webpack_require__(431); -const file_command_1 = __webpack_require__(102); -const utils_1 = __webpack_require__(82); -const os = __importStar(__webpack_require__(87)); -const path = __importStar(__webpack_require__(622)); -const uuid_1 = __webpack_require__(62); -const oidc_utils_1 = __webpack_require__(742); -/** - * The code to exit an action - */ -var ExitCode; -(function (ExitCode) { - /** - * A code indicating that the action was successful - */ - ExitCode[ExitCode["Success"] = 0] = "Success"; - /** - * A code indicating that the action was a failure - */ - ExitCode[ExitCode["Failure"] = 1] = "Failure"; -})(ExitCode = exports.ExitCode || (exports.ExitCode = {})); -//----------------------------------------------------------------------- -// Variables -//----------------------------------------------------------------------- -/** - * Sets env variable for this action and future actions in the job - * @param name the name of the variable to set - * @param val the value of the variable. Non-string values will be converted to a string via JSON.stringify - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function exportVariable(name, val) { - const convertedVal = utils_1.toCommandValue(val); - process.env[name] = convertedVal; - const filePath = process.env['GITHUB_ENV'] || ''; - if (filePath) { - const delimiter = `ghadelimiter_${uuid_1.v4()}`; - // These should realistically never happen, but just in case someone finds a way to exploit uuid generation let's not allow keys or values that contain the delimiter. - if (name.includes(delimiter)) { - throw new Error(`Unexpected input: name should not contain the delimiter "${delimiter}"`); - } - if (convertedVal.includes(delimiter)) { - throw new Error(`Unexpected input: value should not contain the delimiter "${delimiter}"`); - } - const commandValue = `${name}<<${delimiter}${os.EOL}${convertedVal}${os.EOL}${delimiter}`; - file_command_1.issueCommand('ENV', commandValue); - } - else { - command_1.issueCommand('set-env', { name }, convertedVal); - } +function expandTop(str) { + if (!str) + return []; + + // I don't know why Bash 4.3 does this, but it does. + // Anything starting with {} will have the first two bytes preserved + // but *only* at the top level, so {},a}b will not expand to anything, + // but a{},b}c will be expanded to [a}c,abc]. + // One could argue that this is a bug in Bash, but since the goal of + // this module is to match Bash's rules, we escape a leading {} + if (str.substr(0, 2) === '{}') { + str = '\\{\\}' + str.substr(2); + } + + return expand(escapeBraces(str), true).map(unescapeBraces); } -exports.exportVariable = exportVariable; -/** - * Registers a secret which will get masked from logs - * @param secret value of the secret - */ -function setSecret(secret) { - command_1.issueCommand('add-mask', {}, secret); + +function identity(e) { + return e; } -exports.setSecret = setSecret; -/** - * Prepends inputPath to the PATH (for this action and future actions) - * @param inputPath - */ -function addPath(inputPath) { - const filePath = process.env['GITHUB_PATH'] || ''; - if (filePath) { - file_command_1.issueCommand('PATH', inputPath); + +function embrace(str) { + return '{' + str + '}'; +} +function isPadded(el) { + return /^-?0\d/.test(el); +} + +function lte(i, y) { + return i <= y; +} +function gte(i, y) { + return i >= y; +} + +function expand(str, isTop) { + var expansions = []; + + var m = balanced('{', '}', str); + if (!m || /\$$/.test(m.pre)) return [str]; + + var isNumericSequence = /^-?\d+\.\.-?\d+(?:\.\.-?\d+)?$/.test(m.body); + var isAlphaSequence = /^[a-zA-Z]\.\.[a-zA-Z](?:\.\.-?\d+)?$/.test(m.body); + var isSequence = isNumericSequence || isAlphaSequence; + var isOptions = m.body.indexOf(',') >= 0; + if (!isSequence && !isOptions) { + // {a},b} + if (m.post.match(/,.*\}/)) { + str = m.pre + '{' + m.body + escClose + m.post; + return expand(str); } - else { - command_1.issueCommand('add-path', {}, inputPath); + return [str]; + } + + var n; + if (isSequence) { + n = m.body.split(/\.\./); + } else { + n = parseCommaParts(m.body); + if (n.length === 1) { + // x{{a,b}}y ==> x{a}y x{b}y + n = expand(n[0], false).map(embrace); + if (n.length === 1) { + var post = m.post.length + ? expand(m.post, false) + : ['']; + return post.map(function(p) { + return m.pre + n[0] + p; + }); + } } - process.env['PATH'] = `${inputPath}${path.delimiter}${process.env['PATH']}`; -} -exports.addPath = addPath; -/** - * Gets the value of an input. - * Unless trimWhitespace is set to false in InputOptions, the value is also trimmed. - * Returns an empty string if the value is not defined. - * - * @param name name of the input to get - * @param options optional. See InputOptions. - * @returns string - */ -function getInput(name, options) { - const val = process.env[`INPUT_${name.replace(/ /g, '_').toUpperCase()}`] || ''; - if (options && options.required && !val) { - throw new Error(`Input required and not supplied: ${name}`); + } + + // at this point, n is the parts, and we know it's not a comma set + // with a single entry. + + // no need to expand pre, since it is guaranteed to be free of brace-sets + var pre = m.pre; + var post = m.post.length + ? expand(m.post, false) + : ['']; + + var N; + + if (isSequence) { + var x = numeric(n[0]); + var y = numeric(n[1]); + var width = Math.max(n[0].length, n[1].length) + var incr = n.length == 3 + ? Math.abs(numeric(n[2])) + : 1; + var test = lte; + var reverse = y < x; + if (reverse) { + incr *= -1; + test = gte; + } + var pad = n.some(isPadded); + + N = []; + + for (var i = x; test(i, y); i += incr) { + var c; + if (isAlphaSequence) { + c = String.fromCharCode(i); + if (c === '\\') + c = ''; + } else { + c = String(i); + if (pad) { + var need = width - c.length; + if (need > 0) { + var z = new Array(need + 1).join('0'); + if (i < 0) + c = '-' + z + c.slice(1); + else + c = z + c; + } + } + } + N.push(c); } - if (options && options.trimWhitespace === false) { - return val; + } else { + N = concatMap(n, function(el) { return expand(el, false) }); + } + + for (var j = 0; j < N.length; j++) { + for (var k = 0; k < post.length; k++) { + var expansion = pre + N[j] + post[k]; + if (!isTop || isSequence || expansion) + expansions.push(expansion); } - return val.trim(); -} -exports.getInput = getInput; -/** - * Gets the values of an multiline input. Each value is also trimmed. - * - * @param name name of the input to get - * @param options optional. See InputOptions. - * @returns string[] - * - */ -function getMultilineInput(name, options) { - const inputs = getInput(name, options) - .split('\n') - .filter(x => x !== ''); - return inputs; -} -exports.getMultilineInput = getMultilineInput; -/** - * Gets the input value of the boolean type in the YAML 1.2 "core schema" specification. - * Support boolean input list: `true | True | TRUE | false | False | FALSE` . - * The return value is also in boolean type. - * ref: https://yaml.org/spec/1.2/spec.html#id2804923 - * - * @param name name of the input to get - * @param options optional. See InputOptions. - * @returns boolean - */ -function getBooleanInput(name, options) { - const trueValue = ['true', 'True', 'TRUE']; - const falseValue = ['false', 'False', 'FALSE']; - const val = getInput(name, options); - if (trueValue.includes(val)) - return true; - if (falseValue.includes(val)) - return false; - throw new TypeError(`Input does not meet YAML 1.2 "Core Schema" specification: ${name}\n` + - `Support boolean input list: \`true | True | TRUE | false | False | FALSE\``); -} -exports.getBooleanInput = getBooleanInput; -/** - * Sets the value of an output. - * - * @param name name of the output to set - * @param value value to store. Non-string values will be converted to a string via JSON.stringify - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function setOutput(name, value) { - process.stdout.write(os.EOL); - command_1.issueCommand('set-output', { name }, value); + } + + return expansions; } -exports.setOutput = setOutput; -/** - * Enables or disables the echoing of commands into stdout for the rest of the step. - * Echoing is disabled by default if ACTIONS_STEP_DEBUG is not set. - * - */ -function setCommandEcho(enabled) { - command_1.issue('echo', enabled ? 'on' : 'off'); + + + +/***/ }), + +/***/ 6891: +/***/ ((module) => { + +module.exports = function (xs, fn) { + var res = []; + for (var i = 0; i < xs.length; i++) { + var x = fn(xs[i], i); + if (isArray(x)) res.push.apply(res, x); + else res.push(x); + } + return res; +}; + +var isArray = Array.isArray || function (xs) { + return Object.prototype.toString.call(xs) === '[object Array]'; +}; + + +/***/ }), + +/***/ 3973: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +module.exports = minimatch +minimatch.Minimatch = Minimatch + +var path = (function () { try { return __nccwpck_require__(1017) } catch (e) {}}()) || { + sep: '/' } -exports.setCommandEcho = setCommandEcho; -//----------------------------------------------------------------------- -// Results -//----------------------------------------------------------------------- -/** - * Sets the action status to failed. - * When the action exits it will be with an exit code of 1 - * @param message add error issue message - */ -function setFailed(message) { - process.exitCode = ExitCode.Failure; - error(message); +minimatch.sep = path.sep + +var GLOBSTAR = minimatch.GLOBSTAR = Minimatch.GLOBSTAR = {} +var expand = __nccwpck_require__(3717) + +var plTypes = { + '!': { open: '(?:(?!(?:', close: '))[^/]*?)'}, + '?': { open: '(?:', close: ')?' }, + '+': { open: '(?:', close: ')+' }, + '*': { open: '(?:', close: ')*' }, + '@': { open: '(?:', close: ')' } } -exports.setFailed = setFailed; -//----------------------------------------------------------------------- -// Logging Commands -//----------------------------------------------------------------------- -/** - * Gets whether Actions Step Debug is on or not - */ -function isDebug() { - return process.env['RUNNER_DEBUG'] === '1'; + +// any single thing other than / +// don't need to escape / when using new RegExp() +var qmark = '[^/]' + +// * => any number of characters +var star = qmark + '*?' + +// ** when dots are allowed. Anything goes, except .. and . +// not (^ or / followed by one or two dots followed by $ or /), +// followed by anything, any number of times. +var twoStarDot = '(?:(?!(?:\\\/|^)(?:\\.{1,2})($|\\\/)).)*?' + +// not a ^ or / followed by a dot, +// followed by anything, any number of times. +var twoStarNoDot = '(?:(?!(?:\\\/|^)\\.).)*?' + +// characters that need to be escaped in RegExp. +var reSpecials = charSet('().*{}+?[]^$\\!') + +// "abc" -> { a:true, b:true, c:true } +function charSet (s) { + return s.split('').reduce(function (set, c) { + set[c] = true + return set + }, {}) } -exports.isDebug = isDebug; -/** - * Writes debug message to user log - * @param message debug message - */ -function debug(message) { - command_1.issueCommand('debug', {}, message); + +// normalizes slashes. +var slashSplit = /\/+/ + +minimatch.filter = filter +function filter (pattern, options) { + options = options || {} + return function (p, i, list) { + return minimatch(p, pattern, options) + } } -exports.debug = debug; -/** - * Adds an error issue - * @param message error issue message. Errors will be converted to string via toString() - * @param properties optional properties to add to the annotation. - */ -function error(message, properties = {}) { - command_1.issueCommand('error', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); + +function ext (a, b) { + b = b || {} + var t = {} + Object.keys(a).forEach(function (k) { + t[k] = a[k] + }) + Object.keys(b).forEach(function (k) { + t[k] = b[k] + }) + return t } -exports.error = error; -/** - * Adds a warning issue - * @param message warning issue message. Errors will be converted to string via toString() - * @param properties optional properties to add to the annotation. - */ -function warning(message, properties = {}) { - command_1.issueCommand('warning', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); + +minimatch.defaults = function (def) { + if (!def || typeof def !== 'object' || !Object.keys(def).length) { + return minimatch + } + + var orig = minimatch + + var m = function minimatch (p, pattern, options) { + return orig(p, pattern, ext(def, options)) + } + + m.Minimatch = function Minimatch (pattern, options) { + return new orig.Minimatch(pattern, ext(def, options)) + } + m.Minimatch.defaults = function defaults (options) { + return orig.defaults(ext(def, options)).Minimatch + } + + m.filter = function filter (pattern, options) { + return orig.filter(pattern, ext(def, options)) + } + + m.defaults = function defaults (options) { + return orig.defaults(ext(def, options)) + } + + m.makeRe = function makeRe (pattern, options) { + return orig.makeRe(pattern, ext(def, options)) + } + + m.braceExpand = function braceExpand (pattern, options) { + return orig.braceExpand(pattern, ext(def, options)) + } + + m.match = function (list, pattern, options) { + return orig.match(list, pattern, ext(def, options)) + } + + return m } -exports.warning = warning; -/** - * Adds a notice issue - * @param message notice issue message. Errors will be converted to string via toString() - * @param properties optional properties to add to the annotation. - */ -function notice(message, properties = {}) { - command_1.issueCommand('notice', utils_1.toCommandProperties(properties), message instanceof Error ? message.toString() : message); + +Minimatch.defaults = function (def) { + return minimatch.defaults(def).Minimatch } -exports.notice = notice; -/** - * Writes info to log with console.log. - * @param message info message - */ -function info(message) { - process.stdout.write(message + os.EOL); + +function minimatch (p, pattern, options) { + assertValidPattern(pattern) + + if (!options) options = {} + + // shortcut: comments match nothing. + if (!options.nocomment && pattern.charAt(0) === '#') { + return false + } + + return new Minimatch(pattern, options).match(p) } -exports.info = info; -/** - * Begin an output group. - * - * Output until the next `groupEnd` will be foldable in this group - * - * @param name The name of the output group - */ -function startGroup(name) { - command_1.issue('group', name); + +function Minimatch (pattern, options) { + if (!(this instanceof Minimatch)) { + return new Minimatch(pattern, options) + } + + assertValidPattern(pattern) + + if (!options) options = {} + + pattern = pattern.trim() + + // windows support: need to use /, not \ + if (!options.allowWindowsEscape && path.sep !== '/') { + pattern = pattern.split(path.sep).join('/') + } + + this.options = options + this.set = [] + this.pattern = pattern + this.regexp = null + this.negate = false + this.comment = false + this.empty = false + this.partial = !!options.partial + + // make the set of regexps etc. + this.make() } -exports.startGroup = startGroup; -/** - * End an output group. - */ -function endGroup() { - command_1.issue('endgroup'); + +Minimatch.prototype.debug = function () {} + +Minimatch.prototype.make = make +function make () { + var pattern = this.pattern + var options = this.options + + // empty patterns and comments match nothing. + if (!options.nocomment && pattern.charAt(0) === '#') { + this.comment = true + return + } + if (!pattern) { + this.empty = true + return + } + + // step 1: figure out negation, etc. + this.parseNegate() + + // step 2: expand braces + var set = this.globSet = this.braceExpand() + + if (options.debug) this.debug = function debug() { console.error.apply(console, arguments) } + + this.debug(this.pattern, set) + + // step 3: now we have a set, so turn each one into a series of path-portion + // matching patterns. + // These will be regexps, except in the case of "**", which is + // set to the GLOBSTAR object for globstar behavior, + // and will not contain any / characters + set = this.globParts = set.map(function (s) { + return s.split(slashSplit) + }) + + this.debug(this.pattern, set) + + // glob --> regexps + set = set.map(function (s, si, set) { + return s.map(this.parse, this) + }, this) + + this.debug(this.pattern, set) + + // filter out everything that didn't compile properly. + set = set.filter(function (s) { + return s.indexOf(false) === -1 + }) + + this.debug(this.pattern, set) + + this.set = set } -exports.endGroup = endGroup; -/** - * Wrap an asynchronous function call in a group. - * - * Returns the same type as the function itself. - * - * @param name The name of the group - * @param fn The function to wrap in the group - */ -function group(name, fn) { - return __awaiter(this, void 0, void 0, function* () { - startGroup(name); - let result; - try { - result = yield fn(); - } - finally { - endGroup(); - } - return result; - }); + +Minimatch.prototype.parseNegate = parseNegate +function parseNegate () { + var pattern = this.pattern + var negate = false + var options = this.options + var negateOffset = 0 + + if (options.nonegate) return + + for (var i = 0, l = pattern.length + ; i < l && pattern.charAt(i) === '!' + ; i++) { + negate = !negate + negateOffset++ + } + + if (negateOffset) this.pattern = pattern.substr(negateOffset) + this.negate = negate } -exports.group = group; -//----------------------------------------------------------------------- -// Wrapper action state -//----------------------------------------------------------------------- -/** - * Saves state for current action, the state can only be retrieved by this action's post job execution. - * - * @param name name of the state to store - * @param value value to store. Non-string values will be converted to a string via JSON.stringify - */ -// eslint-disable-next-line @typescript-eslint/no-explicit-any -function saveState(name, value) { - command_1.issueCommand('save-state', { name }, value); + +// Brace expansion: +// a{b,c}d -> abd acd +// a{b,}c -> abc ac +// a{0..3}d -> a0d a1d a2d a3d +// a{b,c{d,e}f}g -> abg acdfg acefg +// a{b,c}d{e,f}g -> abdeg acdeg abdeg abdfg +// +// Invalid sets are not expanded. +// a{2..}b -> a{2..}b +// a{b}c -> a{b}c +minimatch.braceExpand = function (pattern, options) { + return braceExpand(pattern, options) } -exports.saveState = saveState; -/** - * Gets the value of an state set by this action's main execution. - * - * @param name name of the state to get - * @returns string - */ -function getState(name) { - return process.env[`STATE_${name}`] || ''; + +Minimatch.prototype.braceExpand = braceExpand + +function braceExpand (pattern, options) { + if (!options) { + if (this instanceof Minimatch) { + options = this.options + } else { + options = {} + } + } + + pattern = typeof pattern === 'undefined' + ? this.pattern : pattern + + assertValidPattern(pattern) + + // Thanks to Yeting Li for + // improving this regexp to avoid a ReDOS vulnerability. + if (options.nobrace || !/\{(?:(?!\{).)*\}/.test(pattern)) { + // shortcut. no need to expand. + return [pattern] + } + + return expand(pattern) } -exports.getState = getState; -function getIDToken(aud) { - return __awaiter(this, void 0, void 0, function* () { - return yield oidc_utils_1.OidcClient.getIDToken(aud); - }); + +var MAX_PATTERN_LENGTH = 1024 * 64 +var assertValidPattern = function (pattern) { + if (typeof pattern !== 'string') { + throw new TypeError('invalid pattern') + } + + if (pattern.length > MAX_PATTERN_LENGTH) { + throw new TypeError('pattern is too long') + } } -exports.getIDToken = getIDToken; -/** - * Summary exports - */ -var summary_1 = __webpack_require__(665); -Object.defineProperty(exports, "summary", { enumerable: true, get: function () { return summary_1.summary; } }); -/** - * @deprecated use core.summary - */ -var summary_2 = __webpack_require__(665); -Object.defineProperty(exports, "markdownSummary", { enumerable: true, get: function () { return summary_2.markdownSummary; } }); -/** - * Path exports - */ -var path_utils_1 = __webpack_require__(573); -Object.defineProperty(exports, "toPosixPath", { enumerable: true, get: function () { return path_utils_1.toPosixPath; } }); -Object.defineProperty(exports, "toWin32Path", { enumerable: true, get: function () { return path_utils_1.toWin32Path; } }); -Object.defineProperty(exports, "toPlatformPath", { enumerable: true, get: function () { return path_utils_1.toPlatformPath; } }); -//# sourceMappingURL=core.js.map -/***/ }), +// parse a component of the expanded set. +// At this point, no pattern may contain "/" in it +// so we're going to return a 2d array, where each entry is the full +// pattern, split on '/', and then turned into a regular expression. +// A regexp is made at the end which joins each array with an +// escaped /, and another full one which joins each regexp with |. +// +// Following the lead of Bash 4.1, note that "**" only has special meaning +// when it is the *only* thing in a path portion. Otherwise, any series +// of * is equivalent to a single *. Globstar behavior is enabled by +// default, and can be disabled by setting options.noglobstar. +Minimatch.prototype.parse = parse +var SUBPARSE = {} +function parse (pattern, isSub) { + assertValidPattern(pattern) + + var options = this.options + + // shortcuts + if (pattern === '**') { + if (!options.noglobstar) + return GLOBSTAR + else + pattern = '*' + } + if (pattern === '') return '' + + var re = '' + var hasMagic = !!options.nocase + var escaping = false + // ? => one single character + var patternListStack = [] + var negativeLists = [] + var stateChar + var inClass = false + var reClassStart = -1 + var classStart = -1 + // . and .. never match anything that doesn't start with ., + // even when options.dot is set. + var patternStart = pattern.charAt(0) === '.' ? '' // anything + // not (start or / followed by . or .. followed by / or end) + : options.dot ? '(?!(?:^|\\\/)\\.{1,2}(?:$|\\\/))' + : '(?!\\.)' + var self = this + + function clearStateChar () { + if (stateChar) { + // we had some state-tracking character + // that wasn't consumed by this pass. + switch (stateChar) { + case '*': + re += star + hasMagic = true + break + case '?': + re += qmark + hasMagic = true + break + default: + re += '\\' + stateChar + break + } + self.debug('clearStateChar %j %j', stateChar, re) + stateChar = false + } + } + + for (var i = 0, len = pattern.length, c + ; (i < len) && (c = pattern.charAt(i)) + ; i++) { + this.debug('%s\t%s %s %j', pattern, i, re, c) -/***/ 498: -/***/ (function(__unusedmodule, exports, __webpack_require__) { + // skip over any that are escaped. + if (escaping && reSpecials[c]) { + re += '\\' + c + escaping = false + continue + } -"use strict"; + switch (c) { + /* istanbul ignore next */ + case '/': { + // completely not allowed, even escaped. + // Should already be path-split by now. + return false + } + case '\\': + clearStateChar() + escaping = true + continue -Object.defineProperty(exports, "__esModule", { - value: true -}); -exports.default = void 0; + // the various stateChar values + // for the "extglob" stuff. + case '?': + case '*': + case '+': + case '@': + case '!': + this.debug('%s\t%s %s %j <-- stateChar', pattern, i, re, c) -var _crypto = _interopRequireDefault(__webpack_require__(417)); + // all of those are literals inside a class, except that + // the glob [!a] means [^a] in regexp + if (inClass) { + this.debug(' in class') + if (c === '!' && i === classStart + 1) c = '^' + re += c + continue + } -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + // if we already have a stateChar, then it means + // that there was something like ** or +? in there. + // Handle the stateChar, then proceed with this one. + self.debug('call clearStateChar %j', stateChar) + clearStateChar() + stateChar = c + // if extglob is disabled, then +(asdf|foo) isn't a thing. + // just clear the statechar *now*, rather than even diving into + // the patternList stuff. + if (options.noext) clearStateChar() + continue -function sha1(bytes) { - if (Array.isArray(bytes)) { - bytes = Buffer.from(bytes); - } else if (typeof bytes === 'string') { - bytes = Buffer.from(bytes, 'utf8'); - } + case '(': + if (inClass) { + re += '(' + continue + } - return _crypto.default.createHash('sha1').update(bytes).digest(); -} + if (!stateChar) { + re += '\\(' + continue + } -var _default = sha1; -exports.default = _default; + patternListStack.push({ + type: stateChar, + start: i - 1, + reStart: re.length, + open: plTypes[stateChar].open, + close: plTypes[stateChar].close + }) + // negation is (?:(?!js)[^/]*) + re += stateChar === '!' ? '(?:(?!(?:' : '(?:' + this.debug('plType %j %j', stateChar, re) + stateChar = false + continue -/***/ }), + case ')': + if (inClass || !patternListStack.length) { + re += '\\)' + continue + } -/***/ 554: -/***/ (function(__unusedmodule, exports) { + clearStateChar() + hasMagic = true + var pl = patternListStack.pop() + // negation is (?:(?!js)[^/]*) + // The others are (?:) + re += pl.close + if (pl.type === '!') { + negativeLists.push(pl) + } + pl.reEnd = re.length + continue -"use strict"; + case '|': + if (inClass || !patternListStack.length || escaping) { + re += '\\|' + escaping = false + continue + } -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.PersonalAccessTokenCredentialHandler = exports.BearerCredentialHandler = exports.BasicCredentialHandler = void 0; -class BasicCredentialHandler { - constructor(username, password) { - this.username = username; - this.password = password; - } - prepareRequest(options) { - if (!options.headers) { - throw Error('The request has no headers'); + clearStateChar() + re += '|' + continue + + // these are mostly the same in regexp and glob + case '[': + // swallow any state-tracking char before the [ + clearStateChar() + + if (inClass) { + re += '\\' + c + continue } - options.headers['Authorization'] = `Basic ${Buffer.from(`${this.username}:${this.password}`).toString('base64')}`; - } - // This handler cannot handle 401 - canHandleAuthentication() { - return false; - } - handleAuthentication() { - return __awaiter(this, void 0, void 0, function* () { - throw new Error('not implemented'); - }); - } -} -exports.BasicCredentialHandler = BasicCredentialHandler; -class BearerCredentialHandler { - constructor(token) { - this.token = token; - } - // currently implements pre-authorization - // TODO: support preAuth = false where it hooks on 401 - prepareRequest(options) { - if (!options.headers) { - throw Error('The request has no headers'); + + inClass = true + classStart = i + reClassStart = re.length + re += c + continue + + case ']': + // a right bracket shall lose its special + // meaning and represent itself in + // a bracket expression if it occurs + // first in the list. -- POSIX.2 2.8.3.2 + if (i === classStart + 1 || !inClass) { + re += '\\' + c + escaping = false + continue } - options.headers['Authorization'] = `Bearer ${this.token}`; - } - // This handler cannot handle 401 - canHandleAuthentication() { - return false; - } - handleAuthentication() { - return __awaiter(this, void 0, void 0, function* () { - throw new Error('not implemented'); - }); - } -} -exports.BearerCredentialHandler = BearerCredentialHandler; -class PersonalAccessTokenCredentialHandler { - constructor(token) { - this.token = token; - } - // currently implements pre-authorization - // TODO: support preAuth = false where it hooks on 401 - prepareRequest(options) { - if (!options.headers) { - throw Error('The request has no headers'); + + // handle the case where we left a class open. + // "[z-a]" is valid, equivalent to "\[z-a\]" + // split where the last [ was, make sure we don't have + // an invalid re. if so, re-walk the contents of the + // would-be class to re-translate any characters that + // were passed through as-is + // TODO: It would probably be faster to determine this + // without a try/catch and a new RegExp, but it's tricky + // to do safely. For now, this is safe and works. + var cs = pattern.substring(classStart + 1, i) + try { + RegExp('[' + cs + ']') + } catch (er) { + // not a valid class! + var sp = this.parse(cs, SUBPARSE) + re = re.substr(0, reClassStart) + '\\[' + sp[0] + '\\]' + hasMagic = hasMagic || sp[1] + inClass = false + continue } - options.headers['Authorization'] = `Basic ${Buffer.from(`PAT:${this.token}`).toString('base64')}`; - } - // This handler cannot handle 401 - canHandleAuthentication() { - return false; - } - handleAuthentication() { - return __awaiter(this, void 0, void 0, function* () { - throw new Error('not implemented'); - }); - } -} -exports.PersonalAccessTokenCredentialHandler = PersonalAccessTokenCredentialHandler; -//# sourceMappingURL=auth.js.map -/***/ }), + // finish up the class. + hasMagic = true + inClass = false + re += c + continue + + default: + // swallow any state char that wasn't consumed + clearStateChar() + + if (escaping) { + // no need + escaping = false + } else if (reSpecials[c] + && !(c === '^' && inClass)) { + re += '\\' + } + + re += c + + } // switch + } // for + + // handle the case where we left a class open. + // "[abc" is valid, equivalent to "\[abc" + if (inClass) { + // split where the last [ was, and escape it + // this is a huge pita. We now have to re-walk + // the contents of the would-be class to re-translate + // any characters that were passed through as-is + cs = pattern.substr(classStart + 1) + sp = this.parse(cs, SUBPARSE) + re = re.substr(0, reClassStart) + '\\[' + sp[0] + hasMagic = hasMagic || sp[1] + } + + // handle the case where we had a +( thing at the *end* + // of the pattern. + // each pattern list stack adds 3 chars, and we need to go through + // and escape any | chars that were passed through as-is for the regexp. + // Go through and escape them, taking care not to double-escape any + // | chars that were already escaped. + for (pl = patternListStack.pop(); pl; pl = patternListStack.pop()) { + var tail = re.slice(pl.reStart + pl.open.length) + this.debug('setting tail', re, pl) + // maybe some even number of \, then maybe 1 \, followed by a | + tail = tail.replace(/((?:\\{2}){0,64})(\\?)\|/g, function (_, $1, $2) { + if (!$2) { + // the | isn't already escaped, so escape it. + $2 = '\\' + } -/***/ 573: -/***/ (function(__unusedmodule, exports, __webpack_require__) { + // need to escape all those slashes *again*, without escaping the + // one that we need for escaping the | character. As it works out, + // escaping an even number of slashes can be done by simply repeating + // it exactly after itself. That's why this trick works. + // + // I am sorry that you have to see this. + return $1 + $1 + $2 + '|' + }) -"use strict"; + this.debug('tail=%j\n %s', tail, tail, pl, re) + var t = pl.type === '*' ? star + : pl.type === '?' ? qmark + : '\\' + pl.type -var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } }); -}) : (function(o, m, k, k2) { - if (k2 === undefined) k2 = k; - o[k2] = m[k]; -})); -var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) { - Object.defineProperty(o, "default", { enumerable: true, value: v }); -}) : function(o, v) { - o["default"] = v; -}); -var __importStar = (this && this.__importStar) || function (mod) { - if (mod && mod.__esModule) return mod; - var result = {}; - if (mod != null) for (var k in mod) if (k !== "default" && Object.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k); - __setModuleDefault(result, mod); - return result; -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.toPlatformPath = exports.toWin32Path = exports.toPosixPath = void 0; -const path = __importStar(__webpack_require__(622)); -/** - * toPosixPath converts the given path to the posix form. On Windows, \\ will be - * replaced with /. - * - * @param pth. Path to transform. - * @return string Posix path. - */ -function toPosixPath(pth) { - return pth.replace(/[\\]/g, '/'); -} -exports.toPosixPath = toPosixPath; -/** - * toWin32Path converts the given path to the win32 form. On Linux, / will be - * replaced with \\. - * - * @param pth. Path to transform. - * @return string Win32 path. - */ -function toWin32Path(pth) { - return pth.replace(/[/]/g, '\\'); -} -exports.toWin32Path = toWin32Path; -/** - * toPlatformPath converts the given path to a platform-specific path. It does - * this by replacing instances of / and \ with the platform-specific path - * separator. - * - * @param pth The path to platformize. - * @return string The platform-specific path. - */ -function toPlatformPath(pth) { - return pth.replace(/[/\\]/g, path.sep); -} -exports.toPlatformPath = toPlatformPath; -//# sourceMappingURL=path-utils.js.map + hasMagic = true + re = re.slice(0, pl.reStart) + t + '\\(' + tail + } -/***/ }), + // handle trailing things that only matter at the very end. + clearStateChar() + if (escaping) { + // trailing \\ + re += '\\\\' + } -/***/ 597: -/***/ (function(__unusedmodule, exports, __webpack_require__) { + // only need to apply the nodot start if the re starts with + // something that could conceivably capture a dot + var addPatternStart = false + switch (re.charAt(0)) { + case '[': case '.': case '(': addPatternStart = true + } -"use strict"; + // Hack to work around lack of negative lookbehind in JS + // A pattern like: *.!(x).!(y|z) needs to ensure that a name + // like 'a.xyz.yz' doesn't match. So, the first negative + // lookahead, has to look ALL the way ahead, to the end of + // the pattern. + for (var n = negativeLists.length - 1; n > -1; n--) { + var nl = negativeLists[n] -Object.defineProperty(exports, "__esModule", { value: true }); -const pathHelper = __webpack_require__(972); -const internal_match_kind_1 = __webpack_require__(36); -const IS_WINDOWS = process.platform === 'win32'; -/** - * Given an array of patterns, returns an array of paths to search. - * Duplicates and paths under other included paths are filtered out. - */ -function getSearchPaths(patterns) { - // Ignore negate patterns - patterns = patterns.filter(x => !x.negate); - // Create a map of all search paths - const searchPathMap = {}; - for (const pattern of patterns) { - const key = IS_WINDOWS - ? pattern.searchPath.toUpperCase() - : pattern.searchPath; - searchPathMap[key] = 'candidate'; - } - const result = []; - for (const pattern of patterns) { - // Check if already included - const key = IS_WINDOWS - ? pattern.searchPath.toUpperCase() - : pattern.searchPath; - if (searchPathMap[key] === 'included') { - continue; - } - // Check for an ancestor search path - let foundAncestor = false; - let tempKey = key; - let parent = pathHelper.dirname(tempKey); - while (parent !== tempKey) { - if (searchPathMap[parent]) { - foundAncestor = true; - break; - } - tempKey = parent; - parent = pathHelper.dirname(tempKey); - } - // Include the search pattern in the result - if (!foundAncestor) { - result.push(pattern.searchPath); - searchPathMap[key] = 'included'; - } + var nlBefore = re.slice(0, nl.reStart) + var nlFirst = re.slice(nl.reStart, nl.reEnd - 8) + var nlLast = re.slice(nl.reEnd - 8, nl.reEnd) + var nlAfter = re.slice(nl.reEnd) + + nlLast += nlAfter + + // Handle nested stuff like *(*.js|!(*.json)), where open parens + // mean that we should *not* include the ) in the bit that is considered + // "after" the negated section. + var openParensBefore = nlBefore.split('(').length - 1 + var cleanAfter = nlAfter + for (i = 0; i < openParensBefore; i++) { + cleanAfter = cleanAfter.replace(/\)[+*?]?/, '') } - return result; -} -exports.getSearchPaths = getSearchPaths; -/** - * Matches the patterns against the path - */ -function match(patterns, itemPath) { - let result = internal_match_kind_1.MatchKind.None; - for (const pattern of patterns) { - if (pattern.negate) { - result &= ~pattern.match(itemPath); - } - else { - result |= pattern.match(itemPath); - } + nlAfter = cleanAfter + + var dollar = '' + if (nlAfter === '' && isSub !== SUBPARSE) { + dollar = '$' } - return result; + var newRe = nlBefore + nlFirst + nlAfter + dollar + nlLast + re = newRe + } + + // if the re is not "" at this point, then we need to make sure + // it doesn't match against an empty path part. + // Otherwise a/* will match a/, which it should not. + if (re !== '' && hasMagic) { + re = '(?=.)' + re + } + + if (addPatternStart) { + re = patternStart + re + } + + // parsing just a piece of a larger pattern. + if (isSub === SUBPARSE) { + return [re, hasMagic] + } + + // skip the regexp for non-magical patterns + // unescape anything in it, though, so that it'll be + // an exact match against a file etc. + if (!hasMagic) { + return globUnescape(pattern) + } + + var flags = options.nocase ? 'i' : '' + try { + var regExp = new RegExp('^' + re + '$', flags) + } catch (er) /* istanbul ignore next - should be impossible */ { + // If it was an invalid regular expression, then it can't match + // anything. This trick looks for a character after the end of + // the string, which is of course impossible, except in multi-line + // mode, but it's not a /m regex. + return new RegExp('$.') + } + + regExp._glob = pattern + regExp._src = re + + return regExp } -exports.match = match; -/** - * Checks whether to descend further into the directory - */ -function partialMatch(patterns, itemPath) { - return patterns.some(x => !x.negate && x.partialMatch(itemPath)); + +minimatch.makeRe = function (pattern, options) { + return new Minimatch(pattern, options || {}).makeRe() } -exports.partialMatch = partialMatch; -//# sourceMappingURL=internal-pattern-helper.js.map -/***/ }), +Minimatch.prototype.makeRe = makeRe +function makeRe () { + if (this.regexp || this.regexp === false) return this.regexp + + // at this point, this.set is a 2d array of partial + // pattern strings, or "**". + // + // It's better to use .match(). This function shouldn't + // be used, really, but it's pretty convenient sometimes, + // when you just want to work with a regex. + var set = this.set + + if (!set.length) { + this.regexp = false + return this.regexp + } + var options = this.options -/***/ 601: -/***/ (function(__unusedmodule, exports, __webpack_require__) { + var twoStar = options.noglobstar ? star + : options.dot ? twoStarDot + : twoStarNoDot + var flags = options.nocase ? 'i' : '' -"use strict"; + var re = set.map(function (pattern) { + return pattern.map(function (p) { + return (p === GLOBSTAR) ? twoStar + : (typeof p === 'string') ? regExpEscape(p) + : p._src + }).join('\\\/') + }).join('|') -Object.defineProperty(exports, "__esModule", { value: true }); -const core = __webpack_require__(470); -/** - * Returns a copy with defaults filled in. - */ -function getOptions(copy) { - const result = { - followSymbolicLinks: true, - implicitDescendants: true, - omitBrokenSymbolicLinks: true - }; - if (copy) { - if (typeof copy.followSymbolicLinks === 'boolean') { - result.followSymbolicLinks = copy.followSymbolicLinks; - core.debug(`followSymbolicLinks '${result.followSymbolicLinks}'`); - } - if (typeof copy.implicitDescendants === 'boolean') { - result.implicitDescendants = copy.implicitDescendants; - core.debug(`implicitDescendants '${result.implicitDescendants}'`); - } - if (typeof copy.omitBrokenSymbolicLinks === 'boolean') { - result.omitBrokenSymbolicLinks = copy.omitBrokenSymbolicLinks; - core.debug(`omitBrokenSymbolicLinks '${result.omitBrokenSymbolicLinks}'`); - } - } - return result; -} -exports.getOptions = getOptions; -//# sourceMappingURL=internal-glob-options-helper.js.map + // must match entire pattern + // ending in a * or ** will make it less strict. + re = '^(?:' + re + ')$' -/***/ }), + // can match anything, as long as it's not this. + if (this.negate) re = '^(?!' + re + ').*$' -/***/ 605: -/***/ (function(module) { + try { + this.regexp = new RegExp(re, flags) + } catch (ex) /* istanbul ignore next - should be impossible */ { + this.regexp = false + } + return this.regexp +} -module.exports = require("http"); +minimatch.match = function (list, pattern, options) { + options = options || {} + var mm = new Minimatch(pattern, options) + list = list.filter(function (f) { + return mm.match(f) + }) + if (mm.options.nonull && !list.length) { + list.push(pattern) + } + return list +} -/***/ }), +Minimatch.prototype.match = function match (f, partial) { + if (typeof partial === 'undefined') partial = this.partial + this.debug('match', f, this.pattern) + // short-circuit in the case of busted things. + // comments, etc. + if (this.comment) return false + if (this.empty) return f === '' -/***/ 614: -/***/ (function(module) { + if (f === '/' && partial) return true -module.exports = require("events"); + var options = this.options -/***/ }), + // windows: need to use /, not \ + if (path.sep !== '/') { + f = f.split(path.sep).join('/') + } -/***/ 621: -/***/ (function(module) { + // treat the test path as a set of pathparts. + f = f.split(slashSplit) + this.debug(this.pattern, 'split', f) -"use strict"; + // just ONE of the pattern sets in this.set needs to match + // in order for it to be valid. If negating, then just one + // match means that we have failed. + // Either way, return on the first hit. -module.exports = balanced; -function balanced(a, b, str) { - if (a instanceof RegExp) a = maybeMatch(a, str); - if (b instanceof RegExp) b = maybeMatch(b, str); + var set = this.set + this.debug(this.pattern, 'set', set) - var r = range(a, b, str); + // Find the basename of the path by looking for the last non-empty segment + var filename + var i + for (i = f.length - 1; i >= 0; i--) { + filename = f[i] + if (filename) break + } - return r && { - start: r[0], - end: r[1], - pre: str.slice(0, r[0]), - body: str.slice(r[0] + a.length, r[1]), - post: str.slice(r[1] + b.length) - }; -} + for (i = 0; i < set.length; i++) { + var pattern = set[i] + var file = f + if (options.matchBase && pattern.length === 1) { + file = [filename] + } + var hit = this.matchOne(file, pattern, partial) + if (hit) { + if (options.flipNegate) return true + return !this.negate + } + } -function maybeMatch(reg, str) { - var m = str.match(reg); - return m ? m[0] : null; + // didn't get any hits. this is success if it's a negative + // pattern, failure otherwise. + if (options.flipNegate) return false + return this.negate } -balanced.range = range; -function range(a, b, str) { - var begs, beg, left, right, result; - var ai = str.indexOf(a); - var bi = str.indexOf(b, ai + 1); - var i = ai; +// set partial to true to test if, for example, +// "/a/b" matches the start of "/*/b/*/d" +// Partial means, if you run out of file before you run +// out of pattern, then that's fine, as long as all +// the parts match. +Minimatch.prototype.matchOne = function (file, pattern, partial) { + var options = this.options - if (ai >= 0 && bi > 0) { - begs = []; - left = str.length; + this.debug('matchOne', + { 'this': this, file: file, pattern: pattern }) - while (i >= 0 && !result) { - if (i == ai) { - begs.push(i); - ai = str.indexOf(a, i + 1); - } else if (begs.length == 1) { - result = [ begs.pop(), bi ]; - } else { - beg = begs.pop(); - if (beg < left) { - left = beg; - right = bi; + this.debug('matchOne', file.length, pattern.length) + + for (var fi = 0, + pi = 0, + fl = file.length, + pl = pattern.length + ; (fi < fl) && (pi < pl) + ; fi++, pi++) { + this.debug('matchOne loop') + var p = pattern[pi] + var f = file[fi] + + this.debug(pattern, p, f) + + // should be impossible. + // some invalid regexp stuff in the set. + /* istanbul ignore if */ + if (p === false) return false + + if (p === GLOBSTAR) { + this.debug('GLOBSTAR', [pattern, p, f]) + + // "**" + // a/**/b/**/c would match the following: + // a/b/x/y/z/c + // a/x/y/z/b/c + // a/b/x/b/x/c + // a/b/c + // To do this, take the rest of the pattern after + // the **, and see if it would match the file remainder. + // If so, return success. + // If not, the ** "swallows" a segment, and try again. + // This is recursively awful. + // + // a/**/b/**/c matching a/b/x/y/z/c + // - a matches a + // - doublestar + // - matchOne(b/x/y/z/c, b/**/c) + // - b matches b + // - doublestar + // - matchOne(x/y/z/c, c) -> no + // - matchOne(y/z/c, c) -> no + // - matchOne(z/c, c) -> no + // - matchOne(c, c) yes, hit + var fr = fi + var pr = pi + 1 + if (pr === pl) { + this.debug('** at the end') + // a ** at the end will just swallow the rest. + // We have found a match. + // however, it will not swallow /.x, unless + // options.dot is set. + // . and .. are *never* matched by **, for explosively + // exponential reasons. + for (; fi < fl; fi++) { + if (file[fi] === '.' || file[fi] === '..' || + (!options.dot && file[fi].charAt(0) === '.')) return false } + return true + } - bi = str.indexOf(b, i + 1); + // ok, let's see if we can swallow whatever we can. + while (fr < fl) { + var swallowee = file[fr] + + this.debug('\nglobstar while', file, fr, pattern, pr, swallowee) + + // XXX remove this slice. Just pass the start index. + if (this.matchOne(file.slice(fr), pattern.slice(pr), partial)) { + this.debug('globstar found match!', fr, fl, swallowee) + // found a match. + return true + } else { + // can't swallow "." or ".." ever. + // can only swallow ".foo" when explicitly asked. + if (swallowee === '.' || swallowee === '..' || + (!options.dot && swallowee.charAt(0) === '.')) { + this.debug('dot detected!', file, fr, pattern, pr) + break + } + + // ** swallows a segment, and continue. + this.debug('globstar swallow a segment, and continue') + fr++ + } } - i = ai < bi && ai >= 0 ? ai : bi; + // no match was found. + // However, in partial mode, we can't say this is necessarily over. + // If there's more *pattern* left, then + /* istanbul ignore if */ + if (partial) { + // ran out of file + this.debug('\n>>> no match, partial?', file, fr, pattern, pr) + if (fr === fl) return true + } + return false } - if (begs.length) { - result = [ left, right ]; + // something other than ** + // non-magic patterns just have to match exactly + // patterns with magic have been turned into regexps. + var hit + if (typeof p === 'string') { + hit = f === p + this.debug('string match', p, f, hit) + } else { + hit = f.match(p) + this.debug('pattern match', p, f, hit) } + + if (!hit) return false } - return result; -} + // Note: ending in / means that we'll get a final "" + // at the end of the pattern. This can only match a + // corresponding "" at the end of the file. + // If the file ends in /, then it can only match a + // a pattern that ends in /, unless the pattern just + // doesn't have any more for it. But, a/b/ should *not* + // match "a/b/*", even though "" matches against the + // [^/]*? pattern, except in partial mode, where it might + // simply not be reached yet. + // However, a/b/ should still satisfy a/* + + // now either we fell off the end of the pattern, or we're done. + if (fi === fl && pi === pl) { + // ran out of pattern and filename at the same time. + // an exact hit! + return true + } else if (fi === fl) { + // ran out of file, but still had pattern left. + // this is ok if we're doing the match as part of + // a glob fs traversal. + return partial + } else /* istanbul ignore else */ if (pi === pl) { + // ran out of pattern, still have file left. + // this is only acceptable if we're on the very last + // empty segment of a file with a trailing slash. + // a/* should match a/b/ + return (fi === fl - 1) && (file[fi] === '') + } + // should be unreachable. + /* istanbul ignore next */ + throw new Error('wtf?') +} -/***/ }), +// replace stuff like \* with * +function globUnescape (s) { + return s.replace(/\\(.)/g, '$1') +} -/***/ 622: -/***/ (function(module) { +function regExpEscape (s) { + return s.replace(/[-[\]{}()*+?.,\\^$|#\s]/g, '\\$&') +} -module.exports = require("path"); /***/ }), -/***/ 631: -/***/ (function(module) { +/***/ 4294: +/***/ ((module, __unused_webpack_exports, __nccwpck_require__) => { + +module.exports = __nccwpck_require__(4219); -module.exports = require("net"); /***/ }), -/***/ 665: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 4219: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.summary = exports.markdownSummary = exports.SUMMARY_DOCS_URL = exports.SUMMARY_ENV_VAR = void 0; -const os_1 = __webpack_require__(87); -const fs_1 = __webpack_require__(747); -const { access, appendFile, writeFile } = fs_1.promises; -exports.SUMMARY_ENV_VAR = 'GITHUB_STEP_SUMMARY'; -exports.SUMMARY_DOCS_URL = 'https://docs.github.com/actions/using-workflows/workflow-commands-for-github-actions#adding-a-job-summary'; -class Summary { - constructor() { - this._buffer = ''; - } - /** - * Finds the summary file path from the environment, rejects if env var is not found or file does not exist - * Also checks r/w permissions. - * - * @returns step summary file path - */ - filePath() { - return __awaiter(this, void 0, void 0, function* () { - if (this._filePath) { - return this._filePath; - } - const pathFromEnv = process.env[exports.SUMMARY_ENV_VAR]; - if (!pathFromEnv) { - throw new Error(`Unable to find environment variable for $${exports.SUMMARY_ENV_VAR}. Check if your runtime environment supports job summaries.`); - } - try { - yield access(pathFromEnv, fs_1.constants.R_OK | fs_1.constants.W_OK); - } - catch (_a) { - throw new Error(`Unable to access summary file: '${pathFromEnv}'. Check if the file has correct read/write permissions.`); - } - this._filePath = pathFromEnv; - return this._filePath; - }); - } - /** - * Wraps content in an HTML tag, adding any HTML attributes - * - * @param {string} tag HTML tag to wrap - * @param {string | null} content content within the tag - * @param {[attribute: string]: string} attrs key-value list of HTML attributes to add - * - * @returns {string} content wrapped in HTML element - */ - wrap(tag, content, attrs = {}) { - const htmlAttrs = Object.entries(attrs) - .map(([key, value]) => ` ${key}="${value}"`) - .join(''); - if (!content) { - return `<${tag}${htmlAttrs}>`; - } - return `<${tag}${htmlAttrs}>${content}`; - } - /** - * Writes text in the buffer to the summary buffer file and empties buffer. Will append by default. - * - * @param {SummaryWriteOptions} [options] (optional) options for write operation - * - * @returns {Promise} summary instance - */ - write(options) { - return __awaiter(this, void 0, void 0, function* () { - const overwrite = !!(options === null || options === void 0 ? void 0 : options.overwrite); - const filePath = yield this.filePath(); - const writeFunc = overwrite ? writeFile : appendFile; - yield writeFunc(filePath, this._buffer, { encoding: 'utf8' }); - return this.emptyBuffer(); - }); - } - /** - * Clears the summary buffer and wipes the summary file - * - * @returns {Summary} summary instance - */ - clear() { - return __awaiter(this, void 0, void 0, function* () { - return this.emptyBuffer().write({ overwrite: true }); - }); - } - /** - * Returns the current summary buffer as a string - * - * @returns {string} string of summary buffer - */ - stringify() { - return this._buffer; - } - /** - * If the summary buffer is empty - * - * @returns {boolen} true if the buffer is empty - */ - isEmptyBuffer() { - return this._buffer.length === 0; - } - /** - * Resets the summary buffer without writing to summary file - * - * @returns {Summary} summary instance - */ - emptyBuffer() { - this._buffer = ''; - return this; - } - /** - * Adds raw text to the summary buffer - * - * @param {string} text content to add - * @param {boolean} [addEOL=false] (optional) append an EOL to the raw text (default: false) - * - * @returns {Summary} summary instance - */ - addRaw(text, addEOL = false) { - this._buffer += text; - return addEOL ? this.addEOL() : this; - } - /** - * Adds the operating system-specific end-of-line marker to the buffer - * - * @returns {Summary} summary instance - */ - addEOL() { - return this.addRaw(os_1.EOL); - } - /** - * Adds an HTML codeblock to the summary buffer - * - * @param {string} code content to render within fenced code block - * @param {string} lang (optional) language to syntax highlight code - * - * @returns {Summary} summary instance - */ - addCodeBlock(code, lang) { - const attrs = Object.assign({}, (lang && { lang })); - const element = this.wrap('pre', this.wrap('code', code), attrs); - return this.addRaw(element).addEOL(); - } - /** - * Adds an HTML list to the summary buffer - * - * @param {string[]} items list of items to render - * @param {boolean} [ordered=false] (optional) if the rendered list should be ordered or not (default: false) - * - * @returns {Summary} summary instance - */ - addList(items, ordered = false) { - const tag = ordered ? 'ol' : 'ul'; - const listItems = items.map(item => this.wrap('li', item)).join(''); - const element = this.wrap(tag, listItems); - return this.addRaw(element).addEOL(); - } - /** - * Adds an HTML table to the summary buffer - * - * @param {SummaryTableCell[]} rows table rows - * - * @returns {Summary} summary instance - */ - addTable(rows) { - const tableBody = rows - .map(row => { - const cells = row - .map(cell => { - if (typeof cell === 'string') { - return this.wrap('td', cell); - } - const { header, data, colspan, rowspan } = cell; - const tag = header ? 'th' : 'td'; - const attrs = Object.assign(Object.assign({}, (colspan && { colspan })), (rowspan && { rowspan })); - return this.wrap(tag, data, attrs); - }) - .join(''); - return this.wrap('tr', cells); - }) - .join(''); - const element = this.wrap('table', tableBody); - return this.addRaw(element).addEOL(); - } - /** - * Adds a collapsable HTML details element to the summary buffer - * - * @param {string} label text for the closed state - * @param {string} content collapsable content - * - * @returns {Summary} summary instance - */ - addDetails(label, content) { - const element = this.wrap('details', this.wrap('summary', label) + content); - return this.addRaw(element).addEOL(); + +var net = __nccwpck_require__(1808); +var tls = __nccwpck_require__(4404); +var http = __nccwpck_require__(3685); +var https = __nccwpck_require__(5687); +var events = __nccwpck_require__(2361); +var assert = __nccwpck_require__(9491); +var util = __nccwpck_require__(3837); + + +exports.httpOverHttp = httpOverHttp; +exports.httpsOverHttp = httpsOverHttp; +exports.httpOverHttps = httpOverHttps; +exports.httpsOverHttps = httpsOverHttps; + + +function httpOverHttp(options) { + var agent = new TunnelingAgent(options); + agent.request = http.request; + return agent; +} + +function httpsOverHttp(options) { + var agent = new TunnelingAgent(options); + agent.request = http.request; + agent.createSocket = createSecureSocket; + agent.defaultPort = 443; + return agent; +} + +function httpOverHttps(options) { + var agent = new TunnelingAgent(options); + agent.request = https.request; + return agent; +} + +function httpsOverHttps(options) { + var agent = new TunnelingAgent(options); + agent.request = https.request; + agent.createSocket = createSecureSocket; + agent.defaultPort = 443; + return agent; +} + + +function TunnelingAgent(options) { + var self = this; + self.options = options || {}; + self.proxyOptions = self.options.proxy || {}; + self.maxSockets = self.options.maxSockets || http.Agent.defaultMaxSockets; + self.requests = []; + self.sockets = []; + + self.on('free', function onFree(socket, host, port, localAddress) { + var options = toOptions(host, port, localAddress); + for (var i = 0, len = self.requests.length; i < len; ++i) { + var pending = self.requests[i]; + if (pending.host === options.host && pending.port === options.port) { + // Detect the request to connect same origin server, + // reuse the connection. + self.requests.splice(i, 1); + pending.request.onSocket(socket); + return; + } } - /** - * Adds an HTML image tag to the summary buffer - * - * @param {string} src path to the image you to embed - * @param {string} alt text description of the image - * @param {SummaryImageOptions} options (optional) addition image attributes - * - * @returns {Summary} summary instance - */ - addImage(src, alt, options) { - const { width, height } = options || {}; - const attrs = Object.assign(Object.assign({}, (width && { width })), (height && { height })); - const element = this.wrap('img', null, Object.assign({ src, alt }, attrs)); - return this.addRaw(element).addEOL(); + socket.destroy(); + self.removeSocket(socket); + }); +} +util.inherits(TunnelingAgent, events.EventEmitter); + +TunnelingAgent.prototype.addRequest = function addRequest(req, host, port, localAddress) { + var self = this; + var options = mergeOptions({request: req}, self.options, toOptions(host, port, localAddress)); + + if (self.sockets.length >= this.maxSockets) { + // We are over limit so we'll add it to the queue. + self.requests.push(options); + return; + } + + // If we are under maxSockets create a new one. + self.createSocket(options, function(socket) { + socket.on('free', onFree); + socket.on('close', onCloseOrRemove); + socket.on('agentRemove', onCloseOrRemove); + req.onSocket(socket); + + function onFree() { + self.emit('free', socket, options); } - /** - * Adds an HTML section heading element - * - * @param {string} text heading text - * @param {number | string} [level=1] (optional) the heading level, default: 1 - * - * @returns {Summary} summary instance - */ - addHeading(text, level) { - const tag = `h${level}`; - const allowedTag = ['h1', 'h2', 'h3', 'h4', 'h5', 'h6'].includes(tag) - ? tag - : 'h1'; - const element = this.wrap(allowedTag, text); - return this.addRaw(element).addEOL(); + + function onCloseOrRemove(err) { + self.removeSocket(socket); + socket.removeListener('free', onFree); + socket.removeListener('close', onCloseOrRemove); + socket.removeListener('agentRemove', onCloseOrRemove); } - /** - * Adds an HTML thematic break (
) to the summary buffer - * - * @returns {Summary} summary instance - */ - addSeparator() { - const element = this.wrap('hr', null); - return this.addRaw(element).addEOL(); + }); +}; + +TunnelingAgent.prototype.createSocket = function createSocket(options, cb) { + var self = this; + var placeholder = {}; + self.sockets.push(placeholder); + + var connectOptions = mergeOptions({}, self.proxyOptions, { + method: 'CONNECT', + path: options.host + ':' + options.port, + agent: false, + headers: { + host: options.host + ':' + options.port } - /** - * Adds an HTML line break (
) to the summary buffer - * - * @returns {Summary} summary instance - */ - addBreak() { - const element = this.wrap('br', null); - return this.addRaw(element).addEOL(); + }); + if (options.localAddress) { + connectOptions.localAddress = options.localAddress; + } + if (connectOptions.proxyAuth) { + connectOptions.headers = connectOptions.headers || {}; + connectOptions.headers['Proxy-Authorization'] = 'Basic ' + + new Buffer(connectOptions.proxyAuth).toString('base64'); + } + + debug('making CONNECT request'); + var connectReq = self.request(connectOptions); + connectReq.useChunkedEncodingByDefault = false; // for v0.6 + connectReq.once('response', onResponse); // for v0.6 + connectReq.once('upgrade', onUpgrade); // for v0.6 + connectReq.once('connect', onConnect); // for v0.7 or later + connectReq.once('error', onError); + connectReq.end(); + + function onResponse(res) { + // Very hacky. This is necessary to avoid http-parser leaks. + res.upgrade = true; + } + + function onUpgrade(res, socket, head) { + // Hacky. + process.nextTick(function() { + onConnect(res, socket, head); + }); + } + + function onConnect(res, socket, head) { + connectReq.removeAllListeners(); + socket.removeAllListeners(); + + if (res.statusCode !== 200) { + debug('tunneling socket could not be established, statusCode=%d', + res.statusCode); + socket.destroy(); + var error = new Error('tunneling socket could not be established, ' + + 'statusCode=' + res.statusCode); + error.code = 'ECONNRESET'; + options.request.emit('error', error); + self.removeSocket(placeholder); + return; } - /** - * Adds an HTML blockquote to the summary buffer - * - * @param {string} text quote text - * @param {string} cite (optional) citation url - * - * @returns {Summary} summary instance - */ - addQuote(text, cite) { - const attrs = Object.assign({}, (cite && { cite })); - const element = this.wrap('blockquote', text, attrs); - return this.addRaw(element).addEOL(); + if (head.length > 0) { + debug('got illegal response body from proxy'); + socket.destroy(); + var error = new Error('got illegal response body from proxy'); + error.code = 'ECONNRESET'; + options.request.emit('error', error); + self.removeSocket(placeholder); + return; } - /** - * Adds an HTML anchor tag to the summary buffer - * - * @param {string} text link text/content - * @param {string} href hyperlink - * - * @returns {Summary} summary instance - */ - addLink(text, href) { - const element = this.wrap('a', text, { href }); - return this.addRaw(element).addEOL(); + debug('tunneling connection has established'); + self.sockets[self.sockets.indexOf(placeholder)] = socket; + return cb(socket); + } + + function onError(cause) { + connectReq.removeAllListeners(); + + debug('tunneling socket could not be established, cause=%s\n', + cause.message, cause.stack); + var error = new Error('tunneling socket could not be established, ' + + 'cause=' + cause.message); + error.code = 'ECONNRESET'; + options.request.emit('error', error); + self.removeSocket(placeholder); + } +}; + +TunnelingAgent.prototype.removeSocket = function removeSocket(socket) { + var pos = this.sockets.indexOf(socket) + if (pos === -1) { + return; + } + this.sockets.splice(pos, 1); + + var pending = this.requests.shift(); + if (pending) { + // If we have pending requests and a socket gets closed a new one + // needs to be created to take over in the pool for the one that closed. + this.createSocket(pending, function(socket) { + pending.request.onSocket(socket); + }); + } +}; + +function createSecureSocket(options, cb) { + var self = this; + TunnelingAgent.prototype.createSocket.call(self, options, function(socket) { + var hostHeader = options.request.getHeader('host'); + var tlsOptions = mergeOptions({}, self.options, { + socket: socket, + servername: hostHeader ? hostHeader.replace(/:.*$/, '') : options.host + }); + + // 0 is dummy port for v0.6 + var secureSocket = tls.connect(0, tlsOptions); + self.sockets[self.sockets.indexOf(socket)] = secureSocket; + cb(secureSocket); + }); +} + + +function toOptions(host, port, localAddress) { + if (typeof host === 'string') { // since v0.10 + return { + host: host, + port: port, + localAddress: localAddress + }; + } + return host; // for v0.11 or later +} + +function mergeOptions(target) { + for (var i = 1, len = arguments.length; i < len; ++i) { + var overrides = arguments[i]; + if (typeof overrides === 'object') { + var keys = Object.keys(overrides); + for (var j = 0, keyLen = keys.length; j < keyLen; ++j) { + var k = keys[j]; + if (overrides[k] !== undefined) { + target[k] = overrides[k]; + } + } } + } + return target; } -const _summary = new Summary(); -/** - * @deprecated use `core.summary` - */ -exports.markdownSummary = _summary; -exports.summary = _summary; -//# sourceMappingURL=summary.js.map -/***/ }), -/***/ 669: -/***/ (function(module) { +var debug; +if (process.env.NODE_DEBUG && /\btunnel\b/.test(process.env.NODE_DEBUG)) { + debug = function() { + var args = Array.prototype.slice.call(arguments); + if (typeof args[0] === 'string') { + args[0] = 'TUNNEL: ' + args[0]; + } else { + args.unshift('TUNNEL:'); + } + console.error.apply(console, args); + } +} else { + debug = function() {}; +} +exports.debug = debug; // for test -module.exports = require("util"); /***/ }), -/***/ 695: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 5840: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -Object.defineProperty(exports, "__esModule", { +Object.defineProperty(exports, "__esModule", ({ value: true -}); -exports.default = void 0; +})); +Object.defineProperty(exports, "v1", ({ + enumerable: true, + get: function () { + return _v.default; + } +})); +Object.defineProperty(exports, "v3", ({ + enumerable: true, + get: function () { + return _v2.default; + } +})); +Object.defineProperty(exports, "v4", ({ + enumerable: true, + get: function () { + return _v3.default; + } +})); +Object.defineProperty(exports, "v5", ({ + enumerable: true, + get: function () { + return _v4.default; + } +})); +Object.defineProperty(exports, "NIL", ({ + enumerable: true, + get: function () { + return _nil.default; + } +})); +Object.defineProperty(exports, "version", ({ + enumerable: true, + get: function () { + return _version.default; + } +})); +Object.defineProperty(exports, "validate", ({ + enumerable: true, + get: function () { + return _validate.default; + } +})); +Object.defineProperty(exports, "stringify", ({ + enumerable: true, + get: function () { + return _stringify.default; + } +})); +Object.defineProperty(exports, "parse", ({ + enumerable: true, + get: function () { + return _parse.default; + } +})); -var _validate = _interopRequireDefault(__webpack_require__(78)); +var _v = _interopRequireDefault(__nccwpck_require__(8628)); -function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } +var _v2 = _interopRequireDefault(__nccwpck_require__(6409)); -function version(uuid) { - if (!(0, _validate.default)(uuid)) { - throw TypeError('Invalid UUID'); - } +var _v3 = _interopRequireDefault(__nccwpck_require__(5122)); - return parseInt(uuid.substr(14, 1), 16); -} +var _v4 = _interopRequireDefault(__nccwpck_require__(9120)); -var _default = version; -exports.default = _default; +var _nil = _interopRequireDefault(__nccwpck_require__(5332)); -/***/ }), +var _version = _interopRequireDefault(__nccwpck_require__(1595)); -/***/ 728: -/***/ (function(__unusedmodule, exports) { +var _validate = _interopRequireDefault(__nccwpck_require__(6900)); -"use strict"; +var _stringify = _interopRequireDefault(__nccwpck_require__(8950)); -Object.defineProperty(exports, "__esModule", { value: true }); -class SearchState { - constructor(path, level) { - this.path = path; - this.level = level; - } -} -exports.SearchState = SearchState; -//# sourceMappingURL=internal-search-state.js.map +var _parse = _interopRequireDefault(__nccwpck_require__(2746)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } /***/ }), -/***/ 733: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 4569: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -Object.defineProperty(exports, "__esModule", { +Object.defineProperty(exports, "__esModule", ({ value: true -}); -exports.default = void 0; - -var _rng = _interopRequireDefault(__webpack_require__(844)); +})); +exports["default"] = void 0; -var _stringify = _interopRequireDefault(__webpack_require__(411)); +var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } -function v4(options, buf, offset) { - options = options || {}; - - const rnds = options.random || (options.rng || _rng.default)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` +function md5(bytes) { + if (Array.isArray(bytes)) { + bytes = Buffer.from(bytes); + } else if (typeof bytes === 'string') { + bytes = Buffer.from(bytes, 'utf8'); + } + return _crypto.default.createHash('md5').update(bytes).digest(); +} - rnds[6] = rnds[6] & 0x0f | 0x40; - rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided +var _default = md5; +exports["default"] = _default; - if (buf) { - offset = offset || 0; +/***/ }), - for (let i = 0; i < 16; ++i) { - buf[offset + i] = rnds[i]; - } +/***/ 5332: +/***/ ((__unused_webpack_module, exports) => { - return buf; - } +"use strict"; - return (0, _stringify.default)(rnds); -} -var _default = v4; -exports.default = _default; +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; +var _default = '00000000-0000-0000-0000-000000000000'; +exports["default"] = _default; /***/ }), -/***/ 742: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 2746: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.OidcClient = void 0; -const http_client_1 = __webpack_require__(425); -const auth_1 = __webpack_require__(554); -const core_1 = __webpack_require__(470); -class OidcClient { - static createHttpClient(allowRetry = true, maxRetry = 10) { - const requestOptions = { - allowRetries: allowRetry, - maxRetries: maxRetry - }; - return new http_client_1.HttpClient('actions/oidc-client', [new auth_1.BearerCredentialHandler(OidcClient.getRequestToken())], requestOptions); - } - static getRequestToken() { - const token = process.env['ACTIONS_ID_TOKEN_REQUEST_TOKEN']; - if (!token) { - throw new Error('Unable to get ACTIONS_ID_TOKEN_REQUEST_TOKEN env variable'); - } - return token; - } - static getIDTokenUrl() { - const runtimeUrl = process.env['ACTIONS_ID_TOKEN_REQUEST_URL']; - if (!runtimeUrl) { - throw new Error('Unable to get ACTIONS_ID_TOKEN_REQUEST_URL env variable'); - } - return runtimeUrl; - } - static getCall(id_token_url) { - var _a; - return __awaiter(this, void 0, void 0, function* () { - const httpclient = OidcClient.createHttpClient(); - const res = yield httpclient - .getJson(id_token_url) - .catch(error => { - throw new Error(`Failed to get ID Token. \n - Error Code : ${error.statusCode}\n - Error Message: ${error.result.message}`); - }); - const id_token = (_a = res.result) === null || _a === void 0 ? void 0 : _a.value; - if (!id_token) { - throw new Error('Response json body do not have ID Token field'); - } - return id_token; - }); - } - static getIDToken(audience) { - return __awaiter(this, void 0, void 0, function* () { - try { - // New ID Token is requested from action service - let id_token_url = OidcClient.getIDTokenUrl(); - if (audience) { - const encodedAudience = encodeURIComponent(audience); - id_token_url = `${id_token_url}&audience=${encodedAudience}`; - } - core_1.debug(`ID token url is ${id_token_url}`); - const id_token = yield OidcClient.getCall(id_token_url); - core_1.setSecret(id_token); - return id_token; - } - catch (error) { - throw new Error(`Error message: ${error.message}`); - } - }); - } + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; + +var _validate = _interopRequireDefault(__nccwpck_require__(6900)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function parse(uuid) { + if (!(0, _validate.default)(uuid)) { + throw TypeError('Invalid UUID'); + } + + let v; + const arr = new Uint8Array(16); // Parse ########-....-....-....-............ + + arr[0] = (v = parseInt(uuid.slice(0, 8), 16)) >>> 24; + arr[1] = v >>> 16 & 0xff; + arr[2] = v >>> 8 & 0xff; + arr[3] = v & 0xff; // Parse ........-####-....-....-............ + + arr[4] = (v = parseInt(uuid.slice(9, 13), 16)) >>> 8; + arr[5] = v & 0xff; // Parse ........-....-####-....-............ + + arr[6] = (v = parseInt(uuid.slice(14, 18), 16)) >>> 8; + arr[7] = v & 0xff; // Parse ........-....-....-####-............ + + arr[8] = (v = parseInt(uuid.slice(19, 23), 16)) >>> 8; + arr[9] = v & 0xff; // Parse ........-....-....-....-############ + // (Use "/" to avoid 32-bit truncation when bit-shifting high-order bytes) + + arr[10] = (v = parseInt(uuid.slice(24, 36), 16)) / 0x10000000000 & 0xff; + arr[11] = v / 0x100000000 & 0xff; + arr[12] = v >>> 24 & 0xff; + arr[13] = v >>> 16 & 0xff; + arr[14] = v >>> 8 & 0xff; + arr[15] = v & 0xff; + return arr; } -exports.OidcClient = OidcClient; -//# sourceMappingURL=oidc-utils.js.map + +var _default = parse; +exports["default"] = _default; /***/ }), -/***/ 747: -/***/ (function(module) { +/***/ 814: +/***/ ((__unused_webpack_module, exports) => { -module.exports = require("fs"); +"use strict"; + + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; +var _default = /^(?:[0-9a-f]{8}-[0-9a-f]{4}-[1-5][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}|00000000-0000-0000-0000-000000000000)$/i; +exports["default"] = _default; /***/ }), -/***/ 794: -/***/ (function(module) { +/***/ 807: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { -module.exports = require("stream"); +"use strict"; + + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = rng; + +var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +const rnds8Pool = new Uint8Array(256); // # of random values to pre-allocate + +let poolPtr = rnds8Pool.length; + +function rng() { + if (poolPtr > rnds8Pool.length - 16) { + _crypto.default.randomFillSync(rnds8Pool); + + poolPtr = 0; + } + + return rnds8Pool.slice(poolPtr, poolPtr += 16); +} /***/ }), -/***/ 803: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 5274: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -Object.defineProperty(exports, "__esModule", { +Object.defineProperty(exports, "__esModule", ({ value: true -}); -exports.default = void 0; +})); +exports["default"] = void 0; -var _crypto = _interopRequireDefault(__webpack_require__(417)); +var _crypto = _interopRequireDefault(__nccwpck_require__(6113)); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } -function md5(bytes) { +function sha1(bytes) { if (Array.isArray(bytes)) { bytes = Buffer.from(bytes); } else if (typeof bytes === 'string') { bytes = Buffer.from(bytes, 'utf8'); } - return _crypto.default.createHash('md5').update(bytes).digest(); + return _crypto.default.createHash('sha1').update(bytes).digest(); } -var _default = md5; -exports.default = _default; +var _default = sha1; +exports["default"] = _default; /***/ }), -/***/ 844: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 8950: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -Object.defineProperty(exports, "__esModule", { +Object.defineProperty(exports, "__esModule", ({ value: true -}); -exports.default = rng; +})); +exports["default"] = void 0; -var _crypto = _interopRequireDefault(__webpack_require__(417)); +var _validate = _interopRequireDefault(__nccwpck_require__(6900)); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } -const rnds8Pool = new Uint8Array(256); // # of random values to pre-allocate +/** + * Convert array of 16 byte values to UUID string format of the form: + * XXXXXXXX-XXXX-XXXX-XXXX-XXXXXXXXXXXX + */ +const byteToHex = []; -let poolPtr = rnds8Pool.length; +for (let i = 0; i < 256; ++i) { + byteToHex.push((i + 0x100).toString(16).substr(1)); +} -function rng() { - if (poolPtr > rnds8Pool.length - 16) { - _crypto.default.randomFillSync(rnds8Pool); +function stringify(arr, offset = 0) { + // Note: Be careful editing this code! It's been tuned for performance + // and works in ways you may not expect. See https://github.com/uuidjs/uuid/pull/434 + const uuid = (byteToHex[arr[offset + 0]] + byteToHex[arr[offset + 1]] + byteToHex[arr[offset + 2]] + byteToHex[arr[offset + 3]] + '-' + byteToHex[arr[offset + 4]] + byteToHex[arr[offset + 5]] + '-' + byteToHex[arr[offset + 6]] + byteToHex[arr[offset + 7]] + '-' + byteToHex[arr[offset + 8]] + byteToHex[arr[offset + 9]] + '-' + byteToHex[arr[offset + 10]] + byteToHex[arr[offset + 11]] + byteToHex[arr[offset + 12]] + byteToHex[arr[offset + 13]] + byteToHex[arr[offset + 14]] + byteToHex[arr[offset + 15]]).toLowerCase(); // Consistency check for valid UUID. If this throws, it's likely due to one + // of the following: + // - One or more input array values don't map to a hex octet (leading to + // "undefined" in the uuid) + // - Invalid input values for the RFC `version` or `variant` fields - poolPtr = 0; + if (!(0, _validate.default)(uuid)) { + throw TypeError('Stringified UUID is invalid'); } - return rnds8Pool.slice(poolPtr, poolPtr += 16); + return uuid; } +var _default = stringify; +exports["default"] = _default; + /***/ }), -/***/ 893: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 8628: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { "use strict"; -Object.defineProperty(exports, "__esModule", { +Object.defineProperty(exports, "__esModule", ({ value: true -}); -exports.default = void 0; +})); +exports["default"] = void 0; -var _rng = _interopRequireDefault(__webpack_require__(844)); +var _rng = _interopRequireDefault(__nccwpck_require__(807)); -var _stringify = _interopRequireDefault(__webpack_require__(411)); +var _stringify = _interopRequireDefault(__nccwpck_require__(8950)); function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } @@ -4602,462 +4713,391 @@ function v1(options, buf, offset) { b[i++] = tmh >>> 24 & 0xf | 0x10; // include version - b[i++] = tmh >>> 16 & 0xff; // `clock_seq_hi_and_reserved` (Per 4.2.2 - include variant) + b[i++] = tmh >>> 16 & 0xff; // `clock_seq_hi_and_reserved` (Per 4.2.2 - include variant) + + b[i++] = clockseq >>> 8 | 0x80; // `clock_seq_low` + + b[i++] = clockseq & 0xff; // `node` + + for (let n = 0; n < 6; ++n) { + b[i + n] = node[n]; + } + + return buf || (0, _stringify.default)(b); +} + +var _default = v1; +exports["default"] = _default; + +/***/ }), + +/***/ 6409: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; + +var _v = _interopRequireDefault(__nccwpck_require__(5998)); + +var _md = _interopRequireDefault(__nccwpck_require__(4569)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +const v3 = (0, _v.default)('v3', 0x30, _md.default); +var _default = v3; +exports["default"] = _default; + +/***/ }), + +/***/ 5998: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = _default; +exports.URL = exports.DNS = void 0; + +var _stringify = _interopRequireDefault(__nccwpck_require__(8950)); + +var _parse = _interopRequireDefault(__nccwpck_require__(2746)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function stringToBytes(str) { + str = unescape(encodeURIComponent(str)); // UTF8 escape + + const bytes = []; + + for (let i = 0; i < str.length; ++i) { + bytes.push(str.charCodeAt(i)); + } + + return bytes; +} + +const DNS = '6ba7b810-9dad-11d1-80b4-00c04fd430c8'; +exports.DNS = DNS; +const URL = '6ba7b811-9dad-11d1-80b4-00c04fd430c8'; +exports.URL = URL; + +function _default(name, version, hashfunc) { + function generateUUID(value, namespace, buf, offset) { + if (typeof value === 'string') { + value = stringToBytes(value); + } + + if (typeof namespace === 'string') { + namespace = (0, _parse.default)(namespace); + } + + if (namespace.length !== 16) { + throw TypeError('Namespace must be array-like (16 iterable integer values, 0-255)'); + } // Compute hash of namespace and value, Per 4.3 + // Future: Use spread syntax when supported on all platforms, e.g. `bytes = + // hashfunc([...namespace, ... value])` + + + let bytes = new Uint8Array(16 + value.length); + bytes.set(namespace); + bytes.set(value, namespace.length); + bytes = hashfunc(bytes); + bytes[6] = bytes[6] & 0x0f | version; + bytes[8] = bytes[8] & 0x3f | 0x80; + + if (buf) { + offset = offset || 0; + + for (let i = 0; i < 16; ++i) { + buf[offset + i] = bytes[i]; + } + + return buf; + } + + return (0, _stringify.default)(bytes); + } // Function#name is not settable on some platforms (#270) + + + try { + generateUUID.name = name; // eslint-disable-next-line no-empty + } catch (err) {} // For CommonJS default export support + + + generateUUID.DNS = DNS; + generateUUID.URL = URL; + return generateUUID; +} + +/***/ }), + +/***/ 5122: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; + +var _rng = _interopRequireDefault(__nccwpck_require__(807)); + +var _stringify = _interopRequireDefault(__nccwpck_require__(8950)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +function v4(options, buf, offset) { + options = options || {}; + + const rnds = options.random || (options.rng || _rng.default)(); // Per 4.4, set bits for version and `clock_seq_hi_and_reserved` + + + rnds[6] = rnds[6] & 0x0f | 0x40; + rnds[8] = rnds[8] & 0x3f | 0x80; // Copy bytes to buffer, if provided + + if (buf) { + offset = offset || 0; + + for (let i = 0; i < 16; ++i) { + buf[offset + i] = rnds[i]; + } + + return buf; + } + + return (0, _stringify.default)(rnds); +} + +var _default = v4; +exports["default"] = _default; + +/***/ }), + +/***/ 9120: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { + +"use strict"; + + +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; + +var _v = _interopRequireDefault(__nccwpck_require__(5998)); + +var _sha = _interopRequireDefault(__nccwpck_require__(5274)); + +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } + +const v5 = (0, _v.default)('v5', 0x50, _sha.default); +var _default = v5; +exports["default"] = _default; + +/***/ }), + +/***/ 6900: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { + +"use strict"; + - b[i++] = clockseq >>> 8 | 0x80; // `clock_seq_low` +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; - b[i++] = clockseq & 0xff; // `node` +var _regex = _interopRequireDefault(__nccwpck_require__(814)); - for (let n = 0; n < 6; ++n) { - b[i + n] = node[n]; - } +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } - return buf || (0, _stringify.default)(b); +function validate(uuid) { + return typeof uuid === 'string' && _regex.default.test(uuid); } -var _default = v1; -exports.default = _default; +var _default = validate; +exports["default"] = _default; /***/ }), -/***/ 896: -/***/ (function(module) { +/***/ 1595: +/***/ ((__unused_webpack_module, exports, __nccwpck_require__) => { -module.exports = function (xs, fn) { - var res = []; - for (var i = 0; i < xs.length; i++) { - var x = fn(xs[i], i); - if (isArray(x)) res.push.apply(res, x); - else res.push(x); - } - return res; -}; +"use strict"; -var isArray = Array.isArray || function (xs) { - return Object.prototype.toString.call(xs) === '[object Array]'; -}; +Object.defineProperty(exports, "__esModule", ({ + value: true +})); +exports["default"] = void 0; -/***/ }), +var _validate = _interopRequireDefault(__nccwpck_require__(6900)); -/***/ 923: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; } -"use strict"; +function version(uuid) { + if (!(0, _validate.default)(uuid)) { + throw TypeError('Invalid UUID'); + } -Object.defineProperty(exports, "__esModule", { value: true }); -const assert = __webpack_require__(357); -const os = __webpack_require__(87); -const path = __webpack_require__(622); -const pathHelper = __webpack_require__(972); -const minimatch_1 = __webpack_require__(93); -const internal_match_kind_1 = __webpack_require__(36); -const internal_path_1 = __webpack_require__(383); -const IS_WINDOWS = process.platform === 'win32'; -class Pattern { - constructor(patternOrNegate, segments) { - /** - * Indicates whether matches should be excluded from the result set - */ - this.negate = false; - // Pattern overload - let pattern; - if (typeof patternOrNegate === 'string') { - pattern = patternOrNegate.trim(); - } - // Segments overload - else { - // Convert to pattern - segments = segments || []; - assert(segments.length, `Parameter 'segments' must not empty`); - const root = Pattern.getLiteral(segments[0]); - assert(root && pathHelper.hasAbsoluteRoot(root), `Parameter 'segments' first element must be a root path`); - pattern = new internal_path_1.Path(segments).toString().trim(); - if (patternOrNegate) { - pattern = `!${pattern}`; - } - } - // Negate - while (pattern.startsWith('!')) { - this.negate = !this.negate; - pattern = pattern.substr(1).trim(); - } - // Normalize slashes and ensures absolute root - pattern = Pattern.fixupPattern(pattern); - // Segments - this.segments = new internal_path_1.Path(pattern).segments; - // Trailing slash indicates the pattern should only match directories, not regular files - this.trailingSeparator = pathHelper - .normalizeSeparators(pattern) - .endsWith(path.sep); - pattern = pathHelper.safeTrimTrailingSeparator(pattern); - // Search path (literal path prior to the first glob segment) - let foundGlob = false; - const searchSegments = this.segments - .map(x => Pattern.getLiteral(x)) - .filter(x => !foundGlob && !(foundGlob = x === '')); - this.searchPath = new internal_path_1.Path(searchSegments).toString(); - // Root RegExp (required when determining partial match) - this.rootRegExp = new RegExp(Pattern.regExpEscape(searchSegments[0]), IS_WINDOWS ? 'i' : ''); - // Create minimatch - const minimatchOptions = { - dot: true, - nobrace: true, - nocase: IS_WINDOWS, - nocomment: true, - noext: true, - nonegate: true - }; - pattern = IS_WINDOWS ? pattern.replace(/\\/g, '/') : pattern; - this.minimatch = new minimatch_1.Minimatch(pattern, minimatchOptions); - } - /** - * Matches the pattern against the specified path - */ - match(itemPath) { - // Last segment is globstar? - if (this.segments[this.segments.length - 1] === '**') { - // Normalize slashes - itemPath = pathHelper.normalizeSeparators(itemPath); - // Append a trailing slash. Otherwise Minimatch will not match the directory immediately - // preceeding the globstar. For example, given the pattern `/foo/**`, Minimatch returns - // false for `/foo` but returns true for `/foo/`. Append a trailing slash to handle that quirk. - if (!itemPath.endsWith(path.sep)) { - // Note, this is safe because the constructor ensures the pattern has an absolute root. - // For example, formats like C: and C:foo on Windows are resolved to an aboslute root. - itemPath = `${itemPath}${path.sep}`; - } - } - else { - // Normalize slashes and trim unnecessary trailing slash - itemPath = pathHelper.safeTrimTrailingSeparator(itemPath); - } - // Match - if (this.minimatch.match(itemPath)) { - return this.trailingSeparator ? internal_match_kind_1.MatchKind.Directory : internal_match_kind_1.MatchKind.All; - } - return internal_match_kind_1.MatchKind.None; - } - /** - * Indicates whether the pattern may match descendants of the specified path - */ - partialMatch(itemPath) { - // Normalize slashes and trim unnecessary trailing slash - itemPath = pathHelper.safeTrimTrailingSeparator(itemPath); - // matchOne does not handle root path correctly - if (pathHelper.dirname(itemPath) === itemPath) { - return this.rootRegExp.test(itemPath); - } - return this.minimatch.matchOne(itemPath.split(IS_WINDOWS ? /\\+/ : /\/+/), this.minimatch.set[0], true); - } - /** - * Escapes glob patterns within a path - */ - static globEscape(s) { - return (IS_WINDOWS ? s : s.replace(/\\/g, '\\\\')) // escape '\' on Linux/macOS - .replace(/(\[)(?=[^/]+\])/g, '[[]') // escape '[' when ']' follows within the path segment - .replace(/\?/g, '[?]') // escape '?' - .replace(/\*/g, '[*]'); // escape '*' - } - /** - * Normalizes slashes and ensures absolute root - */ - static fixupPattern(pattern) { - // Empty - assert(pattern, 'pattern cannot be empty'); - // Must not contain `.` segment, unless first segment - // Must not contain `..` segment - const literalSegments = new internal_path_1.Path(pattern).segments.map(x => Pattern.getLiteral(x)); - assert(literalSegments.every((x, i) => (x !== '.' || i === 0) && x !== '..'), `Invalid pattern '${pattern}'. Relative pathing '.' and '..' is not allowed.`); - // Must not contain globs in root, e.g. Windows UNC path \\foo\b*r - assert(!pathHelper.hasRoot(pattern) || literalSegments[0], `Invalid pattern '${pattern}'. Root segment must not contain globs.`); - // Normalize slashes - pattern = pathHelper.normalizeSeparators(pattern); - // Replace leading `.` segment - if (pattern === '.' || pattern.startsWith(`.${path.sep}`)) { - pattern = Pattern.globEscape(process.cwd()) + pattern.substr(1); - } - // Replace leading `~` segment - else if (pattern === '~' || pattern.startsWith(`~${path.sep}`)) { - const homedir = os.homedir(); - assert(homedir, 'Unable to determine HOME directory'); - assert(pathHelper.hasAbsoluteRoot(homedir), `Expected HOME directory to be a rooted path. Actual '${homedir}'`); - pattern = Pattern.globEscape(homedir) + pattern.substr(1); - } - // Replace relative drive root, e.g. pattern is C: or C:foo - else if (IS_WINDOWS && - (pattern.match(/^[A-Z]:$/i) || pattern.match(/^[A-Z]:[^\\]/i))) { - let root = pathHelper.ensureAbsoluteRoot('C:\\dummy-root', pattern.substr(0, 2)); - if (pattern.length > 2 && !root.endsWith('\\')) { - root += '\\'; - } - pattern = Pattern.globEscape(root) + pattern.substr(2); - } - // Replace relative root, e.g. pattern is \ or \foo - else if (IS_WINDOWS && (pattern === '\\' || pattern.match(/^\\[^\\]/))) { - let root = pathHelper.ensureAbsoluteRoot('C:\\dummy-root', '\\'); - if (!root.endsWith('\\')) { - root += '\\'; - } - pattern = Pattern.globEscape(root) + pattern.substr(1); - } - // Otherwise ensure absolute root - else { - pattern = pathHelper.ensureAbsoluteRoot(Pattern.globEscape(process.cwd()), pattern); - } - return pathHelper.normalizeSeparators(pattern); - } - /** - * Attempts to unescape a pattern segment to create a literal path segment. - * Otherwise returns empty string. - */ - static getLiteral(segment) { - let literal = ''; - for (let i = 0; i < segment.length; i++) { - const c = segment[i]; - // Escape - if (c === '\\' && !IS_WINDOWS && i + 1 < segment.length) { - literal += segment[++i]; - continue; - } - // Wildcard - else if (c === '*' || c === '?') { - return ''; - } - // Character set - else if (c === '[' && i + 1 < segment.length) { - let set = ''; - let closed = -1; - for (let i2 = i + 1; i2 < segment.length; i2++) { - const c2 = segment[i2]; - // Escape - if (c2 === '\\' && !IS_WINDOWS && i2 + 1 < segment.length) { - set += segment[++i2]; - continue; - } - // Closed - else if (c2 === ']') { - closed = i2; - break; - } - // Otherwise - else { - set += c2; - } - } - // Closed? - if (closed >= 0) { - // Cannot convert - if (set.length > 1) { - return ''; - } - // Convert to literal - if (set) { - literal += set; - i = closed; - continue; - } - } - // Otherwise fall thru - } - // Append - literal += c; - } - return literal; - } - /** - * Escapes regexp special characters - * https://javascript.info/regexp-escaping - */ - static regExpEscape(s) { - return s.replace(/[[\\^$.|?*+()]/g, '\\$&'); - } + return parseInt(uuid.substr(14, 1), 16); } -exports.Pattern = Pattern; -//# sourceMappingURL=internal-pattern.js.map + +var _default = version; +exports["default"] = _default; /***/ }), -/***/ 972: -/***/ (function(__unusedmodule, exports, __webpack_require__) { +/***/ 9491: +/***/ ((module) => { "use strict"; +module.exports = require("assert"); -Object.defineProperty(exports, "__esModule", { value: true }); -const assert = __webpack_require__(357); -const path = __webpack_require__(622); -const IS_WINDOWS = process.platform === 'win32'; -/** - * Similar to path.dirname except normalizes the path separators and slightly better handling for Windows UNC paths. - * - * For example, on Linux/macOS: - * - `/ => /` - * - `/hello => /` - * - * For example, on Windows: - * - `C:\ => C:\` - * - `C:\hello => C:\` - * - `C: => C:` - * - `C:hello => C:` - * - `\ => \` - * - `\hello => \` - * - `\\hello => \\hello` - * - `\\hello\world => \\hello\world` - */ -function dirname(p) { - // Normalize slashes and trim unnecessary trailing slash - p = safeTrimTrailingSeparator(p); - // Windows UNC root, e.g. \\hello or \\hello\world - if (IS_WINDOWS && /^\\\\[^\\]+(\\[^\\]+)?$/.test(p)) { - return p; - } - // Get dirname - let result = path.dirname(p); - // Trim trailing slash for Windows UNC root, e.g. \\hello\world\ - if (IS_WINDOWS && /^\\\\[^\\]+\\[^\\]+\\$/.test(result)) { - result = safeTrimTrailingSeparator(result); - } - return result; -} -exports.dirname = dirname; -/** - * Roots the path if not already rooted. On Windows, relative roots like `\` - * or `C:` are expanded based on the current working directory. - */ -function ensureAbsoluteRoot(root, itemPath) { - assert(root, `ensureAbsoluteRoot parameter 'root' must not be empty`); - assert(itemPath, `ensureAbsoluteRoot parameter 'itemPath' must not be empty`); - // Already rooted - if (hasAbsoluteRoot(itemPath)) { - return itemPath; - } - // Windows - if (IS_WINDOWS) { - // Check for itemPath like C: or C:foo - if (itemPath.match(/^[A-Z]:[^\\/]|^[A-Z]:$/i)) { - let cwd = process.cwd(); - assert(cwd.match(/^[A-Z]:\\/i), `Expected current directory to start with an absolute drive root. Actual '${cwd}'`); - // Drive letter matches cwd? Expand to cwd - if (itemPath[0].toUpperCase() === cwd[0].toUpperCase()) { - // Drive only, e.g. C: - if (itemPath.length === 2) { - // Preserve specified drive letter case (upper or lower) - return `${itemPath[0]}:\\${cwd.substr(3)}`; - } - // Drive + path, e.g. C:foo - else { - if (!cwd.endsWith('\\')) { - cwd += '\\'; - } - // Preserve specified drive letter case (upper or lower) - return `${itemPath[0]}:\\${cwd.substr(3)}${itemPath.substr(2)}`; - } - } - // Different drive - else { - return `${itemPath[0]}:\\${itemPath.substr(2)}`; - } - } - // Check for itemPath like \ or \foo - else if (normalizeSeparators(itemPath).match(/^\\$|^\\[^\\]/)) { - const cwd = process.cwd(); - assert(cwd.match(/^[A-Z]:\\/i), `Expected current directory to start with an absolute drive root. Actual '${cwd}'`); - return `${cwd[0]}:\\${itemPath.substr(1)}`; - } - } - assert(hasAbsoluteRoot(root), `ensureAbsoluteRoot parameter 'root' must have an absolute root`); - // Otherwise ensure root ends with a separator - if (root.endsWith('/') || (IS_WINDOWS && root.endsWith('\\'))) { - // Intentionally empty - } - else { - // Append separator - root += path.sep; - } - return root + itemPath; -} -exports.ensureAbsoluteRoot = ensureAbsoluteRoot; -/** - * On Linux/macOS, true if path starts with `/`. On Windows, true for paths like: - * `\\hello\share` and `C:\hello` (and using alternate separator). - */ -function hasAbsoluteRoot(itemPath) { - assert(itemPath, `hasAbsoluteRoot parameter 'itemPath' must not be empty`); - // Normalize separators - itemPath = normalizeSeparators(itemPath); - // Windows - if (IS_WINDOWS) { - // E.g. \\hello\share or C:\hello - return itemPath.startsWith('\\\\') || /^[A-Z]:\\/i.test(itemPath); - } - // E.g. /hello - return itemPath.startsWith('/'); -} -exports.hasAbsoluteRoot = hasAbsoluteRoot; -/** - * On Linux/macOS, true if path starts with `/`. On Windows, true for paths like: - * `\`, `\hello`, `\\hello\share`, `C:`, and `C:\hello` (and using alternate separator). - */ -function hasRoot(itemPath) { - assert(itemPath, `isRooted parameter 'itemPath' must not be empty`); - // Normalize separators - itemPath = normalizeSeparators(itemPath); - // Windows - if (IS_WINDOWS) { - // E.g. \ or \hello or \\hello - // E.g. C: or C:\hello - return itemPath.startsWith('\\') || /^[A-Z]:/i.test(itemPath); - } - // E.g. /hello - return itemPath.startsWith('/'); -} -exports.hasRoot = hasRoot; -/** - * Removes redundant slashes and converts `/` to `\` on Windows - */ -function normalizeSeparators(p) { - p = p || ''; - // Windows - if (IS_WINDOWS) { - // Convert slashes on Windows - p = p.replace(/\//g, '\\'); - // Remove redundant slashes - const isUnc = /^\\\\+[^\\]/.test(p); // e.g. \\hello - return (isUnc ? '\\' : '') + p.replace(/\\\\+/g, '\\'); // preserve leading \\ for UNC - } - // Remove redundant slashes - return p.replace(/\/\/+/g, '/'); -} -exports.normalizeSeparators = normalizeSeparators; -/** - * Normalizes the path separators and trims the trailing separator (when safe). - * For example, `/foo/ => /foo` but `/ => /` - */ -function safeTrimTrailingSeparator(p) { - // Short-circuit if empty - if (!p) { - return ''; - } - // Normalize separators - p = normalizeSeparators(p); - // No trailing slash - if (!p.endsWith(path.sep)) { - return p; - } - // Check '/' on Linux/macOS and '\' on Windows - if (p === path.sep) { - return p; - } - // On Windows check if drive root. E.g. C:\ - if (IS_WINDOWS && /^[A-Z]:\\$/i.test(p)) { - return p; - } - // Otherwise trim trailing slash - return p.substr(0, p.length - 1); -} -exports.safeTrimTrailingSeparator = safeTrimTrailingSeparator; -//# sourceMappingURL=internal-path-helper.js.map +/***/ }), + +/***/ 6113: +/***/ ((module) => { + +"use strict"; +module.exports = require("crypto"); + +/***/ }), + +/***/ 2361: +/***/ ((module) => { + +"use strict"; +module.exports = require("events"); + +/***/ }), + +/***/ 7147: +/***/ ((module) => { + +"use strict"; +module.exports = require("fs"); + +/***/ }), + +/***/ 3685: +/***/ ((module) => { + +"use strict"; +module.exports = require("http"); + +/***/ }), + +/***/ 5687: +/***/ ((module) => { + +"use strict"; +module.exports = require("https"); + +/***/ }), + +/***/ 1808: +/***/ ((module) => { + +"use strict"; +module.exports = require("net"); + +/***/ }), + +/***/ 2037: +/***/ ((module) => { + +"use strict"; +module.exports = require("os"); + +/***/ }), + +/***/ 1017: +/***/ ((module) => { + +"use strict"; +module.exports = require("path"); + +/***/ }), + +/***/ 2781: +/***/ ((module) => { + +"use strict"; +module.exports = require("stream"); + +/***/ }), + +/***/ 4404: +/***/ ((module) => { + +"use strict"; +module.exports = require("tls"); + +/***/ }), + +/***/ 3837: +/***/ ((module) => { + +"use strict"; +module.exports = require("util"); /***/ }) -/******/ }); \ No newline at end of file +/******/ }); +/************************************************************************/ +/******/ // The module cache +/******/ var __webpack_module_cache__ = {}; +/******/ +/******/ // The require function +/******/ function __nccwpck_require__(moduleId) { +/******/ // Check if module is in cache +/******/ var cachedModule = __webpack_module_cache__[moduleId]; +/******/ if (cachedModule !== undefined) { +/******/ return cachedModule.exports; +/******/ } +/******/ // Create a new module (and put it into the cache) +/******/ var module = __webpack_module_cache__[moduleId] = { +/******/ // no module.id needed +/******/ // no module.loaded needed +/******/ exports: {} +/******/ }; +/******/ +/******/ // Execute the module function +/******/ var threw = true; +/******/ try { +/******/ __webpack_modules__[moduleId].call(module.exports, module, module.exports, __nccwpck_require__); +/******/ threw = false; +/******/ } finally { +/******/ if(threw) delete __webpack_module_cache__[moduleId]; +/******/ } +/******/ +/******/ // Return the exports of the module +/******/ return module.exports; +/******/ } +/******/ +/************************************************************************/ +/******/ /* webpack/runtime/compat */ +/******/ +/******/ if (typeof __nccwpck_require__ !== 'undefined') __nccwpck_require__.ab = __dirname + "/"; +/******/ +/************************************************************************/ +/******/ +/******/ // startup +/******/ // Load entry module and return exports +/******/ // This entry module is referenced by other modules so it can't be inlined +/******/ var __webpack_exports__ = __nccwpck_require__(2627); +/******/ module.exports = __webpack_exports__; +/******/ +/******/ })() +; \ No newline at end of file From f53e06657373fae8126e14c9cb8744ce94b65a3e Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 18 Jan 2023 15:03:24 -0500 Subject: [PATCH 082/145] Prepare 2.301.0 runner release. (#2382) * Prepare 2.301.0 runner release. * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * Update releaseNote.md Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> --- releaseNote.md | 21 ++++++++++++--------- src/runnerversion | 2 +- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 42999201d96..44242944146 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,17 +1,20 @@ ## Features -- Expose github.actor_id, github.workflow_ref & github.workflow_sha as environment variable (#2249) -- Added worker and listener logs to stdout (#2291, #2307) +- Log GitHub RequestId for better traceability (#2332) +- Dual upload summary to Actions and Result service (#2334) +- Allow providing extra User-Agent for better correlation (#2370) +- Show more information in the runner log (#2377) +- New option to remove local config files (#2367) ## Bugs -- Made github.action_status output lowercase to be consistent with job.status' output (#1944) +- Split by regex (#2333) +- Treat jitconfig as secret (#2335) +- Add Header/Footer to multi-line message in StdoutTraceListener (#2336) +- Update Node dependencies (#2381) ## Misc -- Added small size runner image for ARC (#2250) -- Small change to Node.js 12 deprecation message (#2262) -- Added the option to use the --replace argument to the create-latest-svc.sh (#2273) -- Made runner_name optional defaulting to hostname in delete.sh script (#1871) -- Return exit code when MANUALLY_TRAP_SIG is exported (#2285) -- Use results for uploading step summaries (#2301) with limited size (#2321) +- Make runner image print diag log to STDOUT (#2331) +- Update Node.js to 16.16.0 (#2371) +- Add a disclaimer for which runner version is available to a given tenant (#2362) _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index 127ceb9c83d..bf655f6424b 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.300.2 +2.301.0 From 1b73815ba480d0f9a395dabf8ec3db10d094aade Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 18 Jan 2023 19:32:24 -0500 Subject: [PATCH 083/145] Revert "split by regex (#2333)" (#2383) This reverts commit 72830cfc122a12e80c1c5d73c9df60ba435851a9. --- src/Runner.Sdk/ProcessInvoker.cs | 5 +-- src/Test/L0/ProcessInvokerL0.cs | 68 -------------------------------- 2 files changed, 2 insertions(+), 71 deletions(-) diff --git a/src/Runner.Sdk/ProcessInvoker.cs b/src/Runner.Sdk/ProcessInvoker.cs index de980463944..e46b2a057aa 100644 --- a/src/Runner.Sdk/ProcessInvoker.cs +++ b/src/Runner.Sdk/ProcessInvoker.cs @@ -11,7 +11,6 @@ using System.Threading.Channels; using System.Threading.Tasks; using GitHub.Runner.Sdk; -using System.Text.RegularExpressions; namespace GitHub.Runner.Sdk { @@ -266,8 +265,8 @@ public async Task ExecuteAsync( foreach (KeyValuePair kvp in environment) { #if OS_WINDOWS - string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : Regex.Split(kvp.Key, @"\p{C}")[0]; - string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : Regex.Split(kvp.Value, @"\p{C}")[0]; + string tempKey = String.IsNullOrWhiteSpace(kvp.Key) ? kvp.Key : kvp.Key.Split('\0')[0]; + string tempValue = String.IsNullOrWhiteSpace(kvp.Value) ? kvp.Value : kvp.Value.Split('\0')[0]; if(!String.IsNullOrWhiteSpace(tempKey)) { _proc.StartInfo.Environment[tempKey] = tempValue; diff --git a/src/Test/L0/ProcessInvokerL0.cs b/src/Test/L0/ProcessInvokerL0.cs index cd958e2f9de..6d99932e3e2 100644 --- a/src/Test/L0/ProcessInvokerL0.cs +++ b/src/Test/L0/ProcessInvokerL0.cs @@ -164,40 +164,6 @@ public async Task SetTestEnvWithNullInKey() } } - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public async Task SetTestEnvWithTabInKey() - { - using (TestHostContext hc = new(this)) - { - Tracing trace = hc.GetTrace(); - - Int32 exitCode = -1; - var processInvoker = new ProcessInvokerWrapper(); - processInvoker.Initialize(hc); - var stdout = new List(); - var stderr = new List(); - processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => - { - trace.Info(e.Data); - stdout.Add(e.Data); - }; - processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => - { - trace.Info(e.Data); - stderr.Add(e.Data); - }; - - exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary() { { "TEST\u0009second", "first" } }, CancellationToken.None); - - trace.Info("Exit Code: {0}", exitCode); - Assert.Equal(0, exitCode); - Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x))); - - } - } - [Fact] [Trait("Level", "L0")] [Trait("Category", "Common")] @@ -231,40 +197,6 @@ public async Task SetTestEnvWithNullInValue() } } - - [Fact] - [Trait("Level", "L0")] - [Trait("Category", "Common")] - public async Task SetTestEnvWithTabInValue() - { - using (TestHostContext hc = new(this)) - { - Tracing trace = hc.GetTrace(); - - Int32 exitCode = -1; - var processInvoker = new ProcessInvokerWrapper(); - processInvoker.Initialize(hc); - var stdout = new List(); - var stderr = new List(); - processInvoker.OutputDataReceived += (object sender, ProcessDataReceivedEventArgs e) => - { - trace.Info(e.Data); - stdout.Add(e.Data); - }; - processInvoker.ErrorDataReceived += (object sender, ProcessDataReceivedEventArgs e) => - { - trace.Info(e.Data); - stderr.Add(e.Data); - }; - - exitCode = await processInvoker.ExecuteAsync("", "cmd.exe", "/c \"echo %TEST%\"", new Dictionary() { { "TEST", "first\u0009second" } }, CancellationToken.None); - - trace.Info("Exit Code: {0}", exitCode); - Assert.Equal(0, exitCode); - Assert.Equal("first", stdout.First(x => !string.IsNullOrWhiteSpace(x))); - - } - } #endif [Fact] [Trait("Level", "L0")] From 14b88826cfca3bae63891051237c795087131e19 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 18 Jan 2023 19:40:33 -0500 Subject: [PATCH 084/145] Bump runner version to match the released runner. (#2385) * Bump runner version to match the released runner. * . --- releaseNote.md | 1 - src/runnerversion | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 44242944146..a6021fb1526 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -6,7 +6,6 @@ - New option to remove local config files (#2367) ## Bugs -- Split by regex (#2333) - Treat jitconfig as secret (#2335) - Add Header/Footer to multi-line message in StdoutTraceListener (#2336) - Update Node dependencies (#2381) diff --git a/src/runnerversion b/src/runnerversion index bf655f6424b..b886075fa2b 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.301.0 +2.301.1 From b42d461a9f92b7a8392c7ef7f9f75fa3564c1c03 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 23 Jan 2023 13:07:49 -0500 Subject: [PATCH 085/145] Bump dotnet sdk to latest version. (#2392) * Bump dotnet sdk to latest version. * . * . * . * . * . * . * . * . --- .devcontainer/devcontainer.json | 2 +- src/Misc/contentHash/dotnetRuntime/linux-arm | 2 +- src/Misc/contentHash/dotnetRuntime/linux-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/linux-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/osx-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/osx-x64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-arm64 | 2 +- src/Misc/contentHash/dotnetRuntime/win-x64 | 2 +- src/Misc/runnerdotnetruntimeassets | 1 + src/dev.sh | 2 +- src/global.json | 2 +- 11 files changed, 11 insertions(+), 10 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index ec8b7d2831a..aaff41fe100 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -5,7 +5,7 @@ "features": { "ghcr.io/devcontainers/features/docker-in-docker:1": {}, "ghcr.io/devcontainers/features/dotnet": { - "version": "6.0.300" + "version": "6.0.405" }, "ghcr.io/devcontainers/features/node:1": { "version": "16" diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm b/src/Misc/contentHash/dotnetRuntime/linux-arm index 548e328e201..dbfd8434247 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm @@ -1 +1 @@ -1d709d93e5d3c6c6c656a61aa6c1781050224788a05b0e6ecc4c3c0408bdf89c \ No newline at end of file +39f2a931565d6a10e695ac8ed14bb9dcbb568151410349b32dbf9c27bae29602 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-arm64 b/src/Misc/contentHash/dotnetRuntime/linux-arm64 index 6b2e4f7cce6..1115e60a40c 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-arm64 @@ -1 +1 @@ -b92a47cfeaad02255b1f7a377060651b73ae5e5db22a188dbbcb4183ab03a03d \ No newline at end of file +29ffb303537d8ba674fbebc7729292c21c4ebd17b3198f91ed593ef4cbbb67b5 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/linux-x64 b/src/Misc/contentHash/dotnetRuntime/linux-x64 index b92a76c856a..506b437a4a4 100644 --- a/src/Misc/contentHash/dotnetRuntime/linux-x64 +++ b/src/Misc/contentHash/dotnetRuntime/linux-x64 @@ -1 +1 @@ -68a9a8ef0843a8bb74241894f6f63fd76241a82295c5337d3cc7a940a314c78e \ No newline at end of file +de6868a836fa3cb9e5ddddbc079da1c25e819aa2d2fc193cc9931c353687c57c \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-arm64 b/src/Misc/contentHash/dotnetRuntime/osx-arm64 index 04681bcfcc0..e7e6bec44b5 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-arm64 @@ -1 +1 @@ -02c7126ff4d63ee2a0ae390c81434c125630522aadf35903bbeebb1a99d8af99 \ No newline at end of file +339d3e1a5fd28450c0fe6cb820cc7aae291f0f9e2d153ac34e1f7b080e35d30e \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/osx-x64 b/src/Misc/contentHash/dotnetRuntime/osx-x64 index 9da58b8a02a..7df68786a23 100644 --- a/src/Misc/contentHash/dotnetRuntime/osx-x64 +++ b/src/Misc/contentHash/dotnetRuntime/osx-x64 @@ -1 +1 @@ -c9d5a542f8d765168855a89e83ae0a8970d00869041c4f9a766651c04c72b212 \ No newline at end of file +dcb7f606c1d7d290381e5020ee73e7f16dcbd2f20ac9b431362ccbb5120d449c \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-arm64 b/src/Misc/contentHash/dotnetRuntime/win-arm64 index 8d279b0417b..0083dfe9cd8 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-arm64 +++ b/src/Misc/contentHash/dotnetRuntime/win-arm64 @@ -1 +1 @@ -39d0683f0f115a211cb10c473e9574c16549a19d4e9a6c637ded3d7022bf809f +1bbcb0e9a2cf4be4b1fce77458de139b70ac58efcbb415a6db028b9373ae1673 \ No newline at end of file diff --git a/src/Misc/contentHash/dotnetRuntime/win-x64 b/src/Misc/contentHash/dotnetRuntime/win-x64 index 5eaabf91898..447a48f1dc3 100644 --- a/src/Misc/contentHash/dotnetRuntime/win-x64 +++ b/src/Misc/contentHash/dotnetRuntime/win-x64 @@ -1 +1 @@ -d94f2fbaf210297162bc9f3add819d73682c3aa6899e321c3872412b924d5504 \ No newline at end of file +44cd25f3c104d0abb44d262397a80e0b2c4f206465c5d899a22eec043dac0fb3 \ No newline at end of file diff --git a/src/Misc/runnerdotnetruntimeassets b/src/Misc/runnerdotnetruntimeassets index fa2d9e04b4c..a6a407c3ff3 100644 --- a/src/Misc/runnerdotnetruntimeassets +++ b/src/Misc/runnerdotnetruntimeassets @@ -74,6 +74,7 @@ Microsoft.Win32.Registry.dll mscordaccore.dll mscordaccore_amd64_amd64_6.0.522.21309.dll mscordaccore_arm64_arm64_6.0.522.21309.dll +mscordaccore_amd64_amd64_6.0.1322.58009.dll mscordbi.dll mscorlib.dll mscorrc.debug.dll diff --git a/src/dev.sh b/src/dev.sh index 82abcf769ef..fed033d3886 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -22,7 +22,7 @@ DOWNLOAD_DIR="$SCRIPT_DIR/../_downloads/netcore2x" PACKAGE_DIR="$SCRIPT_DIR/../_package" PACKAGE_TRIMS_DIR="$SCRIPT_DIR/../_package_trims" DOTNETSDK_ROOT="$SCRIPT_DIR/../_dotnetsdk" -DOTNETSDK_VERSION="6.0.300" +DOTNETSDK_VERSION="6.0.405" DOTNETSDK_INSTALLDIR="$DOTNETSDK_ROOT/$DOTNETSDK_VERSION" RUNNER_VERSION=$(cat runnerversion) diff --git a/src/global.json b/src/global.json index 19c87f2a583..5f5ece16508 100644 --- a/src/global.json +++ b/src/global.json @@ -1,5 +1,5 @@ { "sdk": { - "version": "6.0.300" + "version": "6.0.405" } } From 380d752dc5b2a8baeb210c69f57986e6f3bf1a8f Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Mon, 30 Jan 2023 10:00:31 -0500 Subject: [PATCH 086/145] fix small bug (#2396) --- src/Runner.Common/RunServer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index c4ccb649441..87904505586 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -29,7 +29,7 @@ public async Task ConnectAsync(Uri serverUri, VssCredentials credentials) { requestUri = serverUri; - _connection = VssUtil.CreateRawConnection(new Uri(serverUri.Authority), credentials); + _connection = VssUtil.CreateRawConnection(serverUri, credentials); _runServiceHttpClient = await _connection.GetClientAsync(); _hasConnection = true; } From 3aabb2ff81e8ac5b91a5c9d14d5f0b129b501778 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Wed, 1 Feb 2023 16:18:31 -0500 Subject: [PATCH 087/145] start calling run service for job completion (#2412) * start calling run service for job completion * cleanup * nit: lines Co-authored-by: Tingluo Huang * clean up * give sanity back to thboop Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> * add clean up back * clean up * clean up more * oops * copied from existing, but :thumb: Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> --------- Co-authored-by: Tingluo Huang Co-authored-by: Thomas Boop <52323235+thboop@users.noreply.github.com> --- src/Runner.Common/RunServer.cs | 9 +- src/Runner.Common/RunnerService.cs | 15 ++- src/Runner.Common/Unit.cs | 8 ++ src/Runner.Sdk/Util/VssUtil.cs | 2 +- src/Runner.Worker/JobRunner.cs | 108 +++++++++++++++--- .../Authentication/VssCredentialsExtension.cs | 20 ---- .../Common/Common/RawHttpMessageHandler.cs | 12 +- .../Pipelines/AgentJobRequestMessage.cs | 5 +- .../DTWebApi/WebApi/RunServiceHttpClient.cs | 26 ++++- src/Sdk/WebApi/WebApi/RawHttpClientBase.cs | 11 ++ src/Test/L0/Worker/JobRunnerL0.cs | 86 +++++++++----- 11 files changed, 222 insertions(+), 80 deletions(-) create mode 100644 src/Runner.Common/Unit.cs delete mode 100644 src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 87904505586..0f9fa1a018b 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -5,7 +5,6 @@ using GitHub.DistributedTask.WebApi; using GitHub.Runner.Sdk; using GitHub.Services.Common; -using GitHub.Services.WebApi; using Sdk.WebApi.WebApi.RawClient; namespace GitHub.Runner.Common @@ -16,6 +15,8 @@ public interface IRunServer : IRunnerService Task ConnectAsync(Uri serverUrl, VssCredentials credentials); Task GetJobMessageAsync(string id, CancellationToken token); + + Task CompleteJobAsync(Guid planId, Guid jobId, CancellationToken token); } public sealed class RunServer : RunnerService, IRunServer @@ -55,5 +56,11 @@ public Task GetJobMessageAsync(string id, CancellationTo return jobMessage; } + public Task CompleteJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) + { + CheckConnection(); + return RetryRequest( + async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, cancellationToken), cancellationToken); + } } } diff --git a/src/Runner.Common/RunnerService.cs b/src/Runner.Common/RunnerService.cs index 403764b036c..a0cb96a0ecf 100644 --- a/src/Runner.Common/RunnerService.cs +++ b/src/Runner.Common/RunnerService.cs @@ -68,6 +68,19 @@ protected async Task EstablishVssConnection(Uri serverUrl, VssCre throw new InvalidOperationException(nameof(EstablishVssConnection)); } + protected async Task RetryRequest(Func func, + CancellationToken cancellationToken, + int maxRetryAttemptsCount = 5 + ) + { + async Task wrappedFunc() + { + await func(); + return Unit.Value; + } + await RetryRequest(wrappedFunc, cancellationToken, maxRetryAttemptsCount); + } + protected async Task RetryRequest(Func> func, CancellationToken cancellationToken, int maxRetryAttemptsCount = 5 @@ -85,7 +98,7 @@ protected async Task RetryRequest(Func> func, // TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122 catch (Exception ex) when (retryCount < maxRetryAttemptsCount) { - Trace.Error("Catch exception during get full job message"); + Trace.Error("Catch exception during request"); Trace.Error(ex); var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); Trace.Warning($"Back off {backOff.TotalSeconds} seconds before next retry. {maxRetryAttemptsCount - retryCount} attempt left."); diff --git a/src/Runner.Common/Unit.cs b/src/Runner.Common/Unit.cs new file mode 100644 index 00000000000..31f4ad55fdf --- /dev/null +++ b/src/Runner.Common/Unit.cs @@ -0,0 +1,8 @@ +// Represents absence of value. +namespace GitHub.Runner.Common +{ + public readonly struct Unit + { + public static readonly Unit Value = default; + } +} \ No newline at end of file diff --git a/src/Runner.Sdk/Util/VssUtil.cs b/src/Runner.Sdk/Util/VssUtil.cs index 6bfc7f131f4..12cc5c11826 100644 --- a/src/Runner.Sdk/Util/VssUtil.cs +++ b/src/Runner.Sdk/Util/VssUtil.cs @@ -116,7 +116,7 @@ public static RawConnection CreateRawConnection( // settings are applied to an HttpRequestMessage. settings.AcceptLanguages.Remove(CultureInfo.InvariantCulture); - RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.ToOAuthCredentials(), settings), additionalDelegatingHandler); + RawConnection connection = new(serverUri, new RawHttpMessageHandler(credentials.Federated, settings), additionalDelegatingHandler); return connection; } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 562ef9068fd..49150c5ca28 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -6,7 +6,6 @@ using System.Text; using System.Threading; using System.Threading.Tasks; -using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; using GitHub.Runner.Common.Util; @@ -40,21 +39,34 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, Trace.Info("Job ID {0}", message.JobId); DateTime jobStartTimeUtc = DateTime.UtcNow; + IRunnerService server = null; ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase)) + { + var runServer = HostContext.GetService(); + VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); + await runServer.ConnectAsync(systemConnection.Url, jobServerCredential); + server = runServer; + } + else + { + // Setup the job server and job server queue. + var jobServer = HostContext.GetService(); + VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); + Uri jobServerUrl = systemConnection.Url; + + Trace.Info($"Creating job server with URL: {jobServerUrl}"); + // jobServerQueue is the throttling reporter. + _jobServerQueue = HostContext.GetService(); + VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) }); + await jobServer.ConnectAsync(jobConnection); + + _jobServerQueue.Start(message); + server = jobServer; + } + - // Setup the job server and job server queue. - var jobServer = HostContext.GetService(); - VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); - Uri jobServerUrl = systemConnection.Url; - - Trace.Info($"Creating job server with URL: {jobServerUrl}"); - // jobServerQueue is the throttling reporter. - _jobServerQueue = HostContext.GetService(); - VssConnection jobConnection = VssUtil.CreateConnection(jobServerUrl, jobServerCredential, new DelegatingHandler[] { new ThrottlingReportHandler(_jobServerQueue) }); - await jobServer.ConnectAsync(jobConnection); - - _jobServerQueue.Start(message); HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); IExecutionContext jobContext = null; @@ -99,7 +111,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, { Trace.Error(ex); jobContext.Error(ex); - return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed); + return await CompleteJobAsync(server, jobContext, message, TaskResult.Failed); } if (jobContext.Global.WriteDebug) @@ -136,7 +148,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job is cancelled during initialize."); Trace.Error($"Caught exception: {ex}"); - return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Canceled); + return await CompleteJobAsync(server, jobContext, message, TaskResult.Canceled); } catch (Exception ex) { @@ -144,7 +156,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, // don't log error issue to job ExecutionContext, since server owns the job level issue Trace.Error($"Job initialize failed."); Trace.Error($"Caught exception from {nameof(jobExtension.InitializeJob)}: {ex}"); - return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed); + return await CompleteJobAsync(server, jobContext, message, TaskResult.Failed); } // trace out all steps @@ -181,7 +193,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, // Log the error and fail the job. Trace.Error($"Caught exception from job steps {nameof(StepsRunner)}: {ex}"); jobContext.Error(ex); - return await CompleteJobAsync(jobServer, jobContext, message, TaskResult.Failed); + return await CompleteJobAsync(server, jobContext, message, TaskResult.Failed); } finally { @@ -192,7 +204,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, Trace.Info($"Job result after all job steps finish: {jobContext.Result ?? TaskResult.Succeeded}"); Trace.Info("Completing the job execution context."); - return await CompleteJobAsync(jobServer, jobContext, message); + return await CompleteJobAsync(server, jobContext, message); } finally { @@ -206,6 +218,66 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, } } + private async Task CompleteJobAsync(IRunnerService server, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null) + { + if (server is IRunServer runServer) + { + return await CompleteJobAsync(runServer, jobContext, message, taskResult); + } + else if (server is IJobServer jobServer) + { + return await CompleteJobAsync(jobServer, jobContext, message, taskResult); + } + else + { + throw new NotSupportedException(); + } + } + + private async Task CompleteJobAsync(IRunServer runServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null) + { + jobContext.Debug($"Finishing: {message.JobDisplayName}"); + TaskResult result = jobContext.Complete(taskResult); + if (jobContext.Global.Variables.TryGetValue("Node12ActionsWarnings", out var node12Warnings)) + { + var actions = string.Join(", ", StringUtil.ConvertFromJson>(node12Warnings)); + jobContext.Warning(string.Format(Constants.Runner.Node12DetectedAfterEndOfLife, actions)); + } + + // Make sure to clean temp after file upload since they may be pending fileupload still use the TEMP dir. + _tempDirectoryManager?.CleanupTempDirectory(); + + // Load any upgrade telemetry + LoadFromTelemetryFile(jobContext.Global.JobTelemetry); + + // Make sure we don't submit secrets as telemetry + MaskTelemetrySecrets(jobContext.Global.JobTelemetry); + + Trace.Info($"Raising job completed against run service"); + var completeJobRetryLimit = 5; + var exceptions = new List(); + while (completeJobRetryLimit-- > 0) + { + try + { + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, default); + return result; + } + catch (Exception ex) + { + Trace.Error($"Catch exception while attempting to complete job {message.JobId}, job request {message.RequestId}."); + Trace.Error(ex); + exceptions.Add(ex); + } + + // delay 5 seconds before next retry. + await Task.Delay(TimeSpan.FromSeconds(5)); + } + + // rethrow exceptions from all attempts. + throw new AggregateException(exceptions); + } + private async Task CompleteJobAsync(IJobServer jobServer, IExecutionContext jobContext, Pipelines.AgentJobRequestMessage message, TaskResult? taskResult = null) { jobContext.Debug($"Finishing: {message.JobDisplayName}"); diff --git a/src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs b/src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs deleted file mode 100644 index c91fdb70a2e..00000000000 --- a/src/Sdk/Common/Common/Authentication/VssCredentialsExtension.cs +++ /dev/null @@ -1,20 +0,0 @@ -using GitHub.Services.OAuth; - -namespace GitHub.Services.Common -{ - public static class VssCredentialsExtension - { - public static VssOAuthCredential ToOAuthCredentials( - this VssCredentials credentials) - { - if (credentials.Federated.CredentialType == VssCredentialsType.OAuth) - { - return credentials.Federated as VssOAuthCredential; - } - else - { - return null; - } - } - } -} diff --git a/src/Sdk/Common/Common/RawHttpMessageHandler.cs b/src/Sdk/Common/Common/RawHttpMessageHandler.cs index 286fac8403e..8b3e75e1844 100644 --- a/src/Sdk/Common/Common/RawHttpMessageHandler.cs +++ b/src/Sdk/Common/Common/RawHttpMessageHandler.cs @@ -12,20 +12,20 @@ namespace GitHub.Services.Common public class RawHttpMessageHandler: HttpMessageHandler { public RawHttpMessageHandler( - VssOAuthCredential credentials) + FederatedCredential credentials) : this(credentials, new RawClientHttpRequestSettings()) { } public RawHttpMessageHandler( - VssOAuthCredential credentials, + FederatedCredential credentials, RawClientHttpRequestSettings settings) : this(credentials, settings, new HttpClientHandler()) { } public RawHttpMessageHandler( - VssOAuthCredential credentials, + FederatedCredential credentials, RawClientHttpRequestSettings settings, HttpMessageHandler innerHandler) { @@ -56,7 +56,7 @@ public RawHttpMessageHandler( /// /// Gets the credentials associated with this handler. /// - public VssOAuthCredential Credentials + public FederatedCredential Credentials { get; private set; @@ -111,7 +111,7 @@ protected override async Task SendAsync( // Ensure that we attempt to use the most appropriate authentication mechanism by default. if (m_tokenProvider == null) { - m_tokenProvider = this.Credentials.GetTokenProvider(request.RequestUri); + m_tokenProvider = this.Credentials.CreateTokenProvider(request.RequestUri, null, null); } } @@ -254,7 +254,7 @@ private static void ApplySettings( private CredentialWrapper m_credentialWrapper; private object m_thisLock; private const Int32 m_maxAuthRetries = 3; - private VssOAuthTokenProvider m_tokenProvider; + private IssuedTokenProvider m_tokenProvider; //.Net Core does not attempt NTLM schema on Linux, unless ICredentials is a CredentialCache instance //This workaround may not be needed after this corefx fix is consumed: https://github.com/dotnet/corefx/pull/7923 diff --git a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs index 74650a12ede..3f93e75e53c 100644 --- a/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs +++ b/src/Sdk/DTPipelines/Pipelines/AgentJobRequestMessage.cs @@ -42,9 +42,10 @@ public AgentJobRequestMessage( IList fileTable, TemplateToken jobOutputs, IList defaults, - ActionsEnvironmentReference actionsEnvironment) + ActionsEnvironmentReference actionsEnvironment, + String messageType = JobRequestMessageTypes.PipelineAgentJobRequest) { - this.MessageType = JobRequestMessageTypes.PipelineAgentJobRequest; + this.MessageType = messageType; this.Plan = plan; this.JobId = jobId; this.JobDisplayName = jobDisplayName; diff --git a/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs b/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs index 64c1784e6d1..7db46d7be87 100644 --- a/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs +++ b/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs @@ -62,14 +62,38 @@ public RunServiceHttpClient( StreamID = messageId }; + requestUri = new Uri(requestUri, "acquirejob"); + var payloadJson = JsonUtility.ToString(payload); var requestContent = new StringContent(payloadJson, System.Text.Encoding.UTF8, "application/json"); return SendAsync( httpMethod, - additionalHeaders: null, requestUri: requestUri, content: requestContent, cancellationToken: cancellationToken); } + + public Task CompleteJobAsync( + Uri requestUri, + Guid planId, + Guid jobId, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + var payload = new { + PlanId = planId, + JobId = jobId + }; + + requestUri = new Uri(requestUri, "completejob"); + + var payloadJson = JsonUtility.ToString(payload); + var requestContent = new StringContent(payloadJson, System.Text.Encoding.UTF8, "application/json"); + return SendAsync( + httpMethod, + requestUri, + content: requestContent, + cancellationToken: cancellationToken); + } } } diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs index ffeecc8084e..8054a81145a 100644 --- a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs +++ b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs @@ -101,6 +101,17 @@ protected async Task SendAsync( } } + protected Task SendAsync( + HttpMethod method, + Uri requestUri, + HttpContent content = null, + IEnumerable> queryParameters = null, + Object userState = null, + CancellationToken cancellationToken = default(CancellationToken)) + { + return SendAsync(method, null, requestUri, content, queryParameters, userState, cancellationToken); + } + protected async Task SendAsync( HttpMethod method, IEnumerable> additionalHeaders, diff --git a/src/Test/L0/Worker/JobRunnerL0.cs b/src/Test/L0/Worker/JobRunnerL0.cs index 152c816aa2b..42616e3d580 100644 --- a/src/Test/L0/Worker/JobRunnerL0.cs +++ b/src/Test/L0/Worker/JobRunnerL0.cs @@ -16,9 +16,10 @@ public sealed class JobRunnerL0 private IExecutionContext _jobEc; private JobRunner _jobRunner; private List _initResult = new(); - private Pipelines.AgentJobRequestMessage _message; private CancellationTokenSource _tokenSource; private Mock _jobServer; + + private Mock _runServer; private Mock _jobServerQueue; private Mock _config; private Mock _extensions; @@ -38,6 +39,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " _extensions = new Mock(); _jobExtension = new Mock(); _jobServer = new Mock(); + _runServer = new Mock(); _jobServerQueue = new Mock(); _stepRunner = new Mock(); _logger = new Mock(); @@ -55,33 +57,6 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " _jobRunner = new JobRunner(); _jobRunner.Initialize(hc); - TaskOrchestrationPlanReference plan = new(); - TimelineReference timeline = new Timeline(Guid.NewGuid()); - Guid jobId = Guid.NewGuid(); - _message = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, testName, testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); - _message.Variables[Constants.Variables.System.Culture] = "en-US"; - _message.Resources.Endpoints.Add(new ServiceEndpoint() - { - Name = WellKnownServiceEndpointNames.SystemVssConnection, - Url = new Uri("https://pipelines.actions.githubusercontent.com"), - Authorization = new EndpointAuthorization() - { - Scheme = "Test", - Parameters = { - {"AccessToken", "token"} - } - }, - - }); - - _message.Resources.Repositories.Add(new Pipelines.RepositoryResource() - { - Alias = Pipelines.PipelineConstants.SelfAlias, - Id = "github", - Version = "sha1" - }); - _message.ContextData.Add("github", new Pipelines.ContextData.DictionaryContextData()); - _initResult.Clear(); _jobExtension.Setup(x => x.InitializeJob(It.IsAny(), It.IsAny())). @@ -102,6 +77,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " hc.SetSingleton(_config.Object); hc.SetSingleton(_jobServer.Object); + hc.SetSingleton(_runServer.Object); hc.SetSingleton(_jobServerQueue.Object); hc.SetSingleton(_stepRunner.Object); hc.SetSingleton(_extensions.Object); @@ -113,6 +89,43 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " return hc; } + private Pipelines.AgentJobRequestMessage GetMessage(String messageType = JobRequestMessageTypes.PipelineAgentJobRequest, [CallerMemberName] String testName = "") + { + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new Timeline(Guid.NewGuid()); + Guid jobId = Guid.NewGuid(); + var message = new Pipelines.AgentJobRequestMessage( + plan, + timeline, + jobId, + testName, + testName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null, + messageType: messageType); + message.Variables[Constants.Variables.System.Culture] = "en-US"; + message.Resources.Endpoints.Add(new ServiceEndpoint() + { + Name = WellKnownServiceEndpointNames.SystemVssConnection, + Url = new Uri("https://pipelines.actions.githubusercontent.com"), + Authorization = new EndpointAuthorization() + { + Scheme = "Test", + Parameters = { + {"AccessToken", "token"} + } + }, + + }); + + message.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + message.ContextData.Add("github", new Pipelines.ContextData.DictionaryContextData()); + return message; + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -123,7 +136,7 @@ public async Task JobExtensionInitializeFailure() _jobExtension.Setup(x => x.InitializeJob(It.IsAny(), It.IsAny())) .Throws(new Exception()); - await _jobRunner.RunAsync(_message, _tokenSource.Token); + await _jobRunner.RunAsync(GetMessage(), _tokenSource.Token); Assert.Equal(TaskResult.Failed, _jobEc.Result); _stepRunner.Verify(x => x.RunAsync(It.IsAny()), Times.Never); @@ -141,11 +154,24 @@ public async Task JobExtensionInitializeCancelled() .Throws(new OperationCanceledException()); _tokenSource.Cancel(); - await _jobRunner.RunAsync(_message, _tokenSource.Token); + await _jobRunner.RunAsync(GetMessage(), _tokenSource.Token); Assert.Equal(TaskResult.Canceled, _jobEc.Result); _stepRunner.Verify(x => x.RunAsync(It.IsAny()), Times.Never); } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public async Task WorksWithRunnerJobRequestMessageType() + { + using (TestHostContext hc = CreateTestContext()) + { + var message = GetMessage(JobRequestMessageTypes.RunnerJobRequest); + await _jobRunner.RunAsync(message, _tokenSource.Token); + Assert.Equal(TaskResult.Succeeded, _jobEc.Result); + } + } } } From 7262b18ee20a3babe3dbf4930de4ddf2c88f7027 Mon Sep 17 00:00:00 2001 From: Erez Testiler Date: Mon, 6 Feb 2023 17:16:38 -0500 Subject: [PATCH 088/145] Add support for ghe.com domain (#2420) --- src/Runner.Sdk/Util/UrlUtil.cs | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/Runner.Sdk/Util/UrlUtil.cs b/src/Runner.Sdk/Util/UrlUtil.cs index 4e74c98201e..2c932e8c1f1 100644 --- a/src/Runner.Sdk/Util/UrlUtil.cs +++ b/src/Runner.Sdk/Util/UrlUtil.cs @@ -6,9 +6,16 @@ public static class UrlUtil { public static bool IsHostedServer(UriBuilder gitHubUrl) { - return string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) || + if (StringUtil.ConvertToBoolean(Environment.GetEnvironmentVariable("GITHUB_ACTIONS_RUNNER_FORCE_GHES"))) + { + return false; + } + + return + string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) || string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || - string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase); + string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase) || + gitHubUrl.Host.EndsWith(".ghe.com", StringComparison.OrdinalIgnoreCase); } public static Uri GetCredentialEmbeddedUrl(Uri baseUrl, string username, string password) From 288e7f886ba7e73bbfc0387c7b5c19cbb23ea81c Mon Sep 17 00:00:00 2001 From: John Wesley Walker III <81404201+jww3@users.noreply.github.com> Date: Tue, 7 Feb 2023 11:42:30 +0100 Subject: [PATCH 089/145] Defer evaluation of a step's DisplayName until its condition is evaluated. (#2313) * Defer evaluation of a step's DisplayName until its condition is evaluated. * Formalize TryUpdateDisplayName and EvaluateDisplayName as members of interface `IStep` (#2374) --- src/Runner.Worker/ActionRunner.cs | 57 +++++++++++++++++--- src/Runner.Worker/JobExtension.cs | 8 +-- src/Runner.Worker/JobExtensionRunner.cs | 14 +++++ src/Runner.Worker/StepsRunner.cs | 21 +++----- src/Test/L0/Worker/ActionRunnerL0.cs | 70 +++++++++++++++++++------ 5 files changed, 128 insertions(+), 42 deletions(-) diff --git a/src/Runner.Worker/ActionRunner.cs b/src/Runner.Worker/ActionRunner.cs index 257d6010942..62fbbb812d6 100644 --- a/src/Runner.Worker/ActionRunner.cs +++ b/src/Runner.Worker/ActionRunner.cs @@ -25,7 +25,6 @@ public enum ActionRunStage public interface IActionRunner : IStep, IRunnerService { ActionRunStage Stage { get; set; } - bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context); Pipelines.ActionStep Action { get; set; } } @@ -285,25 +284,67 @@ Action.Reference is Pipelines.RepositoryPathReference repoAction && } - public bool TryEvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context) + /// + /// Attempts to update the DisplayName. + /// As the "Try..." name implies, this method should never throw an exception. + /// Returns true if the DisplayName is already present or it was successfully updated. + /// + public bool TryUpdateDisplayName(out bool updated) + { + updated = false; + + // REVIEW: This try/catch can be removed if some future implementation of EvaluateDisplayName and UpdateTimelineRecordDisplayName + // can make reasonable guarantees that they won't throw an exception. + try + { + // This attempt is only worthwhile at the "Main" stage. + // When the job starts, there's an initial attempt to evaluate the DisplayName. (see JobExtension::InitializeJob) + // During the "Pre" stage, we expect that no contexts will have changed since the initial evaluation. + // "Main" stage is handled here. + // During the "Post" stage, it no longer matters. + if (this.Stage == ActionRunStage.Main && EvaluateDisplayName(this.ExecutionContext.ExpressionValues, this.ExecutionContext, out updated)) + { + if (updated) + { + this.ExecutionContext.UpdateTimelineRecordDisplayName(this.DisplayName); + } + } + } + catch (Exception ex) + { + Trace.Warning("Caught exception while attempting to evaulate/update the step's DisplayName. Exception Details: {0}", ex); + } + + // For consistency with other implementations of TryUpdateDisplayName we use !string.IsNullOrEmpty below, + // but note that (at the time of this writing) ActionRunner::DisplayName::get always returns a non-empty string due to its fallback logic. + // In other words, the net effect is that this particular implementation of TryUpdateDisplayName will always return true. + return !string.IsNullOrEmpty(this.DisplayName); + } + + + /// + /// Attempts to evaluate the DisplayName of this IActionRunner. + /// Returns true if the DisplayName is already present or it was successfully evaluated. + /// + public bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated) { ArgUtil.NotNull(context, nameof(context)); ArgUtil.NotNull(Action, nameof(Action)); - // If we have already expanded the display name, there is no need to expand it again - // TODO: Remove the ShouldEvaluateDisplayName check and field post m158 deploy, we should do it by default once the server is updated + updated = false; + // If we have already expanded the display name, don't bother attempting [re-]expansion. if (_didFullyEvaluateDisplayName || !string.IsNullOrEmpty(Action.DisplayName)) { - return false; + return true; } - bool didFullyEvaluate; - _displayName = GenerateDisplayName(Action, contextData, context, out didFullyEvaluate); + _displayName = GenerateDisplayName(Action, contextData, context, out bool didFullyEvaluate); - // If we evaluated fully mask any secrets + // If we evaluated, fully mask any secrets if (didFullyEvaluate) { _displayName = HostContext.SecretMasker.MaskSecrets(_displayName); + updated = true; } context.Debug($"Set step '{Action.Name}' display name to: '{_displayName}'"); _didFullyEvaluateDisplayName = didFullyEvaluate; diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 770db66435d..580c9999dd7 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -306,13 +306,13 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel } } - actionRunner.TryEvaluateDisplayName(contextData, context); + actionRunner.EvaluateDisplayName(contextData, context, out _); jobSteps.Add(actionRunner); if (prepareResult.PreStepTracker.TryGetValue(step.Id, out var preStep)) { Trace.Info($"Adding pre-{action.DisplayName}."); - preStep.TryEvaluateDisplayName(contextData, context); + preStep.EvaluateDisplayName(contextData, context, out _); preStep.DisplayName = $"Pre {preStep.DisplayName}"; preJobSteps.Add(preStep); } @@ -328,10 +328,10 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel if (message.ContextData.TryGetValue("inputs", out var pipelineContextData)) { var inputs = pipelineContextData.AssertDictionary("inputs"); - if (inputs.Any()) + if (inputs.Any()) { context.Output($"##[group] Inputs"); - foreach (var input in inputs) + foreach (var input in inputs) { context.Output($" {input.Key}: {input.Value}"); } diff --git a/src/Runner.Worker/JobExtensionRunner.cs b/src/Runner.Worker/JobExtensionRunner.cs index 62fc3287fa3..eeb0a8393d9 100644 --- a/src/Runner.Worker/JobExtensionRunner.cs +++ b/src/Runner.Worker/JobExtensionRunner.cs @@ -1,6 +1,7 @@ using System; using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines.ContextData; namespace GitHub.Runner.Worker { @@ -32,5 +33,18 @@ public async Task RunAsync() { await _runAsync(ExecutionContext, _data); } + + public bool TryUpdateDisplayName(out bool updated) + { + updated = false; + return !string.IsNullOrEmpty(this.DisplayName); + } + + public bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated) + { + updated = false; + return !string.IsNullOrEmpty(this.DisplayName); + } + } } diff --git a/src/Runner.Worker/StepsRunner.cs b/src/Runner.Worker/StepsRunner.cs index dc3e80a25de..05d21d4bb06 100644 --- a/src/Runner.Worker/StepsRunner.cs +++ b/src/Runner.Worker/StepsRunner.cs @@ -1,12 +1,9 @@ using System; using System.Collections.Generic; -using System.Linq; -using System.Text; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; -using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.Pipelines.ObjectTemplating; using GitHub.DistributedTask.WebApi; @@ -14,8 +11,6 @@ using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Runner.Worker.Expressions; -using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; -using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Worker { @@ -26,6 +21,8 @@ public interface IStep string DisplayName { get; set; } IExecutionContext ExecutionContext { get; set; } TemplateToken Timeout { get; } + bool TryUpdateDisplayName(out bool updated); + bool EvaluateDisplayName(DictionaryContextData contextData, IExecutionContext context, out bool updated); Task RunAsync(); } @@ -195,6 +192,12 @@ public async Task RunAsync(IExecutionContext jobContext) } else { + // This is our last, best chance to expand the display name. (At this point, all the requirements for successful expansion should be met.) + // That being said, evaluating the display name should still be considered as a "best effort" exercise. (It's not critical or paramount.) + // For that reason, we call a safe "Try..." wrapper method to ensure that any potential problems we encounter in evaluating the display name + // don't interfere with our ultimate goal within this code block: evaluation of the condition. + step.TryUpdateDisplayName(out _); + try { var templateEvaluator = step.ExecutionContext.ToPipelineTemplateEvaluator(conditionTraceWriter); @@ -256,14 +259,6 @@ public async Task RunAsync(IExecutionContext jobContext) private async Task RunStepAsync(IStep step, CancellationToken jobCancellationToken) { - // Check to see if we can expand the display name - if (step is IActionRunner actionRunner && - actionRunner.Stage == ActionRunStage.Main && - actionRunner.TryEvaluateDisplayName(step.ExecutionContext.ExpressionValues, step.ExecutionContext)) - { - step.ExecutionContext.UpdateTimelineRecordDisplayName(actionRunner.DisplayName); - } - // Start the step Trace.Info("Starting the step."); step.ExecutionContext.Debug($"Starting: {step.DisplayName}"); diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index 31d0743659b..586e9dad2d4 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -3,20 +3,14 @@ using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; -using GitHub.Runner.Common.Util; using GitHub.Runner.Worker; -using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Handlers; using Moq; using Newtonsoft.Json.Linq; using System; using System.Collections.Generic; -using System.IO; -using System.IO.Compression; -using System.Reflection; using System.Runtime.CompilerServices; using System.Threading; -using System.Threading.Tasks; using Xunit; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -149,11 +143,12 @@ public void EvaluateLegacyDisplayName() _context.Add("matrix", matrixData); // Act - // Should not do anything if we don't have a displayNameToken to expand - var didUpdateDisplayName = _actionRunner.TryEvaluateDisplayName(_context, _actionRunner.ExecutionContext); + // Should report success with no updated required if there's already a valid display name. + var validDisplayName = _actionRunner.EvaluateDisplayName(_context, _actionRunner.ExecutionContext, out bool updated); // Assert - Assert.False(didUpdateDisplayName); + Assert.True(validDisplayName); + Assert.False(updated); Assert.Equal(actionDisplayName, _actionRunner.DisplayName); } @@ -183,13 +178,51 @@ public void EvaluateExpansionOfDisplayNameToken() // Act // Should expand the displaynameToken and set the display name to that - var didUpdateDisplayName = _actionRunner.TryEvaluateDisplayName(_context, _actionRunner.ExecutionContext); + var validDisplayName = _actionRunner.EvaluateDisplayName(_context, _actionRunner.ExecutionContext, out bool updated); // Assert - Assert.True(didUpdateDisplayName); + Assert.True(validDisplayName); + Assert.True(updated); Assert.Equal(expectedString, _actionRunner.DisplayName); } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void IgnoreDisplayNameTokenWhenDisplayNameIsExplicitlySet() + { + var explicitDisplayName = "Explcitly Set Name"; + + // Arrange + Setup(); + var actionId = Guid.NewGuid(); + var action = new Pipelines.ActionStep() + { + Name = "action", + Id = actionId, + DisplayName = explicitDisplayName, + DisplayNameToken = new BasicExpressionToken(null, null, null, "matrix.node"), + }; + + _actionRunner.Action = action; + + var matrixData = new DictionaryContextData + { + ["node"] = new StringContextData("8") + }; + _context.Add("matrix", matrixData); + + // Act + // Should ignore the displayNameToken since there's already an explicit value for DisplayName + var validDisplayName = _actionRunner.EvaluateDisplayName(_context, _actionRunner.ExecutionContext, out bool updated); + + // Assert + Assert.True(validDisplayName); + Assert.False(updated); + Assert.Equal(explicitDisplayName, _actionRunner.DisplayName); + } + + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -218,10 +251,11 @@ public void EvaluateExpansionOfScriptDisplayName() // Act // Should expand the displaynameToken and set the display name to that - var didUpdateDisplayName = _actionRunner.TryEvaluateDisplayName(_context, _actionRunner.ExecutionContext); + var validDisplayName = _actionRunner.EvaluateDisplayName(_context, _actionRunner.ExecutionContext, out bool updated); // Assert - Assert.True(didUpdateDisplayName); + Assert.True(validDisplayName); + Assert.True(updated); Assert.Equal("Run 8", _actionRunner.DisplayName); } @@ -246,10 +280,11 @@ public void EvaluateExpansionOfContainerDisplayName() // Act // Should expand the displaynameToken and set the display name to that - var didUpdateDisplayName = _actionRunner.TryEvaluateDisplayName(_context, _actionRunner.ExecutionContext); + var validDisplayName = _actionRunner.EvaluateDisplayName(_context, _actionRunner.ExecutionContext, out bool updated); // Assert - Assert.True(didUpdateDisplayName); + Assert.True(validDisplayName); + Assert.True(updated); Assert.Equal("Run TestImageName:latest", _actionRunner.DisplayName); } @@ -272,10 +307,11 @@ public void EvaluateDisplayNameWithoutContext() // Act // Should not do anything if we don't have context on the display name - var didUpdateDisplayName = _actionRunner.TryEvaluateDisplayName(_context, _actionRunner.ExecutionContext); + var validDisplayName = _actionRunner.EvaluateDisplayName(_context, _actionRunner.ExecutionContext, out bool updated); // Assert - Assert.False(didUpdateDisplayName); + Assert.False(validDisplayName); + Assert.False(updated); // Should use the pretty display name until we can eval Assert.Equal("${{ matrix.node }}", _actionRunner.DisplayName); } From fc5c74045d1e3fa31e0725bb225306a19acaa09a Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Tue, 7 Feb 2023 15:10:53 -0500 Subject: [PATCH 090/145] Run service: send more stuff as part of job completed (#2423) * send more stuff as part of job completed * feedback * set only once * feedback * feedback * fix test * feedback * nit: spacing * nit: line Co-authored-by: Tingluo Huang --------- Co-authored-by: Tingluo Huang --- src/Runner.Common/RunServer.cs | 8 ++- src/Runner.Worker/ExecutionContext.cs | 15 +++++ src/Runner.Worker/GlobalContext.cs | 2 + src/Runner.Worker/JobRunner.cs | 2 +- .../RSWebApi/Contracts/AcquireJobRequest.cs | 13 +++++ .../RSWebApi/Contracts/CompleteJobRequest.cs | 26 +++++++++ src/Sdk/RSWebApi/Contracts/StepResult.cs | 38 +++++++++++++ .../RunServiceHttpClient.cs | 34 ++++++----- src/Test/L0/Worker/ExecutionContextL0.cs | 57 +++++++++++++++++++ src/Test/L0/Worker/HandlerL0.cs | 1 + src/Test/L0/Worker/OutputManagerL0.cs | 1 + 11 files changed, 180 insertions(+), 17 deletions(-) create mode 100644 src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs create mode 100644 src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs create mode 100644 src/Sdk/RSWebApi/Contracts/StepResult.cs rename src/Sdk/{DTWebApi/WebApi => RSWebApi}/RunServiceHttpClient.cs (71%) diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 0f9fa1a018b..c1e25d8b84a 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -1,6 +1,8 @@ using System; +using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; +using GitHub.Actions.RunService.WebApi; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Sdk; @@ -16,7 +18,7 @@ public interface IRunServer : IRunnerService Task GetJobMessageAsync(string id, CancellationToken token); - Task CompleteJobAsync(Guid planId, Guid jobId, CancellationToken token); + Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken token); } public sealed class RunServer : RunnerService, IRunServer @@ -56,11 +58,11 @@ public Task GetJobMessageAsync(string id, CancellationTo return jobMessage; } - public Task CompleteJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) + public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken cancellationToken) { CheckConnection(); return RetryRequest( - async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, cancellationToken), cancellationToken); + async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, cancellationToken), cancellationToken); } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 1b5b0b975a8..9bffb73a993 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using GitHub.Actions.RunService.WebApi; using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.Pipelines.ContextData; @@ -437,6 +438,17 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = PublishStepTelemetry(); + var stepResult = new StepResult(); + stepResult.ExternalID = _record.Id; + stepResult.Conclusion = _record.Result ?? TaskResult.Succeeded; + stepResult.Status = _record.State; + stepResult.Number = _record.Order; + stepResult.Name = _record.Name; + stepResult.StartedAt = _record.StartTime; + stepResult.CompletedAt = _record.FinishTime; + + Global.StepsResult.Add(stepResult); + if (Root != this) { // only dispose TokenSource for step level ExecutionContext @@ -710,6 +722,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // ActionsStepTelemetry for entire job Global.StepsTelemetry = new List(); + // Steps results for entire job + Global.StepsResult = new List(); + // Job Outputs JobOutputs = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Runner.Worker/GlobalContext.cs b/src/Runner.Worker/GlobalContext.cs index 8a3ec525b4e..9ca9cc3a7c6 100644 --- a/src/Runner.Worker/GlobalContext.cs +++ b/src/Runner.Worker/GlobalContext.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using GitHub.Actions.RunService.WebApi; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common.Util; using GitHub.Runner.Worker.Container; @@ -17,6 +18,7 @@ public sealed class GlobalContext public IDictionary> JobDefaults { get; set; } public IDictionary> CompositeDefaults { get; set; } public List StepsTelemetry { get; set; } + public List StepsResult { get; set; } public List JobTelemetry { get; set; } public TaskOrchestrationPlanReference Plan { get; set; } public List PrependPath { get; set; } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 49150c5ca28..41fc062d315 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -260,7 +260,7 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution { try { - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, default); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, default); return result; } catch (Exception ex) diff --git a/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs b/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs new file mode 100644 index 00000000000..022f34b42f1 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs @@ -0,0 +1,13 @@ +using System.Collections.Generic; +using System.Runtime.Serialization; +using GitHub.DistributedTask.WebApi; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class AcquireJobRequest + { + [DataMember(Name = "streamId", EmitDefaultValue = false)] + public string StreamID { get; set; } + } +} \ No newline at end of file diff --git a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs new file mode 100644 index 00000000000..27aa3f963b3 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs @@ -0,0 +1,26 @@ +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; +using GitHub.DistributedTask.WebApi; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class CompleteJobRequest + { + [DataMember(Name = "planId", EmitDefaultValue = false)] + public Guid PlanID { get; set; } + + [DataMember(Name = "jobId", EmitDefaultValue = false)] + public Guid JobID { get; set; } + + [DataMember(Name = "conclusion")] + public TaskResult Conclusion { get; set; } + + [DataMember(Name = "outputs", EmitDefaultValue = false)] + public Dictionary Outputs { get; set; } + + [DataMember(Name = "stepResults", EmitDefaultValue = false)] + public IList StepResults { get; set; } + } +} \ No newline at end of file diff --git a/src/Sdk/RSWebApi/Contracts/StepResult.cs b/src/Sdk/RSWebApi/Contracts/StepResult.cs new file mode 100644 index 00000000000..e7df8d6d41e --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/StepResult.cs @@ -0,0 +1,38 @@ +using System; +using System.Runtime.Serialization; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class StepResult + { + [DataMember(Name = "external_id", EmitDefaultValue = false)] + public Guid ExternalID { get; set; } + + [DataMember(Name = "number", EmitDefaultValue = false)] + public int? Number { get; set; } + + [DataMember(Name = "name", EmitDefaultValue = false)] + public string Name { get; set; } + + [DataMember(Name = "status")] + public TimelineRecordState? Status { get; set; } + + [DataMember(Name = "conclusion")] + public TaskResult? Conclusion { get; set; } + + [DataMember(Name = "started_at", EmitDefaultValue = false)] + public DateTime? StartedAt { get; set; } + + [DataMember(Name = "completed_at", EmitDefaultValue = false)] + public DateTime? CompletedAt { get; set; } + + [DataMember(Name = "completed_log_url", EmitDefaultValue = false)] + public string CompletedLogURL { get; set; } + + [DataMember(Name = "completed_log_lines", EmitDefaultValue = false)] + public long? CompletedLogLines { get; set; } + } +} \ No newline at end of file diff --git a/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs similarity index 71% rename from src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs rename to src/Sdk/RSWebApi/RunServiceHttpClient.cs index 7db46d7be87..0bf46146ee2 100644 --- a/src/Sdk/DTWebApi/WebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -1,15 +1,17 @@ -using System; +using System; +using System.Collections.Generic; using System.Net.Http; using System.Threading; using System.Threading.Tasks; +using GitHub.DistributedTask.Pipelines; +using GitHub.DistributedTask.WebApi; using GitHub.Services.Common; using GitHub.Services.OAuth; using GitHub.Services.WebApi; using Sdk.WebApi.WebApi; -namespace GitHub.DistributedTask.WebApi +namespace GitHub.Actions.RunService.WebApi { - [ResourceArea(TaskResourceIds.AreaId)] public class RunServiceHttpClient : RawHttpClientBase { public RunServiceHttpClient( @@ -52,21 +54,21 @@ public RunServiceHttpClient( { } - public Task GetJobMessageAsync( + public Task GetJobMessageAsync( Uri requestUri, string messageId, CancellationToken cancellationToken = default) { HttpMethod httpMethod = new HttpMethod("POST"); - var payload = new { + var payload = new AcquireJobRequest + { StreamID = messageId }; requestUri = new Uri(requestUri, "acquirejob"); - var payloadJson = JsonUtility.ToString(payload); - var requestContent = new StringContent(payloadJson, System.Text.Encoding.UTF8, "application/json"); - return SendAsync( + var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); + return SendAsync( httpMethod, requestUri: requestUri, content: requestContent, @@ -77,18 +79,24 @@ public Task CompleteJobAsync( Uri requestUri, Guid planId, Guid jobId, + TaskResult result, + Dictionary outputs, + IList stepResults, CancellationToken cancellationToken = default) { HttpMethod httpMethod = new HttpMethod("POST"); - var payload = new { - PlanId = planId, - JobId = jobId + var payload = new CompleteJobRequest() + { + PlanID = planId, + JobID = jobId, + Conclusion = result, + Outputs = outputs, + StepResults = stepResults }; requestUri = new Uri(requestUri, "completejob"); - var payloadJson = JsonUtility.ToString(payload); - var requestContent = new StringContent(payloadJson, System.Text.Encoding.UTF8, "application/json"); + var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); return SendAsync( httpMethod, requestUri, diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 3aef7fb58ff..649e66c863d 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -711,6 +711,63 @@ public void PublishStepTelemetry_EmbeddedStep() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void PublishStepResult_EmbeddedStep() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange: Create a job request message. + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + + // Arrange: Setup the paging logger. + var pagingLogger = new Mock(); + var pagingLogger2 = new Mock(); + var jobServerQueue = new Mock(); + jobServerQueue.Setup(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny())); + + hc.EnqueueInstance(pagingLogger.Object); + hc.EnqueueInstance(pagingLogger2.Object); + hc.SetSingleton(jobServerQueue.Object); + + var ec = new Runner.Worker.ExecutionContext(); + ec.Initialize(hc); + + // Act. + ec.InitializeJob(jobRequest, CancellationToken.None); + ec.Start(); + + var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true); + embeddedStep.Start(); + + embeddedStep.StepTelemetry.Type = "node16"; + embeddedStep.StepTelemetry.Action = "actions/checkout"; + embeddedStep.StepTelemetry.Ref = "v2"; + + embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }); + + ec.Complete(); + + // Assert. + Assert.Equal(1, ec.Global.StepsResult.Count); + Assert.Equal(TaskResult.Succeeded, ec.Global.StepsResult.Single().Conclusion); + } + } + private TestHostContext CreateTestContext([CallerMemberName] String testName = "") { var hc = new TestHostContext(this, testName); diff --git a/src/Test/L0/Worker/HandlerL0.cs b/src/Test/L0/Worker/HandlerL0.cs index f1878b2be39..e46dfc91cb9 100644 --- a/src/Test/L0/Worker/HandlerL0.cs +++ b/src/Test/L0/Worker/HandlerL0.cs @@ -1,5 +1,6 @@ using System; using System.Runtime.CompilerServices; +using GitHub.Actions.RunService.WebApi; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Sdk; diff --git a/src/Test/L0/Worker/OutputManagerL0.cs b/src/Test/L0/Worker/OutputManagerL0.cs index e0364e9f0fc..8bac9393362 100644 --- a/src/Test/L0/Worker/OutputManagerL0.cs +++ b/src/Test/L0/Worker/OutputManagerL0.cs @@ -5,6 +5,7 @@ using System.Runtime.CompilerServices; using System.Threading; using System.Threading.Tasks; +using GitHub.Actions.RunService.WebApi; using GitHub.Runner.Sdk; using GitHub.Runner.Worker; using GitHub.Runner.Worker.Container; From 550aa86886e35afd302fe59511c78b88c82a8972 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 8 Feb 2023 09:21:02 -0500 Subject: [PATCH 091/145] Add docker cli to the runner image. (#2425) --- images/Dockerfile | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/images/Dockerfile b/images/Dockerfile index 27ba988598a..8fe339320c0 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -3,6 +3,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build ARG RUNNER_VERSION ARG RUNNER_ARCH="x64" ARG RUNNER_CONTAINER_HOOKS_VERSION=0.2.0 +ARG DOCKER_VERSION=20.10.23 RUN apt update -y && apt install curl unzip -y @@ -15,6 +16,12 @@ RUN curl -f -L -o runner-container-hooks.zip https://github.com/actions/runner-c && unzip ./runner-container-hooks.zip -d ./k8s \ && rm runner-container-hooks.zip +RUN export DOCKER_ARCH=x86_64 \ + && if [ "$RUNNER_ARCH" = "arm64" ]; then export DOCKER_ARCH=aarch64 ; fi \ + && curl -fLo docker.tgz https://download.docker.com/linux/static/stable/${DOCKER_ARCH}/docker-${DOCKER_VERSION}.tgz \ + && tar zxvf docker.tgz \ + && rm -rf docker.tgz + FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 ENV RUNNER_ALLOW_RUNASROOT=1 @@ -23,3 +30,5 @@ ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1 WORKDIR /actions-runner COPY --from=build /actions-runner . + +RUN install -o root -g root -m 755 docker/* /usr/bin/ && rm -rf docker \ No newline at end of file From 6a9f352918e5407e3964578bb46df5a2f35e0b7d Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Wed, 8 Feb 2023 17:42:27 +0100 Subject: [PATCH 092/145] Replace '(' and ')' with '[' and '] from OS.Description so it doesn't fail User-Agent header validation (#2288) * Sanitize OS Desc for UserAgents * Only drop brackets if needed, refactoring * Add missing ')' * Readd missing brackets around '(header)' * Add comments * Use bracket solution from SDK * Rename tests --- src/Runner.Sdk/ActionPlugin.cs | 2 +- src/Runner.Sdk/Util/StringUtil.cs | 7 +++++++ src/Runner.Sdk/Util/VssUtil.cs | 2 +- src/Test/L0/Util/StringUtilL0.cs | 17 +++++++++++++++-- 4 files changed, 24 insertions(+), 4 deletions(-) diff --git a/src/Runner.Sdk/ActionPlugin.cs b/src/Runner.Sdk/ActionPlugin.cs index 35ffa9ab431..441f8ec024f 100644 --- a/src/Runner.Sdk/ActionPlugin.cs +++ b/src/Runner.Sdk/ActionPlugin.cs @@ -73,7 +73,7 @@ public VssConnection InitializeVssConnection() { var headerValues = new List(); headerValues.Add(new ProductInfoHeaderValue($"GitHubActionsRunner-Plugin", BuildConstants.RunnerPackage.Version)); - headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})")); + headerValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})")); if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) { diff --git a/src/Runner.Sdk/Util/StringUtil.cs b/src/Runner.Sdk/Util/StringUtil.cs index 3110e7e21de..69cff219b35 100644 --- a/src/Runner.Sdk/Util/StringUtil.cs +++ b/src/Runner.Sdk/Util/StringUtil.cs @@ -123,5 +123,12 @@ public static string SubstringPrefix(string value, int count) { return value?.Substring(0, Math.Min(value.Length, count)); } + + // Fixes format violations e.g. https://github.com/actions/runner/issues/2165 + public static string SanitizeUserAgentHeader(string header) + { + return header.Replace("(", "[").Replace(")", "]").Trim(); + } + } } diff --git a/src/Runner.Sdk/Util/VssUtil.cs b/src/Runner.Sdk/Util/VssUtil.cs index 12cc5c11826..b9b0f92d8aa 100644 --- a/src/Runner.Sdk/Util/VssUtil.cs +++ b/src/Runner.Sdk/Util/VssUtil.cs @@ -19,7 +19,7 @@ public static void InitializeVssClientSettings(List addi { var headerValues = new List(); headerValues.AddRange(additionalUserAgents); - headerValues.Add(new ProductInfoHeaderValue($"({RuntimeInformation.OSDescription.Trim()})")); + headerValues.Add(new ProductInfoHeaderValue($"({StringUtil.SanitizeUserAgentHeader(RuntimeInformation.OSDescription)})")); if (VssClientHttpRequestSettings.Default.UserAgent != null && VssClientHttpRequestSettings.Default.UserAgent.Count > 0) { diff --git a/src/Test/L0/Util/StringUtilL0.cs b/src/Test/L0/Util/StringUtilL0.cs index e6618187478..f6eed8ccf47 100644 --- a/src/Test/L0/Util/StringUtilL0.cs +++ b/src/Test/L0/Util/StringUtilL0.cs @@ -1,5 +1,4 @@ -using GitHub.Runner.Common.Util; -using GitHub.Runner.Sdk; +using GitHub.Runner.Sdk; using System.Globalization; using Xunit; @@ -186,5 +185,19 @@ public void ConvertStringToBool() Assert.False(result9, $"'{undefineString3}' should convert to false."); } } + + [Theory] + [InlineData("", "")] + [InlineData("(())", "[[]]")] + [InlineData("()()", "[][]")] + [InlineData(" Liquorix kernel OS Description is poorly formatted (linux version ", "Liquorix kernel OS Description is poorly formatted [linux version")] + [InlineData("Liquorix kernel OS Description is poorly formatted (linux version", "Liquorix kernel OS Description is poorly formatted [linux version")] + [InlineData("()((.", "[][[.")] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void SanitizeUserAgentHeader(string input, string expected) + { + Assert.Equal(expected, StringUtil.SanitizeUserAgentHeader(input)); + } } } From c58f11bae78b9ab51a612477800a6ca0c32d4bce Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 13 Feb 2023 13:18:56 -0500 Subject: [PATCH 093/145] Uploading step logs to Results as well (#2422) * Rename queue to results queue * Add results contracts * Add Results logging handling * Adding calls to create and finalize append blob * Modifications for azurite upload * Only call upload complete on final section and remove size * Make method specific to step log so we can support job log later * Change contract for results * Add totalline count to the result log upload file * Actually pass lineCount to Results Service * Fix typos * Code cleanup * Fixing typos * Apply suggestions from code review Co-authored-by: Konrad Pabjan --------- Co-authored-by: Brittany Ellich Co-authored-by: Konrad Pabjan --- src/Runner.Common/JobServer.cs | 14 +- src/Runner.Common/JobServerQueue.cs | 116 +++++++++++----- src/Runner.Common/Logging.cs | 63 +++++++-- src/Runner.Worker/ExecutionContext.cs | 2 +- src/Sdk/DTWebApi/WebApi/TaskAttachment.cs | 1 + src/Sdk/WebApi/WebApi/Contracts.cs | 48 +++++++ src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 151 ++++++++++++++++++++- 7 files changed, 346 insertions(+), 49 deletions(-) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 8a6c4a6a76e..085cc84979d 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -30,7 +30,8 @@ public interface IJobServer : IRunnerService, IAsyncDisposable Task AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); - Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken); + Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken); + Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); @@ -316,7 +317,7 @@ public Task CreateAttachmentAsync(Guid scopeIdentifier, string h return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken); } - public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) + public Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) { if (_resultsClient != null) { @@ -325,6 +326,15 @@ public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, s throw new InvalidOperationException("Results client is not initialized."); } + public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadResultsStepLogAsync(planId, jobId, stepId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken); + } + throw new InvalidOperationException("Results client is not initialized."); + } + public Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) { diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 6440da73607..bc20fed4404 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -20,7 +20,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter void Start(Pipelines.AgentJobRequestMessage jobRequest); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); - void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource); + void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines = 0); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); } @@ -31,7 +31,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000); - private static readonly TimeSpan _delayForSummaryUploadDequeue = TimeSpan.FromMilliseconds(1000); + private static readonly TimeSpan _delayForResultsUploadDequeue = TimeSpan.FromMilliseconds(1000); // Job message information private Guid _scopeIdentifier; @@ -46,7 +46,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // queue for file upload (log file or attachment) private readonly ConcurrentQueue _fileUploadQueue = new(); - private readonly ConcurrentQueue _summaryFileUploadQueue = new(); + private readonly ConcurrentQueue _resultsFileUploadQueue = new(); // queue for timeline or timeline record update (one queue per timeline) private readonly ConcurrentDictionary> _timelineUpdateQueue = new(); @@ -60,7 +60,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // Task for each queue's dequeue process private Task _webConsoleLineDequeueTask; private Task _fileUploadDequeueTask; - private Task _summaryUploadDequeueTask; + private Task _resultsUploadDequeueTask; private Task _timelineUpdateDequeueTask; // common @@ -140,12 +140,12 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) _fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); Trace.Info("Start results file upload queue."); - _summaryUploadDequeueTask = ProcessSummaryUploadQueueAsync(); + _resultsUploadDequeueTask = ProcessResultsUploadQueueAsync(); Trace.Info("Start process timeline update queue."); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); - _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _summaryUploadDequeueTask }; + _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _resultsUploadDequeueTask }; _queueInProcess = true; } @@ -176,9 +176,9 @@ public async Task ShutdownAsync() await ProcessFilesUploadQueueAsync(runOnce: true); Trace.Info("File upload queue drained."); - Trace.Verbose("Draining results summary upload queue."); - await ProcessSummaryUploadQueueAsync(runOnce: true); - Trace.Info("Results summary upload queue drained."); + Trace.Verbose("Draining results upload queue."); + await ProcessResultsUploadQueueAsync(runOnce: true); + Trace.Info("Results upload queue drained."); // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown // if there is any timeline records that failed to update contains output variabls. @@ -230,21 +230,31 @@ public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, _fileUploadQueue.Enqueue(newFile); } - public void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource) + public void QueueResultsUpload(Guid recordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines) { + if (recordId == _jobTimelineRecordId) + { + Trace.Verbose("Skipping job log {0} for record {1}", path, recordId); + return; + } + // all parameter not null, file path exist. - var newFile = new SummaryUploadFileInfo() + var newFile = new ResultsUploadFileInfo() { Name = name, Path = path, + Type = type, PlanId = _planId.ToString(), JobId = _jobTimelineRecordId.ToString(), - StepId = stepRecordId.ToString(), - DeleteSource = deleteSource + RecordId = recordId, + DeleteSource = deleteSource, + Finalize = finalize, + FirstBlock = firstBlock, + TotalLines = totalLines, }; - Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, stepRecordId); - _summaryFileUploadQueue.Enqueue(newFile); + Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, recordId); + _resultsFileUploadQueue.Enqueue(newFile); } public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord) @@ -437,18 +447,18 @@ private async Task ProcessFilesUploadQueueAsync(bool runOnce = false) } } - private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) + private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) { Trace.Info("Starting results-based upload queue..."); while (!_jobCompletionSource.Task.IsCompleted || runOnce) { - List filesToUpload = new(); - SummaryUploadFileInfo dequeueFile; - while (_summaryFileUploadQueue.TryDequeue(out dequeueFile)) + List filesToUpload = new(); + ResultsUploadFileInfo dequeueFile; + while (_resultsFileUploadQueue.TryDequeue(out dequeueFile)) { filesToUpload.Add(dequeueFile); - // process at most 10 file upload. + // process at most 10 file uploads. if (!runOnce && filesToUpload.Count > 10) { break; @@ -459,7 +469,7 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) { if (runOnce) { - Trace.Info($"Uploading {filesToUpload.Count} summary files in one shot through results service."); + Trace.Info($"Uploading {filesToUpload.Count} file(s) in one shot through results service."); } int errorCount = 0; @@ -467,11 +477,22 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) { try { - await UploadSummaryFile(file); + if (String.Equals(file.Type, ChecksAttachmentType.StepSummary, StringComparison.OrdinalIgnoreCase)) + { + await UploadSummaryFile(file); + } + else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase)) + { + if (file.RecordId != _jobTimelineRecordId) + { + Trace.Info($"Got a step log file to send to results service."); + await UploadResultsStepLogFile(file); + } + } } catch (Exception ex) { - var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during summary file upload to results. {ex.Message}" }; + var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during file upload to results. {ex.Message}" }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; var telemetryRecord = new TimelineRecord() @@ -481,16 +502,13 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) telemetryRecord.Issues.Add(issue); QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord); - Trace.Info("Catch exception during summary file upload to results, keep going since the process is best effort."); + Trace.Info("Catch exception during file upload to results, keep going since the process is best effort."); Trace.Error(ex); - } - finally - { errorCount++; } } - Trace.Info("Tried to upload {0} summary files to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); + Trace.Info("Tried to upload {0} file(s) to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); } if (runOnce) @@ -499,7 +517,7 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) } else { - await Task.Delay(_delayForSummaryUploadDequeue); + await Task.Delay(_delayForResultsUploadDequeue); } } } @@ -776,7 +794,7 @@ private async Task UploadFile(UploadFileInfo file) } } - private async Task UploadSummaryFile(SummaryUploadFileInfo file) + private async Task UploadSummaryFile(ResultsUploadFileInfo file) { bool uploadSucceed = false; try @@ -784,7 +802,7 @@ private async Task UploadSummaryFile(SummaryUploadFileInfo file) // Upload the step summary Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); var cancellationTokenSource = new CancellationTokenSource(); - await _jobServer.CreateStepSymmaryAsync(file.PlanId, file.JobId, file.StepId, file.Path, cancellationTokenSource.Token); + await _jobServer.CreateStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, cancellationTokenSource.Token); uploadSucceed = true; } @@ -804,6 +822,34 @@ private async Task UploadSummaryFile(SummaryUploadFileInfo file) } } } + + private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file) + { + bool uploadSucceed = false; + try + { + Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); + var cancellationTokenSource = new CancellationTokenSource(); + await _jobServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, cancellationTokenSource.Token); + + uploadSucceed = true; + } + finally + { + if (uploadSucceed && file.DeleteSource) + { + try + { + File.Delete(file.Path); + } + catch (Exception ex) + { + Trace.Info("Exception encountered during deletion of a temporary file that was already successfully uploaded to results."); + Trace.Error(ex); + } + } + } + } } internal class PendingTimelineRecord @@ -822,14 +868,18 @@ internal class UploadFileInfo public bool DeleteSource { get; set; } } - internal class SummaryUploadFileInfo + internal class ResultsUploadFileInfo { public string Name { get; set; } + public string Type { get; set; } public string Path { get; set; } public string PlanId { get; set; } public string JobId { get; set; } - public string StepId { get; set; } + public Guid RecordId { get; set; } public bool DeleteSource { get; set; } + public bool Finalize { get; set; } + public bool FirstBlock { get; set; } + public long TotalLines { get; set; } } diff --git a/src/Runner.Common/Logging.cs b/src/Runner.Common/Logging.cs index 40be5cdcff0..8cf23749d57 100644 --- a/src/Runner.Common/Logging.cs +++ b/src/Runner.Common/Logging.cs @@ -32,6 +32,19 @@ public class PagingLogger : RunnerService, IPagingLogger private string _pagesFolder; private IJobServerQueue _jobServerQueue; + // For Results + public static string BlocksFolder = "blocks"; + + // 2 MB + public const int BlockSize = 2 * 1024 * 1024; + + private string _resultsDataFileName; + private FileStream _resultsBlockData; + private StreamWriter _resultsBlockWriter; + private string _resultsBlockFolder; + private int _blockByteCount; + private int _blockCount; + public long TotalLines => _totalLines; public override void Initialize(IHostContext hostContext) @@ -39,8 +52,10 @@ public override void Initialize(IHostContext hostContext) base.Initialize(hostContext); _totalLines = 0; _pagesFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), PagingFolder); - _jobServerQueue = HostContext.GetService(); Directory.CreateDirectory(_pagesFolder); + _resultsBlockFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), BlocksFolder); + Directory.CreateDirectory(_resultsBlockFolder); + _jobServerQueue = HostContext.GetService(); } public void Setup(Guid timelineId, Guid timelineRecordId) @@ -60,11 +75,17 @@ public void Write(string message) // lazy creation on write if (_pageWriter == null) { - Create(); + NewPage(); + } + + if (_resultsBlockWriter == null) + { + NewBlock(); } string line = $"{DateTime.UtcNow.ToString("O")} {message}"; _pageWriter.WriteLine(line); + _resultsBlockWriter.WriteLine(line); _totalLines++; if (line.IndexOf('\n') != -1) @@ -78,21 +99,25 @@ public void Write(string message) } } - _byteCount += System.Text.Encoding.UTF8.GetByteCount(line); + var bytes = System.Text.Encoding.UTF8.GetByteCount(line); + _byteCount += bytes; + _blockByteCount += bytes; if (_byteCount >= PageSize) { NewPage(); } + + if (_blockByteCount >= BlockSize) + { + NewBlock(); + } + } public void End() { EndPage(); - } - - private void Create() - { - NewPage(); + EndBlock(true); } private void NewPage() @@ -117,5 +142,27 @@ private void EndPage() _jobServerQueue.QueueFileUpload(_timelineId, _timelineRecordId, "DistributedTask.Core.Log", "CustomToolLog", _dataFileName, true); } } + + private void NewBlock() + { + EndBlock(false); + _blockByteCount = 0; + _resultsDataFileName = Path.Combine(_resultsBlockFolder, $"{_timelineId}_{_timelineRecordId}.{++_blockCount}"); + _resultsBlockData = new FileStream(_resultsDataFileName, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); + _resultsBlockWriter = new StreamWriter(_resultsBlockData, System.Text.Encoding.UTF8); + } + + private void EndBlock(bool finalize) + { + if (_resultsBlockWriter != null) + { + _resultsBlockWriter.Flush(); + _resultsBlockData.Flush(); + _resultsBlockWriter.Dispose(); + _resultsBlockWriter = null; + _resultsBlockData = null; + _jobServerQueue.QueueResultsUpload(_timelineRecordId, "ResultsLog", _resultsDataFileName, "Results.Core.Log", deleteSource: true, finalize, firstBlock: _resultsDataFileName.EndsWith(".1"), totalLines: _totalLines); + } + } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 9bffb73a993..0f2f2267a33 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -872,7 +872,7 @@ public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist."); } - _jobServerQueue.QueueSummaryUpload(stepRecordId, name, filePath, deleteSource: false); + _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: false, firstBlock: false); } // Add OnMatcherChanged diff --git a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs index 17027d1261a..572b1f6426b 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs @@ -100,6 +100,7 @@ public class CoreAttachmentType public static readonly String Summary = "DistributedTask.Core.Summary"; public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment"; public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog"; + public static readonly String ResultsLog = "Results.Core.Log"; } [GenerateAllConstants] diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index d240cc1e230..bc6361d62bc 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -28,6 +28,30 @@ public class GetSignedStepSummaryURLResponse public string BlobStorageType; } + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepLogsURLRequest + { + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string WorkflowRunBackendId; + [DataMember] + public string StepBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepLogsURLResponse + { + [DataMember] + public string LogsUrl; + [DataMember] + public long SoftSizeLimit; + [DataMember] + public string BlobStorageType; + } + [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] public class StepSummaryMetadataCreate @@ -52,6 +76,30 @@ public class CreateStepSummaryMetadataResponse public bool Ok; } + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class StepLogsMetadataCreate + { + [DataMember] + public string StepBackendId; + [DataMember] + public string WorkflowRunBackendId; + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string UploadedAt; + [DataMember] + public long LineCount; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class CreateStepLogsMetadataResponse + { + [DataMember] + public bool Ok; + } + public static class BlobStorageTypes { public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE"; diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 77a733eaeba..989bc239563 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -24,13 +24,13 @@ public ResultsHttpClient( m_formatter = new JsonMediaTypeFormatter(); } - public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, string stepId, CancellationToken cancellationToken) + public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) { var request = new GetSignedStepSummaryURLRequest() { WorkflowJobRunBackendId= jobId, WorkflowRunBackendId= planId, - StepBackendId= stepId + StepBackendId= stepId.ToString() }; var stepSummaryUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepSummarySignedBlobURL"); @@ -51,14 +51,41 @@ public async Task GetStepSummaryUploadUrlAsync( } } - private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, string stepId, long size, CancellationToken cancellationToken) + public async Task GetStepLogUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) + { + var request = new GetSignedStepLogsURLRequest() + { + WorkflowJobRunBackendId= jobId, + WorkflowRunBackendId= planId, + StepBackendId= stepId.ToString(), + }; + + var stepLogsUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepLogsSignedBlobURL"); + + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadRequest)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + return await ReadJsonContentAsync(response, cancellationToken); + } + } + } + } + + private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, Guid stepId, long size, CancellationToken cancellationToken) { var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); var request = new StepSummaryMetadataCreate() { WorkflowJobRunBackendId= jobId, WorkflowRunBackendId= planId, - StepBackendId = stepId, + StepBackendId = stepId.ToString(), Size = size, UploadedAt = timestamp }; @@ -85,6 +112,40 @@ private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, s } } + private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken) + { + var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); + var request = new StepLogsMetadataCreate() + { + WorkflowJobRunBackendId= jobId, + WorkflowRunBackendId= planId, + StepBackendId = stepId.ToString(), + UploadedAt = timestamp, + LineCount = lineCount, + }; + + var stepLogsUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepLogsMetadata"); + + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadCompleteRequest)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); + if (!jsonResponse.Ok) + { + throw new Exception($"Failed to mark step log upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, timestamp: {timestamp}"); + } + } + } + } + } + private async Task UploadFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) { // Upload the file to the url @@ -108,8 +169,55 @@ private async Task UploadFileAsync(string url, string blobS } } + private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StringContent("") + }; + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("x-ms-blob-type", "AppendBlob"); + request.Content.Headers.Add("Content-Length", "0"); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } + return response; + } + } + + private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) + { + var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url + comp) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("Content-Length", fileSize.ToString()); + request.Content.Headers.Add("x-ms-blob-sealed", finalize.ToString()); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); + } + return response; + } + } + // Handle file upload for step summary - public async Task UploadStepSummaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) + public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) { // Get the upload url var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken); @@ -135,6 +243,39 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, string ste await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); } + // Handle file upload for step log + public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + // Get the upload url + var uploadUrlResponse = await GetStepLogUploadUrlAsync(planId, jobId, stepId, cancellationToken); + if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null) + { + throw new Exception("Failed to get step log upload url"); + } + + // Do we want to throw an exception here or should we just be uploading/truncating the data + var fileSize = new FileInfo(file).Length; + + // Create the Append blob + if (firstBlock) + { + await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + } + + // Upload content + using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) + { + var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken); + } + + // Update metadata + if (finalize) + { + // Send step log upload complete message + await StepLogUploadCompleteAsync(planId, jobId, stepId, lineCount, cancellationToken); + } + } + private MediaTypeFormatter m_formatter; private Uri m_resultsServiceUrl; private string m_token; From 9eefe99460f6bf22182d1485c427a6989025ec39 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 14 Feb 2023 09:50:09 -0500 Subject: [PATCH 094/145] Prepare runner release 2.302.0 (#2433) --- releaseNote.md | 19 ++++++++----------- src/runnerversion | 2 +- 2 files changed, 9 insertions(+), 12 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index a6021fb1526..0c47fbc92d6 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,19 +1,16 @@ ## Features -- Log GitHub RequestId for better traceability (#2332) -- Dual upload summary to Actions and Result service (#2334) -- Allow providing extra User-Agent for better correlation (#2370) -- Show more information in the runner log (#2377) -- New option to remove local config files (#2367) +- Add support for ghe.com domain (#2420) +- Add docker cli to the runner image. (#2425) +- Uploading step logs to Results service (#2422) ## Bugs -- Treat jitconfig as secret (#2335) -- Add Header/Footer to multi-line message in StdoutTraceListener (#2336) -- Update Node dependencies (#2381) +- Fix URL construction bug for RunService (#2396) +- Defer evaluation of a step's DisplayName until its condition is evaluated. (#2313) +- Replace '(' and ')' with '[' and '] from OS.Description for fixing User-Agent header validation (#2288) ## Misc -- Make runner image print diag log to STDOUT (#2331) -- Update Node.js to 16.16.0 (#2371) -- Add a disclaimer for which runner version is available to a given tenant (#2362) +- Bump dotnet sdk to latest version. (#2392) +- Start calling run service for job completion (#2412, #2423) _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index b886075fa2b..ed039ac8fab 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.301.1 +2.302.0 From a04bf63557207e064b69eebfd8255d46003a5133 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Tue, 14 Feb 2023 11:01:52 -0500 Subject: [PATCH 095/145] Use v2 version based on https://github.blog/changelog/2023-01-18-code-scanning-codeql-action-v1-is-now-deprecated/ (#2434) --- .github/workflows/codeql.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 2d0e6efbbba..5b6e0523689 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,7 +27,7 @@ jobs: # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL - uses: github/codeql-action/init@v1 + uses: github/codeql-action/init@v2 # Override language selection by uncommenting this and choosing your languages # with: # languages: go, javascript, csharp, python, cpp, java @@ -38,4 +38,4 @@ jobs: working-directory: src - name: Perform CodeQL Analysis - uses: github/codeql-action/analyze@v1 + uses: github/codeql-action/analyze@v2 From 1b711d581e65677aa8783669d19eb5e89f787346 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Wed, 15 Feb 2023 10:50:22 +0000 Subject: [PATCH 096/145] Don't add Needs constant twice --- .../Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs index 43d9d4faa82..331b102469b 100644 --- a/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs +++ b/src/Sdk/DTPipelines/Pipelines/ObjectTemplating/PipelineTemplateEvaluator.cs @@ -455,7 +455,6 @@ private TemplateContext CreateContext( private readonly String[] s_expressionValueNames = new[] { PipelineTemplateConstants.GitHub, - PipelineTemplateConstants.Needs, PipelineTemplateConstants.Strategy, PipelineTemplateConstants.Matrix, PipelineTemplateConstants.Needs, From 7c29bf5bd7d00bd915aa4f30c4b599760e79537c Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Wed, 15 Feb 2023 11:21:32 -0500 Subject: [PATCH 097/145] Revert "Uploading step logs to Results as well (#2422)" (#2437) This reverts commit e979331be42e147b2d1b398b94f7807162e13138. --- src/Runner.Common/JobServer.cs | 14 +- src/Runner.Common/JobServerQueue.cs | 116 +++++----------- src/Runner.Common/Logging.cs | 63 ++------- src/Runner.Worker/ExecutionContext.cs | 2 +- src/Sdk/DTWebApi/WebApi/TaskAttachment.cs | 1 - src/Sdk/WebApi/WebApi/Contracts.cs | 48 ------- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 151 +-------------------- 7 files changed, 49 insertions(+), 346 deletions(-) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 085cc84979d..8a6c4a6a76e 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -30,8 +30,7 @@ public interface IJobServer : IRunnerService, IAsyncDisposable Task AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); - Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken); - Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); + Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); @@ -317,7 +316,7 @@ public Task CreateAttachmentAsync(Guid scopeIdentifier, string h return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken); } - public Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) + public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) { if (_resultsClient != null) { @@ -326,15 +325,6 @@ public Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, str throw new InvalidOperationException("Results client is not initialized."); } - public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) - { - if (_resultsClient != null) - { - return _resultsClient.UploadResultsStepLogAsync(planId, jobId, stepId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken); - } - throw new InvalidOperationException("Results client is not initialized."); - } - public Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) { diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index bc20fed4404..6440da73607 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -20,7 +20,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter void Start(Pipelines.AgentJobRequestMessage jobRequest); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); - void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines = 0); + void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); } @@ -31,7 +31,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000); - private static readonly TimeSpan _delayForResultsUploadDequeue = TimeSpan.FromMilliseconds(1000); + private static readonly TimeSpan _delayForSummaryUploadDequeue = TimeSpan.FromMilliseconds(1000); // Job message information private Guid _scopeIdentifier; @@ -46,7 +46,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // queue for file upload (log file or attachment) private readonly ConcurrentQueue _fileUploadQueue = new(); - private readonly ConcurrentQueue _resultsFileUploadQueue = new(); + private readonly ConcurrentQueue _summaryFileUploadQueue = new(); // queue for timeline or timeline record update (one queue per timeline) private readonly ConcurrentDictionary> _timelineUpdateQueue = new(); @@ -60,7 +60,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // Task for each queue's dequeue process private Task _webConsoleLineDequeueTask; private Task _fileUploadDequeueTask; - private Task _resultsUploadDequeueTask; + private Task _summaryUploadDequeueTask; private Task _timelineUpdateDequeueTask; // common @@ -140,12 +140,12 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) _fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); Trace.Info("Start results file upload queue."); - _resultsUploadDequeueTask = ProcessResultsUploadQueueAsync(); + _summaryUploadDequeueTask = ProcessSummaryUploadQueueAsync(); Trace.Info("Start process timeline update queue."); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); - _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _resultsUploadDequeueTask }; + _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _summaryUploadDequeueTask }; _queueInProcess = true; } @@ -176,9 +176,9 @@ public async Task ShutdownAsync() await ProcessFilesUploadQueueAsync(runOnce: true); Trace.Info("File upload queue drained."); - Trace.Verbose("Draining results upload queue."); - await ProcessResultsUploadQueueAsync(runOnce: true); - Trace.Info("Results upload queue drained."); + Trace.Verbose("Draining results summary upload queue."); + await ProcessSummaryUploadQueueAsync(runOnce: true); + Trace.Info("Results summary upload queue drained."); // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown // if there is any timeline records that failed to update contains output variabls. @@ -230,31 +230,21 @@ public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, _fileUploadQueue.Enqueue(newFile); } - public void QueueResultsUpload(Guid recordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines) + public void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource) { - if (recordId == _jobTimelineRecordId) - { - Trace.Verbose("Skipping job log {0} for record {1}", path, recordId); - return; - } - // all parameter not null, file path exist. - var newFile = new ResultsUploadFileInfo() + var newFile = new SummaryUploadFileInfo() { Name = name, Path = path, - Type = type, PlanId = _planId.ToString(), JobId = _jobTimelineRecordId.ToString(), - RecordId = recordId, - DeleteSource = deleteSource, - Finalize = finalize, - FirstBlock = firstBlock, - TotalLines = totalLines, + StepId = stepRecordId.ToString(), + DeleteSource = deleteSource }; - Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, recordId); - _resultsFileUploadQueue.Enqueue(newFile); + Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, stepRecordId); + _summaryFileUploadQueue.Enqueue(newFile); } public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord) @@ -447,18 +437,18 @@ private async Task ProcessFilesUploadQueueAsync(bool runOnce = false) } } - private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) + private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) { Trace.Info("Starting results-based upload queue..."); while (!_jobCompletionSource.Task.IsCompleted || runOnce) { - List filesToUpload = new(); - ResultsUploadFileInfo dequeueFile; - while (_resultsFileUploadQueue.TryDequeue(out dequeueFile)) + List filesToUpload = new(); + SummaryUploadFileInfo dequeueFile; + while (_summaryFileUploadQueue.TryDequeue(out dequeueFile)) { filesToUpload.Add(dequeueFile); - // process at most 10 file uploads. + // process at most 10 file upload. if (!runOnce && filesToUpload.Count > 10) { break; @@ -469,7 +459,7 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) { if (runOnce) { - Trace.Info($"Uploading {filesToUpload.Count} file(s) in one shot through results service."); + Trace.Info($"Uploading {filesToUpload.Count} summary files in one shot through results service."); } int errorCount = 0; @@ -477,22 +467,11 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) { try { - if (String.Equals(file.Type, ChecksAttachmentType.StepSummary, StringComparison.OrdinalIgnoreCase)) - { - await UploadSummaryFile(file); - } - else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase)) - { - if (file.RecordId != _jobTimelineRecordId) - { - Trace.Info($"Got a step log file to send to results service."); - await UploadResultsStepLogFile(file); - } - } + await UploadSummaryFile(file); } catch (Exception ex) { - var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during file upload to results. {ex.Message}" }; + var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during summary file upload to results. {ex.Message}" }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; var telemetryRecord = new TimelineRecord() @@ -502,13 +481,16 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) telemetryRecord.Issues.Add(issue); QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord); - Trace.Info("Catch exception during file upload to results, keep going since the process is best effort."); + Trace.Info("Catch exception during summary file upload to results, keep going since the process is best effort."); Trace.Error(ex); + } + finally + { errorCount++; } } - Trace.Info("Tried to upload {0} file(s) to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); + Trace.Info("Tried to upload {0} summary files to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); } if (runOnce) @@ -517,7 +499,7 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) } else { - await Task.Delay(_delayForResultsUploadDequeue); + await Task.Delay(_delayForSummaryUploadDequeue); } } } @@ -794,7 +776,7 @@ private async Task UploadFile(UploadFileInfo file) } } - private async Task UploadSummaryFile(ResultsUploadFileInfo file) + private async Task UploadSummaryFile(SummaryUploadFileInfo file) { bool uploadSucceed = false; try @@ -802,7 +784,7 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) // Upload the step summary Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); var cancellationTokenSource = new CancellationTokenSource(); - await _jobServer.CreateStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, cancellationTokenSource.Token); + await _jobServer.CreateStepSymmaryAsync(file.PlanId, file.JobId, file.StepId, file.Path, cancellationTokenSource.Token); uploadSucceed = true; } @@ -822,34 +804,6 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) } } } - - private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file) - { - bool uploadSucceed = false; - try - { - Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); - var cancellationTokenSource = new CancellationTokenSource(); - await _jobServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, cancellationTokenSource.Token); - - uploadSucceed = true; - } - finally - { - if (uploadSucceed && file.DeleteSource) - { - try - { - File.Delete(file.Path); - } - catch (Exception ex) - { - Trace.Info("Exception encountered during deletion of a temporary file that was already successfully uploaded to results."); - Trace.Error(ex); - } - } - } - } } internal class PendingTimelineRecord @@ -868,18 +822,14 @@ internal class UploadFileInfo public bool DeleteSource { get; set; } } - internal class ResultsUploadFileInfo + internal class SummaryUploadFileInfo { public string Name { get; set; } - public string Type { get; set; } public string Path { get; set; } public string PlanId { get; set; } public string JobId { get; set; } - public Guid RecordId { get; set; } + public string StepId { get; set; } public bool DeleteSource { get; set; } - public bool Finalize { get; set; } - public bool FirstBlock { get; set; } - public long TotalLines { get; set; } } diff --git a/src/Runner.Common/Logging.cs b/src/Runner.Common/Logging.cs index 8cf23749d57..40be5cdcff0 100644 --- a/src/Runner.Common/Logging.cs +++ b/src/Runner.Common/Logging.cs @@ -32,19 +32,6 @@ public class PagingLogger : RunnerService, IPagingLogger private string _pagesFolder; private IJobServerQueue _jobServerQueue; - // For Results - public static string BlocksFolder = "blocks"; - - // 2 MB - public const int BlockSize = 2 * 1024 * 1024; - - private string _resultsDataFileName; - private FileStream _resultsBlockData; - private StreamWriter _resultsBlockWriter; - private string _resultsBlockFolder; - private int _blockByteCount; - private int _blockCount; - public long TotalLines => _totalLines; public override void Initialize(IHostContext hostContext) @@ -52,10 +39,8 @@ public override void Initialize(IHostContext hostContext) base.Initialize(hostContext); _totalLines = 0; _pagesFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), PagingFolder); - Directory.CreateDirectory(_pagesFolder); - _resultsBlockFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), BlocksFolder); - Directory.CreateDirectory(_resultsBlockFolder); _jobServerQueue = HostContext.GetService(); + Directory.CreateDirectory(_pagesFolder); } public void Setup(Guid timelineId, Guid timelineRecordId) @@ -75,17 +60,11 @@ public void Write(string message) // lazy creation on write if (_pageWriter == null) { - NewPage(); - } - - if (_resultsBlockWriter == null) - { - NewBlock(); + Create(); } string line = $"{DateTime.UtcNow.ToString("O")} {message}"; _pageWriter.WriteLine(line); - _resultsBlockWriter.WriteLine(line); _totalLines++; if (line.IndexOf('\n') != -1) @@ -99,25 +78,21 @@ public void Write(string message) } } - var bytes = System.Text.Encoding.UTF8.GetByteCount(line); - _byteCount += bytes; - _blockByteCount += bytes; + _byteCount += System.Text.Encoding.UTF8.GetByteCount(line); if (_byteCount >= PageSize) { NewPage(); } - - if (_blockByteCount >= BlockSize) - { - NewBlock(); - } - } public void End() { EndPage(); - EndBlock(true); + } + + private void Create() + { + NewPage(); } private void NewPage() @@ -142,27 +117,5 @@ private void EndPage() _jobServerQueue.QueueFileUpload(_timelineId, _timelineRecordId, "DistributedTask.Core.Log", "CustomToolLog", _dataFileName, true); } } - - private void NewBlock() - { - EndBlock(false); - _blockByteCount = 0; - _resultsDataFileName = Path.Combine(_resultsBlockFolder, $"{_timelineId}_{_timelineRecordId}.{++_blockCount}"); - _resultsBlockData = new FileStream(_resultsDataFileName, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); - _resultsBlockWriter = new StreamWriter(_resultsBlockData, System.Text.Encoding.UTF8); - } - - private void EndBlock(bool finalize) - { - if (_resultsBlockWriter != null) - { - _resultsBlockWriter.Flush(); - _resultsBlockData.Flush(); - _resultsBlockWriter.Dispose(); - _resultsBlockWriter = null; - _resultsBlockData = null; - _jobServerQueue.QueueResultsUpload(_timelineRecordId, "ResultsLog", _resultsDataFileName, "Results.Core.Log", deleteSource: true, finalize, firstBlock: _resultsDataFileName.EndsWith(".1"), totalLines: _totalLines); - } - } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 0f2f2267a33..9bffb73a993 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -872,7 +872,7 @@ public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist."); } - _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: false, firstBlock: false); + _jobServerQueue.QueueSummaryUpload(stepRecordId, name, filePath, deleteSource: false); } // Add OnMatcherChanged diff --git a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs index 572b1f6426b..17027d1261a 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs @@ -100,7 +100,6 @@ public class CoreAttachmentType public static readonly String Summary = "DistributedTask.Core.Summary"; public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment"; public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog"; - public static readonly String ResultsLog = "Results.Core.Log"; } [GenerateAllConstants] diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index bc6361d62bc..d240cc1e230 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -28,30 +28,6 @@ public class GetSignedStepSummaryURLResponse public string BlobStorageType; } - [DataContract] - [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class GetSignedStepLogsURLRequest - { - [DataMember] - public string WorkflowJobRunBackendId; - [DataMember] - public string WorkflowRunBackendId; - [DataMember] - public string StepBackendId; - } - - [DataContract] - [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class GetSignedStepLogsURLResponse - { - [DataMember] - public string LogsUrl; - [DataMember] - public long SoftSizeLimit; - [DataMember] - public string BlobStorageType; - } - [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] public class StepSummaryMetadataCreate @@ -76,30 +52,6 @@ public class CreateStepSummaryMetadataResponse public bool Ok; } - [DataContract] - [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class StepLogsMetadataCreate - { - [DataMember] - public string StepBackendId; - [DataMember] - public string WorkflowRunBackendId; - [DataMember] - public string WorkflowJobRunBackendId; - [DataMember] - public string UploadedAt; - [DataMember] - public long LineCount; - } - - [DataContract] - [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class CreateStepLogsMetadataResponse - { - [DataMember] - public bool Ok; - } - public static class BlobStorageTypes { public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE"; diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 989bc239563..77a733eaeba 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -24,13 +24,13 @@ public ResultsHttpClient( m_formatter = new JsonMediaTypeFormatter(); } - public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) + public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, string stepId, CancellationToken cancellationToken) { var request = new GetSignedStepSummaryURLRequest() { WorkflowJobRunBackendId= jobId, WorkflowRunBackendId= planId, - StepBackendId= stepId.ToString() + StepBackendId= stepId }; var stepSummaryUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepSummarySignedBlobURL"); @@ -51,41 +51,14 @@ public async Task GetStepSummaryUploadUrlAsync( } } - public async Task GetStepLogUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) - { - var request = new GetSignedStepLogsURLRequest() - { - WorkflowJobRunBackendId= jobId, - WorkflowRunBackendId= planId, - StepBackendId= stepId.ToString(), - }; - - var stepLogsUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepLogsSignedBlobURL"); - - using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadRequest)) - { - requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); - requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); - - using (HttpContent content = new ObjectContent(request, m_formatter)) - { - requestMessage.Content = content; - using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) - { - return await ReadJsonContentAsync(response, cancellationToken); - } - } - } - } - - private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, Guid stepId, long size, CancellationToken cancellationToken) + private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, string stepId, long size, CancellationToken cancellationToken) { var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); var request = new StepSummaryMetadataCreate() { WorkflowJobRunBackendId= jobId, WorkflowRunBackendId= planId, - StepBackendId = stepId.ToString(), + StepBackendId = stepId, Size = size, UploadedAt = timestamp }; @@ -112,40 +85,6 @@ private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, G } } - private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken) - { - var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); - var request = new StepLogsMetadataCreate() - { - WorkflowJobRunBackendId= jobId, - WorkflowRunBackendId= planId, - StepBackendId = stepId.ToString(), - UploadedAt = timestamp, - LineCount = lineCount, - }; - - var stepLogsUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepLogsMetadata"); - - using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadCompleteRequest)) - { - requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); - requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); - - using (HttpContent content = new ObjectContent(request, m_formatter)) - { - requestMessage.Content = content; - using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) - { - var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); - if (!jsonResponse.Ok) - { - throw new Exception($"Failed to mark step log upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, timestamp: {timestamp}"); - } - } - } - } - } - private async Task UploadFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) { // Upload the file to the url @@ -169,55 +108,8 @@ private async Task UploadFileAsync(string url, string blobS } } - private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) - { - var request = new HttpRequestMessage(HttpMethod.Put, url) - { - Content = new StringContent("") - }; - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) - { - request.Content.Headers.Add("x-ms-blob-type", "AppendBlob"); - request.Content.Headers.Add("Content-Length", "0"); - } - - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) - { - throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); - } - return response; - } - } - - private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) - { - var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; - // Upload the file to the url - var request = new HttpRequestMessage(HttpMethod.Put, url + comp) - { - Content = new StreamContent(file) - }; - - if (blobStorageType == BlobStorageTypes.AzureBlobStorage) - { - request.Content.Headers.Add("Content-Length", fileSize.ToString()); - request.Content.Headers.Add("x-ms-blob-sealed", finalize.ToString()); - } - - using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) - { - if (!response.IsSuccessStatusCode) - { - throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); - } - return response; - } - } - // Handle file upload for step summary - public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) + public async Task UploadStepSummaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) { // Get the upload url var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken); @@ -243,39 +135,6 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepI await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); } - // Handle file upload for step log - public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) - { - // Get the upload url - var uploadUrlResponse = await GetStepLogUploadUrlAsync(planId, jobId, stepId, cancellationToken); - if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null) - { - throw new Exception("Failed to get step log upload url"); - } - - // Do we want to throw an exception here or should we just be uploading/truncating the data - var fileSize = new FileInfo(file).Length; - - // Create the Append blob - if (firstBlock) - { - await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); - } - - // Upload content - using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) - { - var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken); - } - - // Update metadata - if (finalize) - { - // Send step log upload complete message - await StepLogUploadCompleteAsync(planId, jobId, stepId, lineCount, cancellationToken); - } - } - private MediaTypeFormatter m_formatter; private Uri m_resultsServiceUrl; private string m_token; From 876b1b5e85bc42e3a04bea318f0b9263543b0882 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 15 Feb 2023 16:05:36 -0500 Subject: [PATCH 098/145] Wait for docker base on env RUNNER_WAIT_FOR_DOCKER_IN_SECONDS. (#2440) --- releaseNote.md | 1 - src/Misc/layoutroot/run-helper.sh.template | 14 ++++++++++++++ src/runnerversion | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 0c47fbc92d6..2e7367c7a70 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,7 +1,6 @@ ## Features - Add support for ghe.com domain (#2420) - Add docker cli to the runner image. (#2425) -- Uploading step logs to Results service (#2422) ## Bugs - Fix URL construction bug for RunService (#2396) diff --git a/src/Misc/layoutroot/run-helper.sh.template b/src/Misc/layoutroot/run-helper.sh.template index 36ca2d94a8f..743fd8b6959 100755 --- a/src/Misc/layoutroot/run-helper.sh.template +++ b/src/Misc/layoutroot/run-helper.sh.template @@ -18,6 +18,20 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli done DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" +# Wait for docker to start +if [ ! -z "$RUNNER_WAIT_FOR_DOCKER_IN_SECONDS" ]; then + if [ "$RUNNER_WAIT_FOR_DOCKER_IN_SECONDS" -gt 0 ]; then + echo "Waiting for docker to be ready." + for i in $(seq "$RUNNER_WAIT_FOR_DOCKER_IN_SECONDS"); do + if docker ps > /dev/null 2>&1; then + echo "Docker is ready." + break + fi + "$DIR"/safe_sleep.sh 1 + done + fi +fi + updateFile="update.finished" "$DIR"/bin/Runner.Listener run $* diff --git a/src/runnerversion b/src/runnerversion index ed039ac8fab..22155b3fbc4 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.302.0 +2.302.1 From 51e0aa4b330ffaafe9e068ccbde2342acb196794 Mon Sep 17 00:00:00 2001 From: JoannaaKL Date: Thu, 16 Feb 2023 14:33:03 +0100 Subject: [PATCH 099/145] Don't disable lint errors (#2436) * Update lint.yml Don't ignore the formatting errors * Add formatting made by @cory-miller * Use dotnet format * Format only modified files * Add instruction to contribute.md * Use git status instead of git diff --- .github/workflows/lint.yml | 1 - docs/contribute.md | 8 ++++++++ src/dev.sh | 11 ++++++++++- 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 9df0fcc0895..b35bee3c7d9 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -18,7 +18,6 @@ jobs: uses: github/super-linter@v4 env: DEFAULT_BRANCH: ${{ github.base_ref }} - DISABLE_ERRORS: true EDITORCONFIG_FILE_NAME: .editorconfig LINTER_RULES_PATH: /src/ VALIDATE_ALL_CODEBASE: false diff --git a/docs/contribute.md b/docs/contribute.md index 70ec61154ce..a350b846b19 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -158,3 +158,11 @@ cat (Runner/Worker)_TIMESTAMP.log # view your log file We use the .NET Foundation and CoreCLR style guidelines [located here]( https://github.com/dotnet/corefx/blob/master/Documentation/coding-guidelines/coding-style.md) + +### Format C# Code + +To format both staged and unstaged .cs files +``` +cd ./src +./dev.(cmd|sh) format +``` \ No newline at end of file diff --git a/src/dev.sh b/src/dev.sh index fed033d3886..6ed5462a769 100755 --- a/src/dev.sh +++ b/src/dev.sh @@ -203,6 +203,13 @@ function runtest () dotnet msbuild -t:test -p:PackageRuntime="${RUNTIME_ID}" -p:BUILDCONFIG="${BUILD_CONFIG}" -p:RunnerVersion="${RUNNER_VERSION}" ./dir.proj || failed "failed tests" } +function format() +{ + heading "Formatting..." + files="$(git status -s "*.cs" | awk '{print $2}' | tr '\n' ' ')" + dotnet format ${SCRIPT_DIR}/ActionsRunner.sln --exclude / --include $files || failed "failed formatting" +} + function package () { if [ ! -d "${LAYOUT_DIR}/bin" ]; then @@ -360,7 +367,9 @@ case $DEV_CMD in "l") layout;; "package") package;; "p") package;; - *) echo "Invalid cmd. Use build(b), test(t), layout(l) or package(p)";; + "format") format;; + "f") format;; + *) echo "Invalid cmd. Use build(b), test(t), layout(l), package(p), or format(f)";; esac popd From 8e3de94b98fa0ea638bc649fffa4cff21332f82a Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Fri, 17 Feb 2023 10:31:41 -0500 Subject: [PATCH 100/145] Properly guard upload (#2439) * Revert "Revert "Uploading step logs to Results as well (#2422)" (#2437)" This reverts commit 8c096baf49f579380d4e911c74b0b749729028ac. * Properly guard the upload to results feature * Delete skipped file if deletesource is true --- src/Runner.Common/JobServer.cs | 14 +- src/Runner.Common/JobServerQueue.cs | 135 ++++++++++++----- src/Runner.Common/Logging.cs | 63 ++++++-- src/Runner.Worker/ExecutionContext.cs | 5 +- src/Sdk/DTWebApi/WebApi/TaskAttachment.cs | 3 +- src/Sdk/WebApi/WebApi/Contracts.cs | 48 +++++++ src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 159 +++++++++++++++++++-- 7 files changed, 369 insertions(+), 58 deletions(-) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 8a6c4a6a76e..085cc84979d 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -30,7 +30,8 @@ public interface IJobServer : IRunnerService, IAsyncDisposable Task AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); - Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken); + Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken); + Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); @@ -316,7 +317,7 @@ public Task CreateAttachmentAsync(Guid scopeIdentifier, string h return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken); } - public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) + public Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) { if (_resultsClient != null) { @@ -325,6 +326,15 @@ public Task CreateStepSymmaryAsync(string planId, string jobId, string stepId, s throw new InvalidOperationException("Results client is not initialized."); } + public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadResultsStepLogAsync(planId, jobId, stepId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken); + } + throw new InvalidOperationException("Results client is not initialized."); + } + public Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) { diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 6440da73607..f5f5a5494fe 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -20,7 +20,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter void Start(Pipelines.AgentJobRequestMessage jobRequest); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); - void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource); + void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines); void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord); } @@ -31,7 +31,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private static readonly TimeSpan _delayForWebConsoleLineDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForTimelineUpdateDequeue = TimeSpan.FromMilliseconds(500); private static readonly TimeSpan _delayForFileUploadDequeue = TimeSpan.FromMilliseconds(1000); - private static readonly TimeSpan _delayForSummaryUploadDequeue = TimeSpan.FromMilliseconds(1000); + private static readonly TimeSpan _delayForResultsUploadDequeue = TimeSpan.FromMilliseconds(1000); // Job message information private Guid _scopeIdentifier; @@ -46,7 +46,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // queue for file upload (log file or attachment) private readonly ConcurrentQueue _fileUploadQueue = new(); - private readonly ConcurrentQueue _summaryFileUploadQueue = new(); + private readonly ConcurrentQueue _resultsFileUploadQueue = new(); // queue for timeline or timeline record update (one queue per timeline) private readonly ConcurrentDictionary> _timelineUpdateQueue = new(); @@ -60,7 +60,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // Task for each queue's dequeue process private Task _webConsoleLineDequeueTask; private Task _fileUploadDequeueTask; - private Task _summaryUploadDequeueTask; + private Task _resultsUploadDequeueTask; private Task _timelineUpdateDequeueTask; // common @@ -84,6 +84,8 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private bool _webConsoleLineAggressiveDequeue = true; private bool _firstConsoleOutputs = true; + private bool _resultsClientInitiated = false; + public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); @@ -109,9 +111,9 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) { Trace.Info("Initializing results client"); _jobServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken); + _resultsClientInitiated = true; } - if (_queueInProcess) { Trace.Info("No-opt, all queue process tasks are running."); @@ -140,12 +142,12 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) _fileUploadDequeueTask = ProcessFilesUploadQueueAsync(); Trace.Info("Start results file upload queue."); - _summaryUploadDequeueTask = ProcessSummaryUploadQueueAsync(); + _resultsUploadDequeueTask = ProcessResultsUploadQueueAsync(); Trace.Info("Start process timeline update queue."); _timelineUpdateDequeueTask = ProcessTimelinesUpdateQueueAsync(); - _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _summaryUploadDequeueTask }; + _allDequeueTasks = new Task[] { _webConsoleLineDequeueTask, _fileUploadDequeueTask, _timelineUpdateDequeueTask, _resultsUploadDequeueTask }; _queueInProcess = true; } @@ -176,9 +178,9 @@ public async Task ShutdownAsync() await ProcessFilesUploadQueueAsync(runOnce: true); Trace.Info("File upload queue drained."); - Trace.Verbose("Draining results summary upload queue."); - await ProcessSummaryUploadQueueAsync(runOnce: true); - Trace.Info("Results summary upload queue drained."); + Trace.Verbose("Draining results upload queue."); + await ProcessResultsUploadQueueAsync(runOnce: true); + Trace.Info("Results upload queue drained."); // ProcessTimelinesUpdateQueueAsync() will throw exception during shutdown // if there is any timeline records that failed to update contains output variabls. @@ -230,21 +232,49 @@ public void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, _fileUploadQueue.Enqueue(newFile); } - public void QueueSummaryUpload(Guid stepRecordId, string name, string path, bool deleteSource) + public void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines) { + if (!_resultsClientInitiated) + { + Trace.Verbose("Skipping results upload"); + try + { + if (deleteSource) + { + File.Delete(path); + } + } + catch (Exception ex) + { + Trace.Info("Catch exception during delete skipped results upload file."); + Trace.Error(ex); + } + return; + } + + if (timelineRecordId == _jobTimelineRecordId && String.Equals(type, CoreAttachmentType.ResultsLog, StringComparison.Ordinal)) + { + Trace.Verbose("Skipping job log {0} for record {1}", path, timelineRecordId); + return; + } + // all parameter not null, file path exist. - var newFile = new SummaryUploadFileInfo() + var newFile = new ResultsUploadFileInfo() { Name = name, Path = path, + Type = type, PlanId = _planId.ToString(), JobId = _jobTimelineRecordId.ToString(), - StepId = stepRecordId.ToString(), - DeleteSource = deleteSource + RecordId = timelineRecordId, + DeleteSource = deleteSource, + Finalize = finalize, + FirstBlock = firstBlock, + TotalLines = totalLines, }; - Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, stepRecordId); - _summaryFileUploadQueue.Enqueue(newFile); + Trace.Verbose("Enqueue results file upload queue: file '{0}' attach to job {1} step {2}", newFile.Path, _jobTimelineRecordId, timelineRecordId); + _resultsFileUploadQueue.Enqueue(newFile); } public void QueueTimelineRecordUpdate(Guid timelineId, TimelineRecord timelineRecord) @@ -437,18 +467,18 @@ private async Task ProcessFilesUploadQueueAsync(bool runOnce = false) } } - private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) + private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) { Trace.Info("Starting results-based upload queue..."); while (!_jobCompletionSource.Task.IsCompleted || runOnce) { - List filesToUpload = new(); - SummaryUploadFileInfo dequeueFile; - while (_summaryFileUploadQueue.TryDequeue(out dequeueFile)) + List filesToUpload = new(); + ResultsUploadFileInfo dequeueFile; + while (_resultsFileUploadQueue.TryDequeue(out dequeueFile)) { filesToUpload.Add(dequeueFile); - // process at most 10 file upload. + // process at most 10 file uploads. if (!runOnce && filesToUpload.Count > 10) { break; @@ -459,7 +489,7 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) { if (runOnce) { - Trace.Info($"Uploading {filesToUpload.Count} summary files in one shot through results service."); + Trace.Info($"Uploading {filesToUpload.Count} file(s) in one shot through results service."); } int errorCount = 0; @@ -467,11 +497,19 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) { try { - await UploadSummaryFile(file); + if (String.Equals(file.Type, ChecksAttachmentType.StepSummary, StringComparison.OrdinalIgnoreCase)) + { + await UploadSummaryFile(file); + } + else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase)) + { + Trace.Info($"Got a step log file to send to results service."); + await UploadResultsStepLogFile(file); + } } catch (Exception ex) { - var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during summary file upload to results. {ex.Message}" }; + var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during file upload to results. {ex.Message}" }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; var telemetryRecord = new TimelineRecord() @@ -481,16 +519,13 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) telemetryRecord.Issues.Add(issue); QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord); - Trace.Info("Catch exception during summary file upload to results, keep going since the process is best effort."); + Trace.Info("Catch exception during file upload to results, keep going since the process is best effort."); Trace.Error(ex); - } - finally - { errorCount++; } } - Trace.Info("Tried to upload {0} summary files to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); + Trace.Info("Tried to upload {0} file(s) to results, success rate: {1}/{0}.", filesToUpload.Count, filesToUpload.Count - errorCount); } if (runOnce) @@ -499,7 +534,7 @@ private async Task ProcessSummaryUploadQueueAsync(bool runOnce = false) } else { - await Task.Delay(_delayForSummaryUploadDequeue); + await Task.Delay(_delayForResultsUploadDequeue); } } } @@ -776,15 +811,14 @@ private async Task UploadFile(UploadFileInfo file) } } - private async Task UploadSummaryFile(SummaryUploadFileInfo file) + private async Task UploadSummaryFile(ResultsUploadFileInfo file) { bool uploadSucceed = false; try { // Upload the step summary Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); - var cancellationTokenSource = new CancellationTokenSource(); - await _jobServer.CreateStepSymmaryAsync(file.PlanId, file.JobId, file.StepId, file.Path, cancellationTokenSource.Token); + await _jobServer.CreateStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, CancellationToken.None); uploadSucceed = true; } @@ -804,6 +838,33 @@ private async Task UploadSummaryFile(SummaryUploadFileInfo file) } } } + + private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file) + { + bool uploadSucceed = false; + try + { + Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); + await _jobServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); + + uploadSucceed = true; + } + finally + { + if (uploadSucceed && file.DeleteSource) + { + try + { + File.Delete(file.Path); + } + catch (Exception ex) + { + Trace.Info("Exception encountered during deletion of a temporary file that was already successfully uploaded to results."); + Trace.Error(ex); + } + } + } + } } internal class PendingTimelineRecord @@ -822,14 +883,18 @@ internal class UploadFileInfo public bool DeleteSource { get; set; } } - internal class SummaryUploadFileInfo + internal class ResultsUploadFileInfo { public string Name { get; set; } + public string Type { get; set; } public string Path { get; set; } public string PlanId { get; set; } public string JobId { get; set; } - public string StepId { get; set; } + public Guid RecordId { get; set; } public bool DeleteSource { get; set; } + public bool Finalize { get; set; } + public bool FirstBlock { get; set; } + public long TotalLines { get; set; } } diff --git a/src/Runner.Common/Logging.cs b/src/Runner.Common/Logging.cs index 40be5cdcff0..0bc2a13f0e3 100644 --- a/src/Runner.Common/Logging.cs +++ b/src/Runner.Common/Logging.cs @@ -21,6 +21,12 @@ public class PagingLogger : RunnerService, IPagingLogger // 8 MB public const int PageSize = 8 * 1024 * 1024; + // For Results + public static string BlocksFolder = "blocks"; + + // 2 MB + public const int BlockSize = 2 * 1024 * 1024; + private Guid _timelineId; private Guid _timelineRecordId; private FileStream _pageData; @@ -32,6 +38,13 @@ public class PagingLogger : RunnerService, IPagingLogger private string _pagesFolder; private IJobServerQueue _jobServerQueue; + private string _resultsDataFileName; + private FileStream _resultsBlockData; + private StreamWriter _resultsBlockWriter; + private string _resultsBlockFolder; + private int _blockByteCount; + private int _blockCount; + public long TotalLines => _totalLines; public override void Initialize(IHostContext hostContext) @@ -39,8 +52,10 @@ public override void Initialize(IHostContext hostContext) base.Initialize(hostContext); _totalLines = 0; _pagesFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), PagingFolder); - _jobServerQueue = HostContext.GetService(); Directory.CreateDirectory(_pagesFolder); + _resultsBlockFolder = Path.Combine(hostContext.GetDirectory(WellKnownDirectory.Diag), BlocksFolder); + Directory.CreateDirectory(_resultsBlockFolder); + _jobServerQueue = HostContext.GetService(); } public void Setup(Guid timelineId, Guid timelineRecordId) @@ -60,11 +75,17 @@ public void Write(string message) // lazy creation on write if (_pageWriter == null) { - Create(); + NewPage(); + } + + if (_resultsBlockWriter == null) + { + NewBlock(); } string line = $"{DateTime.UtcNow.ToString("O")} {message}"; _pageWriter.WriteLine(line); + _resultsBlockWriter.WriteLine(line); _totalLines++; if (line.IndexOf('\n') != -1) @@ -78,21 +99,25 @@ public void Write(string message) } } - _byteCount += System.Text.Encoding.UTF8.GetByteCount(line); + var bytes = System.Text.Encoding.UTF8.GetByteCount(line); + _byteCount += bytes; + _blockByteCount += bytes; if (_byteCount >= PageSize) { NewPage(); } + + if (_blockByteCount >= BlockSize) + { + NewBlock(); + } + } public void End() { EndPage(); - } - - private void Create() - { - NewPage(); + EndBlock(true); } private void NewPage() @@ -117,5 +142,27 @@ private void EndPage() _jobServerQueue.QueueFileUpload(_timelineId, _timelineRecordId, "DistributedTask.Core.Log", "CustomToolLog", _dataFileName, true); } } + + private void NewBlock() + { + EndBlock(false); + _blockByteCount = 0; + _resultsDataFileName = Path.Combine(_resultsBlockFolder, $"{_timelineId}_{_timelineRecordId}.{++_blockCount}"); + _resultsBlockData = new FileStream(_resultsDataFileName, FileMode.CreateNew, FileAccess.ReadWrite, FileShare.ReadWrite); + _resultsBlockWriter = new StreamWriter(_resultsBlockData, System.Text.Encoding.UTF8); + } + + private void EndBlock(bool finalize) + { + if (_resultsBlockWriter != null) + { + _resultsBlockWriter.Flush(); + _resultsBlockData.Flush(); + _resultsBlockWriter.Dispose(); + _resultsBlockWriter = null; + _resultsBlockData = null; + _jobServerQueue.QueueResultsUpload(_timelineRecordId, "ResultsLog", _resultsDataFileName, "Results.Core.Log", deleteSource: true, finalize, firstBlock: _resultsDataFileName.EndsWith(".1"), totalLines: _totalLines); + } + } } } diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 9bffb73a993..8d981c5492c 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -81,7 +81,7 @@ public interface IExecutionContext : IRunnerService // logging long Write(string tag, string message); void QueueAttachFile(string type, string name, string filePath); - void QueueSummaryFile(string name, string filePath, Guid stepRecordId); + void QueueSummaryFile(string name, string filePath, Guid stepRecordId); // timeline record update methods void Start(string currentOperation = null); @@ -871,8 +871,7 @@ public void QueueSummaryFile(string name, string filePath, Guid stepRecordId) { throw new FileNotFoundException($"Can't upload (name:{name}) file: {filePath}. File does not exist."); } - - _jobServerQueue.QueueSummaryUpload(stepRecordId, name, filePath, deleteSource: false); + _jobServerQueue.QueueResultsUpload(stepRecordId, name, filePath, ChecksAttachmentType.StepSummary, deleteSource: false, finalize: true, firstBlock: true, totalLines: 0); } // Add OnMatcherChanged diff --git a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs index 17027d1261a..1bb2a628edf 100644 --- a/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs +++ b/src/Sdk/DTWebApi/WebApi/TaskAttachment.cs @@ -1,4 +1,4 @@ -using GitHub.Services.Common; +using GitHub.Services.Common; using GitHub.Services.WebApi; using System; using System.Runtime.Serialization; @@ -100,6 +100,7 @@ public class CoreAttachmentType public static readonly String Summary = "DistributedTask.Core.Summary"; public static readonly String FileAttachment = "DistributedTask.Core.FileAttachment"; public static readonly String DiagnosticLog = "DistributedTask.Core.DiagnosticLog"; + public static readonly String ResultsLog = "Results.Core.Log"; } [GenerateAllConstants] diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index d240cc1e230..bc6361d62bc 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -28,6 +28,30 @@ public class GetSignedStepSummaryURLResponse public string BlobStorageType; } + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepLogsURLRequest + { + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string WorkflowRunBackendId; + [DataMember] + public string StepBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepLogsURLResponse + { + [DataMember] + public string LogsUrl; + [DataMember] + public long SoftSizeLimit; + [DataMember] + public string BlobStorageType; + } + [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] public class StepSummaryMetadataCreate @@ -52,6 +76,30 @@ public class CreateStepSummaryMetadataResponse public bool Ok; } + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class StepLogsMetadataCreate + { + [DataMember] + public string StepBackendId; + [DataMember] + public string WorkflowRunBackendId; + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string UploadedAt; + [DataMember] + public long LineCount; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class CreateStepLogsMetadataResponse + { + [DataMember] + public bool Ok; + } + public static class BlobStorageTypes { public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE"; diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 77a733eaeba..009308b98d6 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -24,13 +24,13 @@ public ResultsHttpClient( m_formatter = new JsonMediaTypeFormatter(); } - public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, string stepId, CancellationToken cancellationToken) + public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) { var request = new GetSignedStepSummaryURLRequest() { - WorkflowJobRunBackendId= jobId, - WorkflowRunBackendId= planId, - StepBackendId= stepId + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + StepBackendId = stepId.ToString() }; var stepSummaryUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepSummarySignedBlobURL"); @@ -51,14 +51,41 @@ public async Task GetStepSummaryUploadUrlAsync( } } - private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, string stepId, long size, CancellationToken cancellationToken) + public async Task GetStepLogUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) + { + var request = new GetSignedStepLogsURLRequest() + { + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + StepBackendId = stepId.ToString(), + }; + + var stepLogsUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepLogsSignedBlobURL"); + + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadRequest)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + return await ReadJsonContentAsync(response, cancellationToken); + } + } + } + } + + private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, Guid stepId, long size, CancellationToken cancellationToken) { var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); var request = new StepSummaryMetadataCreate() { - WorkflowJobRunBackendId= jobId, - WorkflowRunBackendId= planId, - StepBackendId = stepId, + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + StepBackendId = stepId.ToString(), Size = size, UploadedAt = timestamp }; @@ -85,6 +112,40 @@ private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, s } } + private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken) + { + var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); + var request = new StepLogsMetadataCreate() + { + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + StepBackendId = stepId.ToString(), + UploadedAt = timestamp, + LineCount = lineCount, + }; + + var stepLogsUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepLogsMetadata"); + + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadCompleteRequest)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); + if (!jsonResponse.Ok) + { + throw new Exception($"Failed to mark step log upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, timestamp: {timestamp}"); + } + } + } + } + } + private async Task UploadFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) { // Upload the file to the url @@ -108,8 +169,55 @@ private async Task UploadFileAsync(string url, string blobS } } + private async Task CreateAppendFileAsync(string url, string blobStorageType, CancellationToken cancellationToken) + { + var request = new HttpRequestMessage(HttpMethod.Put, url) + { + Content = new StringContent("") + }; + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("x-ms-blob-type", "AppendBlob"); + request.Content.Headers.Add("Content-Length", "0"); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to create append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}"); + } + return response; + } + } + + private async Task UploadAppendFileAsync(string url, string blobStorageType, FileStream file, bool finalize, long fileSize, CancellationToken cancellationToken) + { + var comp = finalize ? "&comp=appendblock&seal=true" : "&comp=appendblock"; + // Upload the file to the url + var request = new HttpRequestMessage(HttpMethod.Put, url + comp) + { + Content = new StreamContent(file) + }; + + if (blobStorageType == BlobStorageTypes.AzureBlobStorage) + { + request.Content.Headers.Add("Content-Length", fileSize.ToString()); + request.Content.Headers.Add("x-ms-blob-sealed", finalize.ToString()); + } + + using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) + { + if (!response.IsSuccessStatusCode) + { + throw new Exception($"Failed to upload append file, status code: {response.StatusCode}, reason: {response.ReasonPhrase}, object: {response}, fileSize: {fileSize}"); + } + return response; + } + } + // Handle file upload for step summary - public async Task UploadStepSummaryAsync(string planId, string jobId, string stepId, string file, CancellationToken cancellationToken) + public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) { // Get the upload url var uploadUrlResponse = await GetStepSummaryUploadUrlAsync(planId, jobId, stepId, cancellationToken); @@ -135,6 +243,39 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, string ste await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); } + // Handle file upload for step log + public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + // Get the upload url + var uploadUrlResponse = await GetStepLogUploadUrlAsync(planId, jobId, stepId, cancellationToken); + if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null) + { + throw new Exception("Failed to get step log upload url"); + } + + // Do we want to throw an exception here or should we just be uploading/truncating the data + var fileSize = new FileInfo(file).Length; + + // Create the Append blob + if (firstBlock) + { + await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + } + + // Upload content + using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) + { + var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken); + } + + // Update metadata + if (finalize) + { + // Send step log upload complete message + await StepLogUploadCompleteAsync(planId, jobId, stepId, lineCount, cancellationToken); + } + } + private MediaTypeFormatter m_formatter; private Uri m_resultsServiceUrl; private string m_token; From 348a30e2c301d09b33688925bec91fa501f09d7c Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Tue, 21 Feb 2023 09:55:47 -0500 Subject: [PATCH 101/145] Add job log upload support (#2447) * Refactor and add job log upload support * Rename method to be consistent --- src/Runner.Common/JobServer.cs | 9 + src/Runner.Common/JobServerQueue.cs | 70 ++++---- src/Sdk/WebApi/WebApi/Contracts.cs | 58 +++++-- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 186 ++++++++++++++------- 4 files changed, 211 insertions(+), 112 deletions(-) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 085cc84979d..9d845037868 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -32,6 +32,7 @@ public interface IJobServer : IRunnerService, IAsyncDisposable Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken); Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); + Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); @@ -335,6 +336,14 @@ public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, throw new InvalidOperationException("Results client is not initialized."); } + public Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadResultsJobLogAsync(planId, jobId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken); + } + throw new InvalidOperationException("Results client is not initialized."); + } public Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) { diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index f5f5a5494fe..1fadd99a8df 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -85,6 +85,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private bool _firstConsoleOutputs = true; private bool _resultsClientInitiated = false; + private delegate Task ResultsFileUploadHandler(ResultsUploadFileInfo file); public override void Initialize(IHostContext hostContext) { @@ -252,12 +253,6 @@ public void QueueResultsUpload(Guid timelineRecordId, string name, string path, return; } - if (timelineRecordId == _jobTimelineRecordId && String.Equals(type, CoreAttachmentType.ResultsLog, StringComparison.Ordinal)) - { - Trace.Verbose("Skipping job log {0} for record {1}", path, timelineRecordId); - return; - } - // all parameter not null, file path exist. var newFile = new ResultsUploadFileInfo() { @@ -503,8 +498,16 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) } else if (String.Equals(file.Type, CoreAttachmentType.ResultsLog, StringComparison.OrdinalIgnoreCase)) { - Trace.Info($"Got a step log file to send to results service."); - await UploadResultsStepLogFile(file); + if (file.RecordId != _jobTimelineRecordId) + { + Trace.Info($"Got a step log file to send to results service."); + await UploadResultsStepLogFile(file); + } + else if (file.RecordId == _jobTimelineRecordId) + { + Trace.Info($"Got a job log file to send to results service."); + await UploadResultsJobLogFile(file); + } } } catch (Exception ex) @@ -813,40 +816,43 @@ private async Task UploadFile(UploadFileInfo file) private async Task UploadSummaryFile(ResultsUploadFileInfo file) { - bool uploadSucceed = false; - try + Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); + ResultsFileUploadHandler summaryHandler = async (file) => { - // Upload the step summary - Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); await _jobServer.CreateStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, CancellationToken.None); + }; - uploadSucceed = true; - } - finally - { - if (uploadSucceed && file.DeleteSource) - { - try - { - File.Delete(file.Path); - } - catch (Exception ex) - { - Trace.Info("Catch exception during delete success results uploaded summary file."); - Trace.Error(ex); - } - } - } + await UploadResultsFile(file, summaryHandler); } private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file) { - bool uploadSucceed = false; - try + Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); + ResultsFileUploadHandler stepLogHandler = async (file) => { - Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); await _jobServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); + }; + + await UploadResultsFile(file, stepLogHandler); + } + private async Task UploadResultsJobLogFile(ResultsUploadFileInfo file) + { + Trace.Info($"Starting upload of job log file to results service {file.Name}, {file.Path}"); + ResultsFileUploadHandler jobLogHandler = async (file) => + { + await _jobServer.CreateResultsJobLogAsync(file.PlanId, file.JobId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); + }; + + await UploadResultsFile(file, jobLogHandler); + } + + private async Task UploadResultsFile(ResultsUploadFileInfo file, ResultsFileUploadHandler uploadHandler) + { + bool uploadSucceed = false; + try + { + await uploadHandler(file); uploadSucceed = true; } finally diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index bc6361d62bc..9165d832fa9 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -30,63 +30,89 @@ public class GetSignedStepSummaryURLResponse [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class GetSignedStepLogsURLRequest + public class StepSummaryMetadataCreate { [DataMember] - public string WorkflowJobRunBackendId; + public string StepBackendId; [DataMember] public string WorkflowRunBackendId; [DataMember] - public string StepBackendId; + public string WorkflowJobRunBackendId; + [DataMember] + public long Size; + [DataMember] + public string UploadedAt; } [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class GetSignedStepLogsURLResponse + public class GetSignedJobLogsURLRequest { [DataMember] - public string LogsUrl; + public string WorkflowJobRunBackendId; [DataMember] - public long SoftSizeLimit; + public string WorkflowRunBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedJobLogsURLResponse + { + [DataMember] + public string LogsUrl; [DataMember] public string BlobStorageType; } [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class StepSummaryMetadataCreate + public class GetSignedStepLogsURLRequest { [DataMember] - public string StepBackendId; + public string WorkflowJobRunBackendId; [DataMember] public string WorkflowRunBackendId; [DataMember] - public string WorkflowJobRunBackendId; + public string StepBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class GetSignedStepLogsURLResponse + { [DataMember] - public long Size; + public string LogsUrl; [DataMember] - public string UploadedAt; + public string BlobStorageType; + [DataMember] + public long SoftSizeLimit; } [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class CreateStepSummaryMetadataResponse + public class JobLogsMetadataCreate { [DataMember] - public bool Ok; + public string WorkflowRunBackendId; + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string UploadedAt; + [DataMember] + public long LineCount; } [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] public class StepLogsMetadataCreate { - [DataMember] - public string StepBackendId; [DataMember] public string WorkflowRunBackendId; [DataMember] public string WorkflowJobRunBackendId; [DataMember] + public string StepBackendId; + [DataMember] public string UploadedAt; [DataMember] public long LineCount; @@ -94,7 +120,7 @@ public class StepLogsMetadataCreate [DataContract] [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] - public class CreateStepLogsMetadataResponse + public class CreateMetadataResponse { [DataMember] public bool Ok; diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 009308b98d6..b80ce4db3bc 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -24,34 +24,40 @@ public ResultsHttpClient( m_formatter = new JsonMediaTypeFormatter(); } - public async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) + // Get Sas URL calls + private async Task GetResultsSignedURLResponse(Uri uri, CancellationToken cancellationToken, R request) { - var request = new GetSignedStepSummaryURLRequest() - { - WorkflowJobRunBackendId = jobId, - WorkflowRunBackendId = planId, - StepBackendId = stepId.ToString() - }; - - var stepSummaryUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepSummarySignedBlobURL"); - - using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepSummaryUploadRequest)) + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri)) { requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); - using (HttpContent content = new ObjectContent(request, m_formatter)) + using (HttpContent content = new ObjectContent(request, m_formatter)) { requestMessage.Content = content; using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) { - return await ReadJsonContentAsync(response, cancellationToken); + return await ReadJsonContentAsync(response, cancellationToken); } } } } - public async Task GetStepLogUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) + private async Task GetStepSummaryUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) + { + var request = new GetSignedStepSummaryURLRequest() + { + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + StepBackendId = stepId.ToString() + }; + + var getStepSummarySignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetStepSummarySignedBlobURL); + + return await GetResultsSignedURLResponse(getStepSummarySignedBlobURLEndpoint, cancellationToken, request); + } + + private async Task GetStepLogUploadUrlAsync(string planId, string jobId, Guid stepId, CancellationToken cancellationToken) { var request = new GetSignedStepLogsURLRequest() { @@ -60,19 +66,43 @@ public async Task GetStepLogUploadUrlAsync(string StepBackendId = stepId.ToString(), }; - var stepLogsUploadRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/GetStepLogsSignedBlobURL"); + var getStepLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetStepLogsSignedBlobURL); - using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadRequest)) + return await GetResultsSignedURLResponse(getStepLogsSignedBlobURLEndpoint, cancellationToken, request); + } + + private async Task GetJobLogUploadUrlAsync(string planId, string jobId, CancellationToken cancellationToken) + { + var request = new GetSignedJobLogsURLRequest() + { + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + }; + + var getJobLogsSignedBlobURLEndpoint = new Uri(m_resultsServiceUrl, Constants.GetJobLogsSignedBlobURL); + + return await GetResultsSignedURLResponse(getJobLogsSignedBlobURLEndpoint, cancellationToken, request); + } + + // Create metadata calls + + private async Task CreateMetadata(Uri uri, CancellationToken cancellationToken, R request, string timestamp) + { + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri)) { requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); - using (HttpContent content = new ObjectContent(request, m_formatter)) + using (HttpContent content = new ObjectContent(request, m_formatter)) { requestMessage.Content = content; using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) { - return await ReadJsonContentAsync(response, cancellationToken); + var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); + if (!jsonResponse.Ok) + { + throw new Exception($"Failed to mark {typeof(R).Name} upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, timestamp: {timestamp}"); + } } } } @@ -80,7 +110,7 @@ public async Task GetStepLogUploadUrlAsync(string private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, Guid stepId, long size, CancellationToken cancellationToken) { - var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); + var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat); var request = new StepSummaryMetadataCreate() { WorkflowJobRunBackendId = jobId, @@ -90,31 +120,13 @@ private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, G UploadedAt = timestamp }; - var stepSummaryUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepSummaryMetadata"); - - using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepSummaryUploadCompleteRequest)) - { - requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); - requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); - - using (HttpContent content = new ObjectContent(request, m_formatter)) - { - requestMessage.Content = content; - using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) - { - var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); - if (!jsonResponse.Ok) - { - throw new Exception($"Failed to mark step summary upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, size: {size}, timestamp: {timestamp}"); - } - } - } - } + var createStepSummaryMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepSummaryMetadata); + await CreateMetadata(createStepSummaryMetadataEndpoint, cancellationToken, request, timestamp); } private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken) { - var timestamp = DateTime.UtcNow.ToString("yyyy-MM-dd'T'HH:mm:ss.fffK"); + var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat); var request = new StepLogsMetadataCreate() { WorkflowJobRunBackendId = jobId, @@ -124,29 +136,26 @@ private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid LineCount = lineCount, }; - var stepLogsUploadCompleteRequest = new Uri(m_resultsServiceUrl, "twirp/results.services.receiver.Receiver/CreateStepLogsMetadata"); + var createStepLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepLogsMetadata); + await CreateMetadata(createStepLogsMetadataEndpoint, cancellationToken, request, timestamp); + } - using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, stepLogsUploadCompleteRequest)) + private async Task JobLogUploadCompleteAsync(string planId, string jobId, long lineCount, CancellationToken cancellationToken) + { + var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat); + var request = new JobLogsMetadataCreate() { - requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); - requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + WorkflowJobRunBackendId = jobId, + WorkflowRunBackendId = planId, + UploadedAt = timestamp, + LineCount = lineCount, + }; - using (HttpContent content = new ObjectContent(request, m_formatter)) - { - requestMessage.Content = content; - using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) - { - var jsonResponse = await ReadJsonContentAsync(response, cancellationToken); - if (!jsonResponse.Ok) - { - throw new Exception($"Failed to mark step log upload as complete, status code: {response.StatusCode}, ok: {jsonResponse.Ok}, timestamp: {timestamp}"); - } - } - } - } + var createJobLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateJobLogsMetadata); + await CreateMetadata(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp); } - private async Task UploadFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) + private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) { // Upload the file to the url var request = new HttpRequestMessage(HttpMethod.Put, url) @@ -156,7 +165,7 @@ private async Task UploadFileAsync(string url, string blobS if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - request.Content.Headers.Add("x-ms-blob-type", "BlockBlob"); + request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureBlockBlob); } using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) @@ -177,7 +186,7 @@ private async Task CreateAppendFileAsync(string url, string }; if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { - request.Content.Headers.Add("x-ms-blob-type", "AppendBlob"); + request.Content.Headers.Add(Constants.AzureBlobTypeHeader, Constants.AzureAppendBlob); request.Content.Headers.Add("Content-Length", "0"); } @@ -203,7 +212,7 @@ private async Task UploadAppendFileAsync(string url, string if (blobStorageType == BlobStorageTypes.AzureBlobStorage) { request.Content.Headers.Add("Content-Length", fileSize.ToString()); - request.Content.Headers.Add("x-ms-blob-sealed", finalize.ToString()); + request.Content.Headers.Add(Constants.AzureBlobSealedHeader, finalize.ToString()); } using (var response = await SendAsync(request, HttpCompletionOption.ResponseHeadersRead, userState: null, cancellationToken)) @@ -236,7 +245,7 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepI // Upload the file using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { - var response = await UploadFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); + var response = await UploadBlockFileAsync(uploadUrlResponse.SummaryUrl, uploadUrlResponse.BlobStorageType, fileStream, cancellationToken); } // Send step summary upload complete message @@ -253,8 +262,36 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st throw new Exception("Failed to get step log upload url"); } - // Do we want to throw an exception here or should we just be uploading/truncating the data + // Create the Append blob + if (firstBlock) + { + await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); + } + + // Upload content var fileSize = new FileInfo(file).Length; + using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) + { + var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken); + } + + // Update metadata + if (finalize) + { + // Send step log upload complete message + await StepLogUploadCompleteAsync(planId, jobId, stepId, lineCount, cancellationToken); + } + } + + // Handle file upload for job log + public async Task UploadResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + // Get the upload url + var uploadUrlResponse = await GetJobLogUploadUrlAsync(planId, jobId, cancellationToken); + if (uploadUrlResponse == null || uploadUrlResponse.LogsUrl == null) + { + throw new Exception("Failed to get job log upload url"); + } // Create the Append blob if (firstBlock) @@ -263,6 +300,7 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st } // Upload content + var fileSize = new FileInfo(file).Length; using (var fileStream = new FileStream(file, FileMode.Open, FileAccess.Read, FileShare.Read, 4096, true)) { var response = await UploadAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, fileStream, finalize, fileSize, cancellationToken); @@ -272,7 +310,7 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st if (finalize) { // Send step log upload complete message - await StepLogUploadCompleteAsync(planId, jobId, stepId, lineCount, cancellationToken); + await JobLogUploadCompleteAsync(planId, jobId, lineCount, cancellationToken); } } @@ -280,4 +318,24 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st private Uri m_resultsServiceUrl; private string m_token; } + + // Constants specific to results + public static class Constants + { + public static readonly string TimestampFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public static readonly string ResultsReceiverTwirpEndpoint = "twirp/results.services.receiver.Receiver/"; + public static readonly string GetStepSummarySignedBlobURL = ResultsReceiverTwirpEndpoint + "GetStepSummarySignedBlobURL"; + public static readonly string CreateStepSummaryMetadata = ResultsReceiverTwirpEndpoint + "CreateStepSummaryMetadata"; + public static readonly string GetStepLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetStepLogsSignedBlobURL"; + public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata"; + public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL"; + public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata"; + + public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed"; + public static readonly string AzureBlobTypeHeader = "x-ms-blob-type"; + public static readonly string AzureBlockBlob = "BlockBlob"; + public static readonly string AzureAppendBlob = "AppendBlob"; + } + } From b1431965b2dace4a5144470bd8b08d77b3280d99 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Mon, 27 Feb 2023 11:50:28 -0500 Subject: [PATCH 102/145] call run service renewjob (#2461) * call run service renewjob * format * formatting * make it private and expose internals * lint * fix exception class * lint * fix test as well --- src/Runner.Common/RunServer.cs | 18 +- src/Runner.Common/Util/MessageUtil.cs | 14 + src/Runner.Listener/InternalsVisibleTo.cs | 3 + src/Runner.Listener/JobDispatcher.cs | 405 +++++++++++++----- src/Runner.Listener/Runner.cs | 5 +- src/Runner.Worker/JobRunner.cs | 11 +- src/Sdk/RSWebApi/Contracts/RenewJobRequest.cs | 15 + .../RSWebApi/Contracts/RenewJobResponse.cs | 16 + src/Sdk/RSWebApi/RunServiceHttpClient.cs | 24 ++ src/Test/L0/Listener/JobDispatcherL0.cs | 171 +++++++- 10 files changed, 556 insertions(+), 126 deletions(-) create mode 100644 src/Runner.Common/Util/MessageUtil.cs create mode 100644 src/Runner.Listener/InternalsVisibleTo.cs create mode 100644 src/Sdk/RSWebApi/Contracts/RenewJobRequest.cs create mode 100644 src/Sdk/RSWebApi/Contracts/RenewJobResponse.cs diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index c1e25d8b84a..32c9f063fc0 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Threading; using System.Threading.Tasks; @@ -7,6 +7,7 @@ using GitHub.DistributedTask.WebApi; using GitHub.Runner.Sdk; using GitHub.Services.Common; +using Sdk.RSWebApi.Contracts; using Sdk.WebApi.WebApi.RawClient; namespace GitHub.Runner.Common @@ -19,6 +20,8 @@ public interface IRunServer : IRunnerService Task GetJobMessageAsync(string id, CancellationToken token); Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken token); + + Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken token); } public sealed class RunServer : RunnerService, IRunServer @@ -64,5 +67,18 @@ public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Diction return RetryRequest( async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, cancellationToken), cancellationToken); } + + public Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) + { + CheckConnection(); + var renewJobResponse = RetryRequest( + async () => await _runServiceHttpClient.RenewJobAsync(requestUri, planId, jobId, cancellationToken), cancellationToken); + if (renewJobResponse == null) + { + throw new TaskOrchestrationJobNotFoundException(jobId.ToString()); + } + + return renewJobResponse; + } } } diff --git a/src/Runner.Common/Util/MessageUtil.cs b/src/Runner.Common/Util/MessageUtil.cs new file mode 100644 index 00000000000..4489b447c31 --- /dev/null +++ b/src/Runner.Common/Util/MessageUtil.cs @@ -0,0 +1,14 @@ +namespace GitHub.Runner.Common.Util +{ + using System; + using GitHub.DistributedTask.WebApi; + + public static class MessageUtil + { + public static bool IsRunServiceJob(string messageType) + { + return string.Equals(messageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase); + } + } +} + diff --git a/src/Runner.Listener/InternalsVisibleTo.cs b/src/Runner.Listener/InternalsVisibleTo.cs new file mode 100644 index 00000000000..0012a294927 --- /dev/null +++ b/src/Runner.Listener/InternalsVisibleTo.cs @@ -0,0 +1,3 @@ +using System.Runtime.CompilerServices; + +[assembly: InternalsVisibleTo("Test")] diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index aaa83b80c78..53380b9b220 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -7,6 +7,7 @@ using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; +using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; using GitHub.Runner.Common.Util; @@ -58,6 +59,8 @@ public sealed class JobDispatcher : RunnerService, IJobDispatcher public event EventHandler JobStatus; + private bool _isRunServiceJob; + public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); @@ -86,6 +89,8 @@ public void Run(Pipelines.AgentJobRequestMessage jobRequestMessage, bool runOnce { Trace.Info($"Job request {jobRequestMessage.RequestId} for plan {jobRequestMessage.Plan.PlanId} job {jobRequestMessage.JobId} received."); + _isRunServiceJob = MessageUtil.IsRunServiceJob(jobRequestMessage.MessageType); + WorkerDispatcher currentDispatch = null; if (_jobDispatchedQueue.Count > 0) { @@ -239,6 +244,13 @@ private async Task EnsureDispatchFinished(WorkerDispatcher jobDispatch, bool can return; } + if (this._isRunServiceJob) + { + Trace.Error($"We are not yet checking the state of jobrequest {jobDispatch.JobId} status. Cancel running worker right away."); + jobDispatch.WorkerCancellationTokenSource.Cancel(); + return; + } + // based on the current design, server will only send one job for a given runner at a time. // if the runner received a new job request while a previous job request is still running, this typically indicates two situations // 1. a runner bug caused a server and runner mismatch on the state of the job request, e.g. the runner didn't renew the jobrequest @@ -367,9 +379,11 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc long requestId = message.RequestId; Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat + var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + // start renew job request Trace.Info($"Start renew job request {requestId} for job {message.JobId}."); - Task renewJobRequest = RenewJobRequestAsync(_poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, lockRenewalTokenSource.Token); + Task renewJobRequest = RenewJobRequestAsync(message, systemConnection, _poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, lockRenewalTokenSource.Token); // wait till first renew succeed or job request is cancelled // not even start worker if the first renew fail @@ -426,7 +440,7 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc { workerOutput.Add(stdout.Data); } - + if (printToStdout) { term.WriteLine(stdout.Data, skipTracing: true); @@ -508,7 +522,6 @@ await processChannel.SendAsync( // we get first jobrequest renew succeed and start the worker process with the job message. // send notification to machine provisioner. - var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); var accessToken = systemConnection?.Authorization?.Parameters["AccessToken"]; notification.JobStarted(message.JobId, accessToken, systemConnection.Url); @@ -531,11 +544,8 @@ await processChannel.SendAsync( detailInfo = string.Join(Environment.NewLine, workerOutput); Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result."); - var jobServer = HostContext.GetService(); - VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); - VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential); - await jobServer.ConnectAsync(jobConnection); + var jobServer = await InitializeJobServerAsync(systemConnection); await LogWorkerProcessUnhandledException(jobServer, message, detailInfo); // Go ahead to finish the job with result 'Failed' if the STDERR from worker is System.IO.IOException, since it typically means we are running out of disk space. @@ -675,9 +685,128 @@ await processChannel.SendAsync( } } - public async Task RenewJobRequestAsync(int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource firstJobRequestRenewed, CancellationToken token) + internal async Task RenewJobRequestAsync(Pipelines.AgentJobRequestMessage message, ServiceEndpoint systemConnection, int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource firstJobRequestRenewed, CancellationToken token) + { + if (this._isRunServiceJob) + { + var runServer = await GetRunServerAsync(systemConnection); + await RenewJobRequestAsync(runServer, message.Plan.PlanId, message.JobId, firstJobRequestRenewed, token); + } + else + { + var runnerServer = HostContext.GetService(); + await RenewJobRequestAsync(runnerServer, poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, token); + } + } + + private async Task RenewJobRequestAsync(IRunServer runServer, Guid planId, Guid jobId, TaskCompletionSource firstJobRequestRenewed, CancellationToken token) + { + TaskAgentJobRequest request = null; + int firstRenewRetryLimit = 5; + int encounteringError = 0; + + // renew lock during job running. + // stop renew only if cancellation token for lock renew task been signal or exception still happen after retry. + while (!token.IsCancellationRequested) + { + try + { + var renewResponse = await runServer.RenewJobAsync(planId, jobId, token); + Trace.Info($"Successfully renew job {jobId}, job is valid till {renewResponse.LockedUntil}"); + + if (!firstJobRequestRenewed.Task.IsCompleted) + { + // fire first renew succeed event. + firstJobRequestRenewed.TrySetResult(0); + } + + if (encounteringError > 0) + { + encounteringError = 0; + HostContext.WritePerfCounter("JobRenewRecovered"); + } + + // renew again after 60 sec delay + await HostContext.Delay(TimeSpan.FromSeconds(60), token); + } + catch (TaskOrchestrationJobNotFoundException) + { + // no need for retry. the job is not valid anymore. + Trace.Info($"TaskAgentJobNotFoundException received when renew job {jobId}, job is no longer valid, stop renew job request."); + return; + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + // OperationCanceledException may caused by http timeout or _lockRenewalTokenSource.Cance(); + // Stop renew only on cancellation token fired. + Trace.Info($"job renew has been cancelled, stop renew job {jobId}."); + return; + } + catch (Exception ex) + { + Trace.Error($"Catch exception during renew runner job {jobId}."); + Trace.Error(ex); + encounteringError++; + + // retry + TimeSpan remainingTime = TimeSpan.Zero; + if (!firstJobRequestRenewed.Task.IsCompleted) + { + // retry 5 times every 10 sec for the first renew + if (firstRenewRetryLimit-- > 0) + { + remainingTime = TimeSpan.FromSeconds(10); + } + } + else + { + // retry till reach lockeduntil + 5 mins extra buffer. + remainingTime = request.LockedUntil.Value + TimeSpan.FromMinutes(5) - DateTime.UtcNow; + } + + if (remainingTime > TimeSpan.Zero) + { + TimeSpan delayTime; + if (!firstJobRequestRenewed.Task.IsCompleted) + { + Trace.Info($"Retrying lock renewal for job {jobId}. The first job renew request has failed."); + delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(10)); + } + else + { + Trace.Info($"Retrying lock renewal for job {jobId}. Job is valid until {request.LockedUntil.Value}."); + if (encounteringError > 5) + { + delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30)); + } + else + { + delayTime = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(5), TimeSpan.FromSeconds(15)); + } + } + + try + { + // back-off before next retry. + await HostContext.Delay(delayTime, token); + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + Trace.Info($"job renew has been cancelled, stop renew job {jobId}."); + } + } + else + { + Trace.Info($"Lock renewal has run out of retry, stop renew lock for job {jobId}."); + HostContext.WritePerfCounter("JobRenewReachLimit"); + return; + } + } + } + } + + private async Task RenewJobRequestAsync(IRunnerServer runnerServer, int poolId, long requestId, Guid lockToken, string orchestrationId, TaskCompletionSource firstJobRequestRenewed, CancellationToken token) { - var runnerServer = HostContext.GetService(); TaskAgentJobRequest request = null; int firstRenewRetryLimit = 5; int encounteringError = 0; @@ -840,90 +969,93 @@ private async Task TryUploadUnfinishedLogs(Pipelines.AgentJobRequestMessage mess var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection)); ArgUtil.NotNull(systemConnection, nameof(systemConnection)); - var jobServer = HostContext.GetService(); - VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); - VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential); - - await jobServer.ConnectAsync(jobConnection); - - var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); + var server = await InitializeJobServerAsync(systemConnection); - var updatedRecords = new List(); - var logPages = new Dictionary>(); - var logRecords = new Dictionary(); - foreach (var log in logs) + if (server is IJobServer jobServer) { - var logName = Path.GetFileNameWithoutExtension(log); - var logNameParts = logName.Split('_', StringSplitOptions.RemoveEmptyEntries); - if (logNameParts.Length != 3) - { - Trace.Warning($"log file '{log}' doesn't follow naming convension 'GUID_GUID_INT'."); - continue; - } - var logPageSeperator = logName.IndexOf('_'); - var logRecordId = Guid.Empty; - var pageNumber = 0; + var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); - if (!Guid.TryParse(logNameParts[0], out Guid timelineId) || timelineId != timeline.Id) + var updatedRecords = new List(); + var logPages = new Dictionary>(); + var logRecords = new Dictionary(); + foreach (var log in logs) { - Trace.Warning($"log file '{log}' is not belongs to current job"); - continue; - } + var logName = Path.GetFileNameWithoutExtension(log); + var logNameParts = logName.Split('_', StringSplitOptions.RemoveEmptyEntries); + if (logNameParts.Length != 3) + { + Trace.Warning($"log file '{log}' doesn't follow naming convension 'GUID_GUID_INT'."); + continue; + } + var logPageSeperator = logName.IndexOf('_'); + var logRecordId = Guid.Empty; + var pageNumber = 0; - if (!Guid.TryParse(logNameParts[1], out logRecordId)) - { - Trace.Warning($"log file '{log}' doesn't follow naming convension 'GUID_GUID_INT'."); - continue; - } + if (!Guid.TryParse(logNameParts[0], out Guid timelineId) || timelineId != timeline.Id) + { + Trace.Warning($"log file '{log}' is not belongs to current job"); + continue; + } - if (!int.TryParse(logNameParts[2], out pageNumber)) - { - Trace.Warning($"log file '{log}' doesn't follow naming convension 'GUID_GUID_INT'."); - continue; - } + if (!Guid.TryParse(logNameParts[1], out logRecordId)) + { + Trace.Warning($"log file '{log}' doesn't follow naming convension 'GUID_GUID_INT'."); + continue; + } - var record = timeline.Records.FirstOrDefault(x => x.Id == logRecordId); - if (record != null) - { - if (!logPages.ContainsKey(record.Id)) + if (!int.TryParse(logNameParts[2], out pageNumber)) { - logPages[record.Id] = new Dictionary(); - logRecords[record.Id] = record; + Trace.Warning($"log file '{log}' doesn't follow naming convension 'GUID_GUID_INT'."); + continue; } - logPages[record.Id][pageNumber] = log; + var record = timeline.Records.FirstOrDefault(x => x.Id == logRecordId); + if (record != null) + { + if (!logPages.ContainsKey(record.Id)) + { + logPages[record.Id] = new Dictionary(); + logRecords[record.Id] = record; + } + + logPages[record.Id][pageNumber] = log; + } } - } - foreach (var pages in logPages) - { - var record = logRecords[pages.Key]; - if (record.Log == null) + foreach (var pages in logPages) { - // Create the log - record.Log = await jobServer.CreateLogAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, new TaskLog(String.Format(@"logs\{0:D}", record.Id)), default(CancellationToken)); + var record = logRecords[pages.Key]; + if (record.Log == null) + { + // Create the log + record.Log = await jobServer.CreateLogAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, new TaskLog(String.Format(@"logs\{0:D}", record.Id)), default(CancellationToken)); - // Need to post timeline record updates to reflect the log creation - updatedRecords.Add(record.Clone()); - } + // Need to post timeline record updates to reflect the log creation + updatedRecords.Add(record.Clone()); + } - for (var i = 1; i <= pages.Value.Count; i++) - { - var logFile = pages.Value[i]; - // Upload the contents - using (FileStream fs = File.Open(logFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + for (var i = 1; i <= pages.Value.Count; i++) { - var logUploaded = await jobServer.AppendLogContentAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, record.Log.Id, fs, default(CancellationToken)); + var logFile = pages.Value[i]; + // Upload the contents + using (FileStream fs = File.Open(logFile, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + var logUploaded = await jobServer.AppendLogContentAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, record.Log.Id, fs, default(CancellationToken)); + } + + Trace.Info($"Uploaded unfinished log '{logFile}' for current job."); + IOUtil.DeleteFile(logFile); } + } - Trace.Info($"Uploaded unfinished log '{logFile}' for current job."); - IOUtil.DeleteFile(logFile); + if (updatedRecords.Count > 0) + { + await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, updatedRecords, CancellationToken.None); } } - - if (updatedRecords.Count > 0) + else { - await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, updatedRecords, CancellationToken.None); + Trace.Info("Job server does not support log upload yet."); } } catch (Exception ex) @@ -943,6 +1075,12 @@ private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequest return; } + if (this._isRunServiceJob) + { + Trace.Verbose($"Skip FinishAgentRequest call from Listener because MessageType is {message.MessageType}"); + return; + } + var runnerServer = HostContext.GetService(); int completeJobRequestRetryLimit = 5; List exceptions = new(); @@ -979,66 +1117,117 @@ private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequest } // log an error issue to job level timeline record - private async Task LogWorkerProcessUnhandledException(IJobServer jobServer, Pipelines.AgentJobRequestMessage message, string errorMessage) + private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string errorMessage) { - try + if (server is IJobServer jobServer) { - var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); - ArgUtil.NotNull(timeline, nameof(timeline)); - - TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job"); - ArgUtil.NotNull(jobRecord, nameof(jobRecord)); - try { - if (!string.IsNullOrEmpty(errorMessage) && - message.Variables.TryGetValue("DistributedTask.EnableRunnerIPCDebug", out var enableRunnerIPCDebug) && - StringUtil.ConvertToBoolean(enableRunnerIPCDebug.Value)) + var timeline = await jobServer.GetTimelineAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, CancellationToken.None); + ArgUtil.NotNull(timeline, nameof(timeline)); + + TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job"); + ArgUtil.NotNull(jobRecord, nameof(jobRecord)); + + try { - // the trace should be best effort and not affect any job result - var match = _invalidJsonRegex.Match(errorMessage); - if (match.Success && - match.Groups.Count == 2) + if (!string.IsNullOrEmpty(errorMessage) && + message.Variables.TryGetValue("DistributedTask.EnableRunnerIPCDebug", out var enableRunnerIPCDebug) && + StringUtil.ConvertToBoolean(enableRunnerIPCDebug.Value)) { - var jsonPosition = int.Parse(match.Groups[1].Value); - var serializedJobMessage = JsonUtility.ToString(message); - var originalJson = serializedJobMessage.Substring(jsonPosition - 10, 20); - errorMessage = $"Runner sent Json at position '{jsonPosition}': {originalJson} ({Convert.ToBase64String(Encoding.UTF8.GetBytes(originalJson))})\n{errorMessage}"; + // the trace should be best effort and not affect any job result + var match = _invalidJsonRegex.Match(errorMessage); + if (match.Success && + match.Groups.Count == 2) + { + var jsonPosition = int.Parse(match.Groups[1].Value); + var serializedJobMessage = JsonUtility.ToString(message); + var originalJson = serializedJobMessage.Substring(jsonPosition - 10, 20); + errorMessage = $"Runner sent Json at position '{jsonPosition}': {originalJson} ({Convert.ToBase64String(Encoding.UTF8.GetBytes(originalJson))})\n{errorMessage}"; + } } } + catch (Exception ex) + { + Trace.Error(ex); + errorMessage = $"Fail to check json IPC error: {ex.Message}\n{errorMessage}"; + } + + var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage }; + unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; + jobRecord.ErrorCount++; + jobRecord.Issues.Add(unhandledExceptionIssue); + await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); } catch (Exception ex) { + Trace.Error("Fail to report unhandled exception from Runner.Worker process"); Trace.Error(ex); - errorMessage = $"Fail to check json IPC error: {ex.Message}\n{errorMessage}"; } - - var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage }; - unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; - jobRecord.ErrorCount++; - jobRecord.Issues.Add(unhandledExceptionIssue); - await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); } - catch (Exception ex) + else { - Trace.Error("Fail to report unhandled exception from Runner.Worker process"); - Trace.Error(ex); + Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", errorMessage); + return; } } // raise job completed event to fail the job. - private async Task ForceFailJob(IJobServer jobServer, Pipelines.AgentJobRequestMessage message) + private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message) { - try + if (server is IJobServer jobServer) { - var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed); - await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None); + try + { + var jobCompletedEvent = new JobCompletedEvent(message.RequestId, message.JobId, TaskResult.Failed); + await jobServer.RaisePlanEventAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, jobCompletedEvent, CancellationToken.None); + } + catch (Exception ex) + { + Trace.Error("Fail to raise JobCompletedEvent back to service."); + Trace.Error(ex); + } } - catch (Exception ex) + else if (server is IRunServer runServer) { - Trace.Error("Fail to raise JobCompletedEvent back to service."); - Trace.Error(ex); + try + { + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, CancellationToken.None); + } + catch (Exception ex) + { + Trace.Error("Fail to raise job completion back to service."); + Trace.Error(ex); + } + } + else + { + throw new NotSupportedException($"Server type {server.GetType().FullName} is not supported."); + } + } + + private async Task InitializeJobServerAsync(ServiceEndpoint systemConnection) + { + if (this._isRunServiceJob) + { + return await GetRunServerAsync(systemConnection); } + else + { + var jobServer = HostContext.GetService(); + VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); + VssConnection jobConnection = VssUtil.CreateConnection(systemConnection.Url, jobServerCredential); + await jobServer.ConnectAsync(jobConnection); + return jobServer; + } + } + + private async Task GetRunServerAsync(ServiceEndpoint systemConnection) + { + var runServer = HostContext.GetService(); + VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); + await runServer.ConnectAsync(systemConnection.Url, jobServerCredential); + return runServer; } private class WorkerDispatcher : IDisposable diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index c28e5a9f7c4..30d60b2fc65 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -9,6 +9,7 @@ using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; using GitHub.Runner.Listener.Check; using GitHub.Runner.Listener.Configuration; using GitHub.Runner.Sdk; @@ -136,7 +137,7 @@ public async Task ExecuteCommand(CommandSettings command) if (command.Remove) { // only remove local config files and exit - if(command.RemoveLocalConfig) + if (command.RemoveLocalConfig) { configManager.DeleteLocalRunnerConfig(); return Constants.Runner.ReturnCode.Success; @@ -502,7 +503,7 @@ private async Task RunAsync(RunnerSettings settings, bool runOnce = false) } } // Broker flow - else if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase)) + else if (MessageUtil.IsRunServiceJob(message.MessageType)) { if (autoUpdateInProgress || runOnceJobReceived) { diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 41fc062d315..f63fbf71776 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -6,6 +6,7 @@ using System.Text; using System.Threading; using System.Threading.Tasks; +using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common; using GitHub.Runner.Common.Util; @@ -19,7 +20,7 @@ namespace GitHub.Runner.Worker [ServiceLocator(Default = typeof(JobRunner))] public interface IJobRunner : IRunnerService { - Task RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken); + Task RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken); } public sealed class JobRunner : RunnerService, IJobRunner @@ -28,7 +29,7 @@ public sealed class JobRunner : RunnerService, IJobRunner private RunnerSettings _runnerSettings; private ITempDirectoryManager _tempDirectoryManager; - public async Task RunAsync(Pipelines.AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) + public async Task RunAsync(AgentJobRequestMessage message, CancellationToken jobRequestCancellationToken) { // Validate parameters. Trace.Entering(); @@ -42,14 +43,14 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, IRunnerService server = null; ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - if (string.Equals(message.MessageType, JobRequestMessageTypes.RunnerJobRequest, StringComparison.OrdinalIgnoreCase)) + if (MessageUtil.IsRunServiceJob(message.MessageType)) { var runServer = HostContext.GetService(); VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); await runServer.ConnectAsync(systemConnection.Url, jobServerCredential); server = runServer; } - else + else { // Setup the job server and job server queue. var jobServer = HostContext.GetService(); @@ -65,7 +66,7 @@ public async Task RunAsync(Pipelines.AgentJobRequestMessage message, _jobServerQueue.Start(message); server = jobServer; } - + HostContext.WritePerfCounter($"WorkerJobServerQueueStarted_{message.RequestId.ToString()}"); diff --git a/src/Sdk/RSWebApi/Contracts/RenewJobRequest.cs b/src/Sdk/RSWebApi/Contracts/RenewJobRequest.cs new file mode 100644 index 00000000000..7afb0e3b736 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/RenewJobRequest.cs @@ -0,0 +1,15 @@ +using System; +using System.Runtime.Serialization; + +namespace GitHub.Actions.RunService.WebApi +{ + [DataContract] + public class RenewJobRequest + { + [DataMember(Name = "planId", EmitDefaultValue = false)] + public Guid PlanID { get; set; } + + [DataMember(Name = "jobId", EmitDefaultValue = false)] + public Guid JobID { get; set; } + } +} diff --git a/src/Sdk/RSWebApi/Contracts/RenewJobResponse.cs b/src/Sdk/RSWebApi/Contracts/RenewJobResponse.cs new file mode 100644 index 00000000000..387a3ac0197 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/RenewJobResponse.cs @@ -0,0 +1,16 @@ +using System; +using System.Runtime.Serialization; + +namespace Sdk.RSWebApi.Contracts +{ + [DataContract] + public class RenewJobResponse + { + [DataMember] + public DateTime LockedUntil + { + get; + internal set; + } + } +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 0bf46146ee2..2730e66657a 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -8,6 +8,7 @@ using GitHub.Services.Common; using GitHub.Services.OAuth; using GitHub.Services.WebApi; +using Sdk.RSWebApi.Contracts; using Sdk.WebApi.WebApi; namespace GitHub.Actions.RunService.WebApi @@ -98,6 +99,29 @@ public Task CompleteJobAsync( var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); return SendAsync( + httpMethod, + requestUri, + content: requestContent, + cancellationToken: cancellationToken); + } + + public Task RenewJobAsync( + Uri requestUri, + Guid planId, + Guid jobId, + CancellationToken cancellationToken = default) + { + HttpMethod httpMethod = new HttpMethod("POST"); + var payload = new RenewJobRequest() + { + PlanID = planId, + JobID = jobId + }; + + requestUri = new Uri(requestUri, "renewjob"); + + var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); + return SendAsync( httpMethod, requestUri, content: requestContent, diff --git a/src/Test/L0/Listener/JobDispatcherL0.cs b/src/Test/L0/Listener/JobDispatcherL0.cs index f1a74d5fe36..7788df3d5fc 100644 --- a/src/Test/L0/Listener/JobDispatcherL0.cs +++ b/src/Test/L0/Listener/JobDispatcherL0.cs @@ -1,12 +1,17 @@ -using System; +using System; using System.Collections.Generic; using System.Reflection; using System.Threading; using System.Threading.Tasks; +using GitHub.DistributedTask.ObjectTemplating.Tokens; +using GitHub.DistributedTask.Pipelines; +using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Listener; +using GitHub.Runner.Listener.Configuration; using GitHub.Services.WebApi; using Moq; +using Sdk.RSWebApi.Contracts; using Xunit; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -18,6 +23,8 @@ public sealed class JobDispatcherL0 private Mock _processChannel; private Mock _processInvoker; private Mock _runnerServer; + + private Mock _runServer; private Mock _configurationStore; public JobDispatcherL0() @@ -25,6 +32,7 @@ public JobDispatcherL0() _processChannel = new Mock(); _processInvoker = new Mock(); _runnerServer = new Mock(); + _runServer = new Mock(); _configurationStore = new Mock(); } @@ -139,7 +147,7 @@ public async void DispatcherRenewJobRequest() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully); _runnerServer.Verify(x => x.RenewAgentRequestAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); @@ -197,7 +205,7 @@ public async void DispatcherRenewJobRequestStopOnJobNotFoundExceptions() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.False(cancellationTokenSource.IsCancellationRequested); @@ -205,6 +213,75 @@ public async void DispatcherRenewJobRequestStopOnJobNotFoundExceptions() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Runner")] + public async void DispatcherRenewJobOnRunServiceStopOnJobNotFoundExceptions() + { + //Arrange + using (var hc = new TestHostContext(this)) + { + int poolId = 1; + Int64 requestId = 1000; + int count = 0; + + var trace = hc.GetTrace(nameof(DispatcherRenewJobOnRunServiceStopOnJobNotFoundExceptions)); + TaskCompletionSource firstJobRequestRenewed = new(); + CancellationTokenSource cancellationTokenSource = new(); + + TaskAgentJobRequest request = new(); + PropertyInfo lockUntilProperty = request.GetType().GetProperty("LockedUntil", BindingFlags.Instance | BindingFlags.NonPublic | BindingFlags.Public); + Assert.NotNull(lockUntilProperty); + lockUntilProperty.SetValue(request, DateTime.UtcNow.AddMinutes(5)); + + hc.SetSingleton(_runServer.Object); + hc.SetSingleton(_configurationStore.Object); + _configurationStore.Setup(x => x.GetSettings()).Returns(new RunnerSettings() { PoolId = 1 }); + _ = _runServer.Setup(x => x.RenewJobAsync(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns(() => + { + count++; + if (!firstJobRequestRenewed.Task.IsCompletedSuccessfully) + { + trace.Info("First renew happens."); + } + + if (count < 5) + { + var response = new RenewJobResponse() + { + LockedUntil = request.LockedUntil.Value + }; + return Task.FromResult(response); + } + else if (count == 5) + { + cancellationTokenSource.CancelAfter(10000); + throw new TaskOrchestrationJobNotFoundException(""); + } + else + { + throw new InvalidOperationException("Should not reach here."); + } + }); + + + var jobDispatcher = new JobDispatcher(); + jobDispatcher.Initialize(hc); + EnableRunServiceJobForJobDispatcher(jobDispatcher); + + // Set the value of the _isRunServiceJob field to true + var isRunServiceJobField = typeof(JobDispatcher).GetField("_isRunServiceJob", BindingFlags.NonPublic | BindingFlags.Instance); + isRunServiceJobField.SetValue(jobDispatcher, true); + + await jobDispatcher.RenewJobRequestAsync(GetAgentJobRequestMessage(), GetServiceEndpoint(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + + Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); + Assert.False(cancellationTokenSource.IsCancellationRequested); + _runServer.Verify(x => x.RenewJobAsync(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(5)); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "Runner")] @@ -256,7 +333,7 @@ public async void DispatcherRenewJobRequestStopOnJobTokenExpiredExceptions() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.False(cancellationTokenSource.IsCancellationRequested); @@ -312,8 +389,9 @@ public async void RenewJobRequestNewAgentNameUpdatesSettings() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); + // Act - await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), 0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); // Assert _configurationStore.Verify(x => x.SaveSettings(It.Is(settings => settings.AgentName == newName)), Times.Once); @@ -368,7 +446,7 @@ public async void RenewJobRequestSameAgentNameIgnored() jobDispatcher.Initialize(hc); // Act - await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), 0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); // Assert _configurationStore.Verify(x => x.SaveSettings(It.IsAny()), Times.Never); @@ -421,7 +499,7 @@ public async void RenewJobRequestNullAgentNameIgnored() jobDispatcher.Initialize(hc); // Act - await jobDispatcher.RenewJobRequestAsync(0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), 0, 0, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); // Assert _configurationStore.Verify(x => x.SaveSettings(It.IsAny()), Times.Never); @@ -479,7 +557,7 @@ public async void DispatcherRenewJobRequestRecoverFromExceptions() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.True(cancellationTokenSource.IsCancellationRequested); @@ -536,7 +614,7 @@ public async void DispatcherRenewJobRequestFirstRenewRetrySixTimes() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.False(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should failed."); Assert.False(cancellationTokenSource.IsCancellationRequested); @@ -600,7 +678,7 @@ public async void DispatcherRenewJobRequestStopOnExpiredRequest() var jobDispatcher = new JobDispatcher(); jobDispatcher.Initialize(hc); - await jobDispatcher.RenewJobRequestAsync(poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); + await jobDispatcher.RenewJobRequestAsync(It.IsAny(), It.IsAny(), poolId, requestId, Guid.Empty, Guid.NewGuid().ToString(), firstJobRequestRenewed, cancellationTokenSource.Token); Assert.True(firstJobRequestRenewed.Task.IsCompletedSuccessfully, "First renew should succeed."); Assert.False(cancellationTokenSource.IsCancellationRequested); @@ -659,5 +737,78 @@ public async void DispatchesOneTimeJobRequest() Assert.True(jobDispatcher.RunOnceJobCompleted.Task.Result, "JobDispatcher should set task complete token to 'TRUE' for one time agent."); } } + + private static void EnableRunServiceJobForJobDispatcher(JobDispatcher jobDispatcher) + { + // Set the value of the _isRunServiceJob field to true + var isRunServiceJobField = typeof(JobDispatcher).GetField("_isRunServiceJob", BindingFlags.NonPublic | BindingFlags.Instance); + isRunServiceJobField.SetValue(jobDispatcher, true); + } + + private static ServiceEndpoint GetServiceEndpoint() + { + var serviceEndpoint = new ServiceEndpoint + { + Authorization = new EndpointAuthorization + { + Scheme = EndpointAuthorizationSchemes.OAuth + } + }; + serviceEndpoint.Authorization.Parameters.Add("AccessToken", "token"); + return serviceEndpoint; + } + + private static AgentJobRequestMessage GetAgentJobRequestMessage() + { + var message = new AgentJobRequestMessage( + new TaskOrchestrationPlanReference() + { + PlanType = "Build", + PlanId = Guid.NewGuid(), + Version = 1 + }, + new TimelineReference() + { + Id = Guid.NewGuid() + }, + Guid.NewGuid(), + "jobDisplayName", + "jobName", + null, + null, + new List(), + new Dictionary() + { + { + "variables", + new VariableValue() + { + IsSecret = false, + Value = "variables" + } + } + }, + new List() + { + new MaskHint() + { + Type = MaskType.Variable, + Value = "maskHints" + } + }, + new JobResources(), + new DictionaryContextData(), + new WorkspaceOptions(), + new List(), + new List() + { + "fileTable" + }, + null, + new List(), + new ActionsEnvironmentReference("env") + ); + return message; + } } } From 7572af1a528ad28b67ea6bdcf4ccd067a71b5741 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Tue, 28 Feb 2023 16:00:36 +0100 Subject: [PATCH 103/145] Runner service exit after consecutive re-try exits (#2426) * Runner service exit after consecutive re-try exits * Rename failure counts and include reset count on runner listening for jobs * Changed from graceful shutdown to stopping=true --- src/Misc/layoutbin/RunnerService.js | 34 +++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/Misc/layoutbin/RunnerService.js b/src/Misc/layoutbin/RunnerService.js index bb4b64ed21e..ba0a8c659d7 100644 --- a/src/Misc/layoutbin/RunnerService.js +++ b/src/Misc/layoutbin/RunnerService.js @@ -24,7 +24,8 @@ if (exitServiceAfterNFailures <= 0) { exitServiceAfterNFailures = NaN; } -var consecutiveFailureCount = 0; +var unknownFailureRetryCount = 0; +var retriableFailureRetryCount = 0; var gracefulShutdown = function () { console.log("Shutting down runner listener"); @@ -62,7 +63,8 @@ var runService = function () { listener.stdout.on("data", (data) => { if (data.toString("utf8").includes("Listening for Jobs")) { - consecutiveFailureCount = 0; + unknownFailureRetryCount = 0; + retriableFailureRetryCount = 0; } process.stdout.write(data.toString("utf8")); }); @@ -92,24 +94,38 @@ var runService = function () { console.log( "Runner listener exit with retryable error, re-launch runner in 5 seconds." ); - consecutiveFailureCount = 0; + unknownFailureRetryCount = 0; + retriableFailureRetryCount++; + if (retriableFailureRetryCount >= 10) { + console.error( + "Stopping the runner after 10 consecutive re-tryable failures" + ); + stopping = true; + } } else if (code === 3 || code === 4) { console.log( "Runner listener exit because of updating, re-launch runner in 5 seconds." ); - consecutiveFailureCount = 0; + unknownFailureRetryCount = 0; + retriableFailureRetryCount++; + if (retriableFailureRetryCount >= 10) { + console.error( + "Stopping the runner after 10 consecutive re-tryable failures" + ); + stopping = true; + } } else { var messagePrefix = "Runner listener exit with undefined return code"; - consecutiveFailureCount++; + unknownFailureRetryCount++; + retriableFailureRetryCount = 0; if ( !isNaN(exitServiceAfterNFailures) && - consecutiveFailureCount >= exitServiceAfterNFailures + unknownFailureRetryCount >= exitServiceAfterNFailures ) { console.error( - `${messagePrefix}, exiting service after ${consecutiveFailureCount} consecutive failures` + `${messagePrefix}, exiting service after ${unknownFailureRetryCount} consecutive failures` ); - gracefulShutdown(); - return; + stopping = true } else { console.log(`${messagePrefix}, re-launch runner in 5 seconds.`); } From 5bca5364a082ed40e920232bf60fa8faea4e5495 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Thu, 2 Mar 2023 13:42:23 -0500 Subject: [PATCH 104/145] Change runner image to make user/folder align with ubuntu-latest hosted runner. (#2469) --- images/Dockerfile | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index 8fe339320c0..315196e49c2 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -24,11 +24,26 @@ RUN export DOCKER_ARCH=x86_64 \ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 -ENV RUNNER_ALLOW_RUNASROOT=1 +ENV DEBIAN_FRONTEND=noninteractive ENV RUNNER_MANUALLY_TRAP_SIG=1 ENV ACTIONS_RUNNER_PRINT_LOG_TO_STDOUT=1 -WORKDIR /actions-runner -COPY --from=build /actions-runner . +RUN apt-get update -y \ + && apt-get install -y --no-install-recommends \ + sudo \ + && rm -rf /var/lib/apt/lists/* + +RUN adduser --disabled-password --gecos "" --uid 1001 runner \ + && groupadd docker --gid 123 \ + && usermod -aG sudo runner \ + && usermod -aG docker runner \ + && echo "%sudo ALL=(ALL:ALL) NOPASSWD:ALL" > /etc/sudoers \ + && echo "Defaults env_keep += \"DEBIAN_FRONTEND\"" >> /etc/sudoers + +WORKDIR /home/runner + +COPY --chown=runner:docker --from=build /actions-runner . + +RUN install -o root -g root -m 755 docker/* /usr/bin/ && rm -rf docker -RUN install -o root -g root -m 755 docker/* /usr/bin/ && rm -rf docker \ No newline at end of file +USER runner \ No newline at end of file From 042d452db28736ffb20d34766f2aa3fa64b92fc0 Mon Sep 17 00:00:00 2001 From: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> Date: Mon, 6 Mar 2023 11:01:45 +0100 Subject: [PATCH 105/145] Bypass all proxies for all hosts if `no_proxy='*'` is set (#2395) * Bypass top level domain even if no_proxy specified it with leading '.' E.g. no_proxy='.github.com' will now bypass github.com. * Bypass proxy on all hosts when no_proxy is * (wildcard) * Undo '.' stripping * Simplify unit tests * Respect wildcard even if it's one of many no_proxy items --- src/Runner.Sdk/RunnerWebProxy.cs | 6 ++- src/Test/L0/RunnerWebProxyL0.cs | 72 +++++++++++++++++++++++++++++++- 2 files changed, 76 insertions(+), 2 deletions(-) diff --git a/src/Runner.Sdk/RunnerWebProxy.cs b/src/Runner.Sdk/RunnerWebProxy.cs index 516b1dcec16..ccf83f6faa2 100644 --- a/src/Runner.Sdk/RunnerWebProxy.cs +++ b/src/Runner.Sdk/RunnerWebProxy.cs @@ -164,7 +164,6 @@ public RunnerWebProxy() { continue; } - _noProxyList.Add(noProxyInfo); } } @@ -207,6 +206,11 @@ private bool IsUriInBypassList(Uri input) { foreach (var noProxy in _noProxyList) { + // bypass on wildcard no_proxy + if (string.Equals(noProxy.Host, "*", StringComparison.OrdinalIgnoreCase)) + { + return true; + } var matchHost = false; var matchPort = false; diff --git a/src/Test/L0/RunnerWebProxyL0.cs b/src/Test/L0/RunnerWebProxyL0.cs index 649f06197a7..dacc6373110 100644 --- a/src/Test/L0/RunnerWebProxyL0.cs +++ b/src/Test/L0/RunnerWebProxyL0.cs @@ -351,7 +351,7 @@ public void WebProxyFromEnvironmentVariablesNoProxy() Assert.False(proxy.IsBypassed(new Uri("https://actions.com"))); Assert.False(proxy.IsBypassed(new Uri("https://ggithub.com"))); Assert.False(proxy.IsBypassed(new Uri("https://github.comm"))); - Assert.False(proxy.IsBypassed(new Uri("https://google.com"))); + Assert.False(proxy.IsBypassed(new Uri("https://google.com"))); // no_proxy has '.google.com', specifying only subdomains bypass Assert.False(proxy.IsBypassed(new Uri("https://example.com"))); Assert.False(proxy.IsBypassed(new Uri("http://example.com:333"))); Assert.False(proxy.IsBypassed(new Uri("http://192.168.0.123:123"))); @@ -374,6 +374,76 @@ public void WebProxyFromEnvironmentVariablesNoProxy() CleanProxyEnv(); } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void BypassAllOnWildcardNoProxy() + { + try + { + Environment.SetEnvironmentVariable("http_proxy", "http://user1:pass1%40@127.0.0.1:8888"); + Environment.SetEnvironmentVariable("https_proxy", "http://user2:pass2%40@127.0.0.1:9999"); + Environment.SetEnvironmentVariable("no_proxy", "example.com, * , example2.com"); + var proxy = new RunnerWebProxy(); + + Assert.True(proxy.IsBypassed(new Uri("http://actions.com"))); + Assert.True(proxy.IsBypassed(new Uri("http://localhost"))); + Assert.True(proxy.IsBypassed(new Uri("http://127.0.0.1:8080"))); + Assert.True(proxy.IsBypassed(new Uri("https://actions.com"))); + Assert.True(proxy.IsBypassed(new Uri("https://localhost"))); + Assert.True(proxy.IsBypassed(new Uri("https://127.0.0.1:8080"))); + } + finally + { + CleanProxyEnv(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void IgnoreWildcardInNoProxySubdomain() + { + try + { + Environment.SetEnvironmentVariable("http_proxy", "http://user1:pass1%40@127.0.0.1:8888"); + Environment.SetEnvironmentVariable("https_proxy", "http://user2:pass2%40@127.0.0.1:9999"); + Environment.SetEnvironmentVariable("no_proxy", "*.example.com"); + var proxy = new RunnerWebProxy(); + + Assert.False(proxy.IsBypassed(new Uri("http://sub.example.com"))); + Assert.False(proxy.IsBypassed(new Uri("http://example.com"))); + } + finally + { + CleanProxyEnv(); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void WildcardNoProxyWorksWhenOtherNoProxyAreAround() + { + try + { + Environment.SetEnvironmentVariable("http_proxy", "http://user1:pass1%40@127.0.0.1:8888"); + Environment.SetEnvironmentVariable("https_proxy", "http://user2:pass2%40@127.0.0.1:9999"); + Environment.SetEnvironmentVariable("no_proxy", "example.com,*,example2.com"); + var proxy = new RunnerWebProxy(); + + Assert.True(proxy.IsBypassed(new Uri("http://actions.com"))); + Assert.True(proxy.IsBypassed(new Uri("http://localhost"))); + Assert.True(proxy.IsBypassed(new Uri("http://127.0.0.1:8080"))); + Assert.True(proxy.IsBypassed(new Uri("https://actions.com"))); + Assert.True(proxy.IsBypassed(new Uri("https://localhost"))); + Assert.True(proxy.IsBypassed(new Uri("https://127.0.0.1:8080"))); + } + finally + { + CleanProxyEnv(); + } + } [Fact] [Trait("Level", "L0")] From cf3e1a56813e574ef06c55265090180d7fd403e6 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Tue, 7 Mar 2023 14:09:10 +0100 Subject: [PATCH 106/145] Exit on deprication error (#2299) * terminate the runner on deprication message * added TaskAgentVersion exception to catch deprication * AccessDenied exception with inner exception InvalidTaskAgent * Access denied exception in program and in message listener * Fixed copy * remove trace message from message listener --- src/Runner.Listener/MessageListener.cs | 8 ++++++-- src/Runner.Listener/Program.cs | 11 +++++++++-- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/src/Runner.Listener/MessageListener.cs b/src/Runner.Listener/MessageListener.cs index 59eddfbb12d..cb8edb607fe 100644 --- a/src/Runner.Listener/MessageListener.cs +++ b/src/Runner.Listener/MessageListener.cs @@ -182,7 +182,7 @@ public void OnJobStatus(object sender, JobStatusEventArgs e) try { _getMessagesTokenSource?.Cancel(); - } + } catch (ObjectDisposedException) { Trace.Info("_getMessagesTokenSource is already disposed."); @@ -245,6 +245,10 @@ public async Task GetNextMessageAsync(CancellationToken token) _accessTokenRevoked = true; throw; } + catch (AccessDeniedException e) when (e.InnerException is InvalidTaskAgentVersionException) + { + throw; + } catch (Exception ex) { Trace.Error("Catch exception during get next message."); @@ -289,7 +293,7 @@ public async Task GetNextMessageAsync(CancellationToken token) await HostContext.Delay(_getNextMessageRetryInterval, token); } } - finally + finally { _getMessagesTokenSource.Dispose(); } diff --git a/src/Runner.Listener/Program.cs b/src/Runner.Listener/Program.cs index d4d5e43eb34..b12eda66bd4 100644 --- a/src/Runner.Listener/Program.cs +++ b/src/Runner.Listener/Program.cs @@ -6,6 +6,7 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; namespace GitHub.Runner.Listener { @@ -58,7 +59,7 @@ private async static Task MainAsync(IHostContext context, string[] args) terminal.WriteLine("This runner version is built for Windows. Please install a correct build for your OS."); return Constants.Runner.ReturnCode.TerminatedError; } - #if ARM64 +#if ARM64 // A little hacky, but windows gives no way to differentiate between windows 10 and 11. // By default only 11 supports native x64 app emulation on arm, so we only want to support windows 11 // https://docs.microsoft.com/en-us/windows/arm/overview#build-windows-apps-that-run-on-arm @@ -69,7 +70,7 @@ private async static Task MainAsync(IHostContext context, string[] args) terminal.WriteLine("Win-arm64 runners require windows 11 or later. Please upgrade your operating system."); return Constants.Runner.ReturnCode.TerminatedError; } - #endif +#endif break; default: terminal.WriteLine($"Running the runner on this platform is not supported. The current platform is {RuntimeInformation.OSDescription} and it was built for {Constants.Runner.Platform.ToString()}."); @@ -137,6 +138,12 @@ private async static Task MainAsync(IHostContext context, string[] args) } } + catch (AccessDeniedException e) when (e.InnerException is InvalidTaskAgentVersionException) + { + terminal.WriteError($"An error occured: {e.Message}"); + trace.Error(e); + return Constants.Runner.ReturnCode.TerminatedError; + } catch (Exception e) { terminal.WriteError($"An error occurred: {e.Message}"); From ceebeef48caa860adda46f966b824fc1c131f440 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Wed, 8 Mar 2023 18:29:55 +0100 Subject: [PATCH 107/145] Add update certificates to ./run.sh if RUNNER_UPDATE_CA_CERTS env is set (#2471) * Included entrypoint that will update certs and run ./run.sh * update ca if RUNNER_UPDATE_CA env is set * changed env variable to RUNNER_UPDATE_TRUST_STORE * moved entrypoint to be run.sh, removed Dockerfile entrypoint, added envvar that will update certs * Update src/Misc/layoutroot/run.sh Co-authored-by: Bassem Dghaidi <568794+Link-@users.noreply.github.com> * Update src/Misc/layoutroot/run.sh Co-authored-by: Bassem Dghaidi <568794+Link-@users.noreply.github.com> * Update src/Misc/layoutroot/run.sh Co-authored-by: Bassem Dghaidi <568794+Link-@users.noreply.github.com> * Update src/Misc/layoutroot/run.sh Co-authored-by: Tingluo Huang * Update src/Misc/layoutroot/run.sh Co-authored-by: Tingluo Huang * removed doc comment on func --------- Co-authored-by: Bassem Dghaidi <568794+Link-@users.noreply.github.com> Co-authored-by: Tingluo Huang --- src/Misc/layoutroot/run.sh | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/src/Misc/layoutroot/run.sh b/src/Misc/layoutroot/run.sh index a558793d0df..6b02ea18ff1 100755 --- a/src/Misc/layoutroot/run.sh +++ b/src/Misc/layoutroot/run.sh @@ -53,6 +53,33 @@ runWithManualTrap() { done } +function updateCerts() { + local sudo_prefix="" + local user_id=`id -u` + + if [ $user_id -ne 0 ]; then + if [[ ! -x "$(command -v sudo)" ]]; then + echo "Warning: failed to update certificate store: sudo is required but not found" + return 1 + else + sudo_prefix="sudo" + fi + fi + + if [[ -x "$(command -v update-ca-certificates)" ]]; then + eval $sudo_prefix "update-ca-certificates" + elif [[ -x "$(command -v update-ca-trust)" ]]; then + eval $sudo_prefix "update-ca-trust" + else + echo "Warning: failed to update certificate store: update-ca-certificates or update-ca-trust not found. This can happen if you're using a different runner base image." + return 1 + fi +} + +if [[ ! -z "$RUNNER_UPDATE_CA_CERTS" ]]; then + updateCerts +fi + if [[ -z "$RUNNER_MANUALLY_TRAP_SIG" ]]; then run $* else From 2b94eaac7104c6de183ac4a52fa5157cdef7c275 Mon Sep 17 00:00:00 2001 From: Ajay <40024974+ajaykn@users.noreply.github.com> Date: Fri, 10 Mar 2023 11:30:28 +0900 Subject: [PATCH 108/145] Support matrix context in output keys (#2477) --- src/Sdk/DTPipelines/workflow-v1.0.json | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/Sdk/DTPipelines/workflow-v1.0.json b/src/Sdk/DTPipelines/workflow-v1.0.json index a3745029b3d..c1453f4e6ac 100644 --- a/src/Sdk/DTPipelines/workflow-v1.0.json +++ b/src/Sdk/DTPipelines/workflow-v1.0.json @@ -222,6 +222,9 @@ }, "job-outputs": { + "context": [ + "matrix" + ], "mapping": { "loose-key-type": "non-empty-string", "loose-value-type": "string-runner-context" From ecdd895b9aa23e656b00c488abd5ee62b8114a28 Mon Sep 17 00:00:00 2001 From: Jongwoo Han Date: Fri, 10 Mar 2023 12:09:05 +0900 Subject: [PATCH 109/145] Replace deprecated command with environment file (#2429) Signed-off-by: jongwooo --- .github/workflows/release.yml | 14 +++++++------- src/Test/TestData/conditional_composite_action.yml | 10 +++++----- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ad9f0d77fa9..4bf120014b5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -131,7 +131,7 @@ jobs: file=$(ls) sha=$(sha256sum $file | awk '{ print $1 }') echo "Computed sha256: $sha for $file" - echo "::set-output name=${{matrix.runtime}}-sha256::$sha" + echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT shell: bash id: sha name: Compute SHA256 @@ -140,8 +140,8 @@ jobs: file=$(ls) sha=$(sha256sum $file | awk '{ print $1 }') echo "Computed sha256: $sha for $file" - echo "::set-output name=${{matrix.runtime}}-sha256::$sha" - echo "::set-output name=sha256::$sha" + echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT + echo "sha256=$sha" >> $GITHUB_OUTPUT shell: bash id: sha_noexternals name: Compute SHA256 @@ -150,8 +150,8 @@ jobs: file=$(ls) sha=$(sha256sum $file | awk '{ print $1 }') echo "Computed sha256: $sha for $file" - echo "::set-output name=${{matrix.runtime}}-sha256::$sha" - echo "::set-output name=sha256::$sha" + echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT + echo "sha256=$sha" >> $GITHUB_OUTPUT shell: bash id: sha_noruntime name: Compute SHA256 @@ -160,8 +160,8 @@ jobs: file=$(ls) sha=$(sha256sum $file | awk '{ print $1 }') echo "Computed sha256: $sha for $file" - echo "::set-output name=${{matrix.runtime}}-sha256::$sha" - echo "::set-output name=sha256::$sha" + echo "${{matrix.runtime}}-sha256=$sha" >> $GITHUB_OUTPUT + echo "sha256=$sha" >> $GITHUB_OUTPUT shell: bash id: sha_noruntime_noexternals name: Compute SHA256 diff --git a/src/Test/TestData/conditional_composite_action.yml b/src/Test/TestData/conditional_composite_action.yml index a567c959329..d0d9db1ebf6 100644 --- a/src/Test/TestData/conditional_composite_action.yml +++ b/src/Test/TestData/conditional_composite_action.yml @@ -25,25 +25,25 @@ runs: - run: exit ${{ inputs.exit-code }} shell: bash - - run: echo "::set-output name=default::true" + - run: echo "default=true" >> $GITHUB_OUTPUT id: default-conditional shell: bash - - run: echo "::set-output name=success::true" + - run: echo "success=true" >> $GITHUB_OUTPUT id: success-conditional shell: bash if: success() - - run: echo "::set-output name=failure::true" + - run: echo "failure=true" >> $GITHUB_OUTPUT id: failure-conditional shell: bash if: failure() - - run: echo "::set-output name=always::true" + - run: echo "always=true" >> $GITHUB_OUTPUT id: always-conditional shell: bash if: always() - run: echo "failed" shell: bash - if: ${{ inputs.exit-code == 1 && failure() }} \ No newline at end of file + if: ${{ inputs.exit-code == 1 && failure() }} From 81b6ad1886a61ffa5c9eef1117d0d155b5720646 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Fri, 10 Mar 2023 05:45:56 -0500 Subject: [PATCH 110/145] Prepare runner release 2.303.0 (#2482) --- releaseNote.md | 16 +++++++++------- src/runnerversion | 2 +- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 2e7367c7a70..7ed7306633e 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,15 +1,17 @@ ## Features -- Add support for ghe.com domain (#2420) -- Add docker cli to the runner image. (#2425) +- Support matrix context in output keys (#2477) +- Add update certificates to `./run.sh` if `RUNNER_UPDATE_CA_CERTS` env is set (#2471) +- Bypass all proxies for all hosts if `no_proxy='*'` is set (#2395) +- Change runner image to make user/folder align with `ubuntu-latest` hosted runner. (#2469) ## Bugs -- Fix URL construction bug for RunService (#2396) -- Defer evaluation of a step's DisplayName until its condition is evaluated. (#2313) -- Replace '(' and ')' with '[' and '] from OS.Description for fixing User-Agent header validation (#2288) +- Exit on runner version deprecation error (#2299) +- Runner service exit after consecutive re-try exits (#2426) ## Misc -- Bump dotnet sdk to latest version. (#2392) -- Start calling run service for job completion (#2412, #2423) +- Replace deprecated command with environment file (#2429) +- Make requests to `Run` service to renew job request (#2461) +- Add job/step log upload to Result service (#2447, #2439) _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index 22155b3fbc4..f67bb997a56 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.302.1 +2.303.0 From 3d1be04b9932fab50c0225d163cc75132f99a6ea Mon Sep 17 00:00:00 2001 From: Bassem Dghaidi <568794+Link-@users.noreply.github.com> Date: Fri, 17 Mar 2023 11:00:55 +0100 Subject: [PATCH 111/145] Update container hooks version to 0.3.0 (#2492) --- images/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/images/Dockerfile b/images/Dockerfile index 315196e49c2..4699ee82b6f 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -2,7 +2,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build ARG RUNNER_VERSION ARG RUNNER_ARCH="x64" -ARG RUNNER_CONTAINER_HOOKS_VERSION=0.2.0 +ARG RUNNER_CONTAINER_HOOKS_VERSION=0.3.0 ARG DOCKER_VERSION=20.10.23 RUN apt update -y && apt install curl unzip -y @@ -46,4 +46,4 @@ COPY --chown=runner:docker --from=build /actions-runner . RUN install -o root -g root -m 755 docker/* /usr/bin/ && rm -rf docker -USER runner \ No newline at end of file +USER runner From fddc1dfff804a4ead2a59ac916a5ed8a5d4b774e Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 20 Mar 2023 10:53:47 -0400 Subject: [PATCH 112/145] Bump container hooks version to 0.3.1 (#2496) --- images/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/images/Dockerfile b/images/Dockerfile index 4699ee82b6f..d30263e3c27 100644 --- a/images/Dockerfile +++ b/images/Dockerfile @@ -2,7 +2,7 @@ FROM mcr.microsoft.com/dotnet/runtime-deps:6.0 as build ARG RUNNER_VERSION ARG RUNNER_ARCH="x64" -ARG RUNNER_CONTAINER_HOOKS_VERSION=0.3.0 +ARG RUNNER_CONTAINER_HOOKS_VERSION=0.3.1 ARG DOCKER_VERSION=20.10.23 RUN apt update -y && apt install curl unzip -y From 4b27b4991cbc8ea454e8c8dfa50d96a24db6a60d Mon Sep 17 00:00:00 2001 From: Tatyana Kostromskaya <32135588+takost@users.noreply.github.com> Date: Tue, 21 Mar 2023 16:05:32 +0100 Subject: [PATCH 113/145] Update runner to handle Dotcom/runner-admin based registration flow (#2487) --- src/Runner.Common/ConfigurationStore.cs | 3 + src/Runner.Common/RunnerDotcomServer.cs | 237 ++++++++++++++++++ .../Configuration/ConfigurationManager.cs | 52 ++-- .../Configuration/CredentialManager.cs | 9 +- .../DTWebApi/WebApi/ListRunnersResponse.cs | 50 ++++ src/Sdk/DTWebApi/WebApi/Runner.cs | 18 ++ src/Sdk/DTWebApi/WebApi/RunnerGroup.cs | 98 ++++++++ .../Configuration/ConfigurationManagerL0.cs | 7 + 8 files changed, 455 insertions(+), 19 deletions(-) create mode 100644 src/Runner.Common/RunnerDotcomServer.cs create mode 100644 src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs create mode 100644 src/Sdk/DTWebApi/WebApi/Runner.cs create mode 100644 src/Sdk/DTWebApi/WebApi/RunnerGroup.cs diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index 7daa4d66fdd..49c4229daf4 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -50,6 +50,9 @@ public sealed class RunnerSettings [DataMember(EmitDefaultValue = false)] public string MonitorSocketAddress { get; set; } + [DataMember(EmitDefaultValue = false)] + public bool UseV2Flow { get; set; } + [IgnoreDataMember] public bool IsHostedServer { diff --git a/src/Runner.Common/RunnerDotcomServer.cs b/src/Runner.Common/RunnerDotcomServer.cs new file mode 100644 index 00000000000..510877e2e17 --- /dev/null +++ b/src/Runner.Common/RunnerDotcomServer.cs @@ -0,0 +1,237 @@ +using GitHub.DistributedTask.WebApi; +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Services.WebApi; +using GitHub.Services.Common; +using GitHub.Runner.Sdk; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Linq; + +namespace GitHub.Runner.Common +{ + [ServiceLocator(Default = typeof(RunnerDotcomServer))] + public interface IRunnerDotcomServer : IRunnerService + { + Task> GetRunnersAsync(int runnerGroupId, string githubUrl, string githubToken, string agentName); + + Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, string githubUrl, string githubToken); + Task> GetRunnerGroupsAsync(string githubUrl, string githubToken); + + string GetGitHubRequestId(HttpResponseHeaders headers); + } + + public enum RequestType + { + Get, + Post, + Patch, + Delete + } + + public class RunnerDotcomServer : RunnerService, IRunnerDotcomServer + { + private ITerminal _term; + + public override void Initialize(IHostContext hostContext) + { + base.Initialize(hostContext); + _term = hostContext.GetService(); + } + + + public async Task> GetRunnersAsync(int runnerGroupId, string githubUrl, string githubToken, string agentName = null) + { + var githubApiUrl = ""; + var gitHubUrlBuilder = new UriBuilder(githubUrl); + var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries); + if (path.Length == 1) + { + // org runner + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/orgs/{path[0]}/actions/runner-groups/{runnerGroupId}/runners"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runner-groups/{runnerGroupId}/runners"; + } + } + else if (path.Length == 2) + { + // repo or enterprise runner. + if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups/{runnerGroupId}/runners"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups/{runnerGroupId}/runners"; + } + } + else + { + throw new ArgumentException($"'{githubUrl}' should point to an org or enterprise."); + } + + var runnersList = await RetryRequest(githubApiUrl, githubToken, RequestType.Get, 3, "Failed to get agents pools"); + var agents = runnersList.ToTaskAgents(); + + if (string.IsNullOrEmpty(agentName)) + { + return agents; + } + + return agents.Where(x => string.Equals(x.Name, agentName, StringComparison.OrdinalIgnoreCase)).ToList(); + } + + public async Task> GetRunnerGroupsAsync(string githubUrl, string githubToken) + { + var githubApiUrl = ""; + var gitHubUrlBuilder = new UriBuilder(githubUrl); + var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries); + if (path.Length == 1) + { + // org runner + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/orgs/{path[0]}/actions/runner-groups"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/orgs/{path[0]}/actions/runner-groups"; + } + } + else if (path.Length == 2) + { + // repo or enterprise runner. + if (!string.Equals(path[0], "enterprises", StringComparison.OrdinalIgnoreCase)) + { + return null; + } + + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/{path[0]}/{path[1]}/actions/runner-groups"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/{path[0]}/{path[1]}/actions/runner-groups"; + } + } + else + { + throw new ArgumentException($"'{githubUrl}' should point to an org or enterprise."); + } + + var agentPools = await RetryRequest(githubApiUrl, githubToken, RequestType.Get, 3, "Failed to get agents pools"); + + return agentPools?.ToAgentPoolList(); + } + + public async Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, string githubUrl, string githubToken) + { + var gitHubUrlBuilder = new UriBuilder(githubUrl); + var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries); + string githubApiUrl; + if (UrlUtil.IsHostedServer(gitHubUrlBuilder)) + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://api.{gitHubUrlBuilder.Host}/actions/runners/register"; + } + else + { + githubApiUrl = $"{gitHubUrlBuilder.Scheme}://{gitHubUrlBuilder.Host}/api/v3/actions/runners/register"; + } + + var bodyObject = new Dictionary() + { + {"url", githubUrl}, + {"group_id", runnerGroupId}, + {"name", agent.Name}, + {"version", agent.Version}, + {"updates_disabled", agent.DisableUpdate}, + {"ephemeral", agent.Ephemeral}, + {"labels", agent.Labels}, + }; + + var body = new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"); + var responseAgent = await RetryRequest(githubApiUrl, githubToken, RequestType.Post, 3, "Failed to add agent", body); + agent.Id = responseAgent.Id; + return agent; + } + + private async Task RetryRequest(string githubApiUrl, string githubToken, RequestType requestType, int maxRetryAttemptsCount = 5, string errorMessage = null, StringContent body = null) + { + int retry = 0; + while (true) + { + retry++; + using (var httpClientHandler = HostContext.CreateHttpClientHandler()) + using (var httpClient = new HttpClient(httpClientHandler)) + { + httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("RemoteAuth", githubToken); + httpClient.DefaultRequestHeaders.UserAgent.AddRange(HostContext.UserAgents); + + var responseStatus = System.Net.HttpStatusCode.OK; + try + { + HttpResponseMessage response = null; + if (requestType == RequestType.Get) + { + response = await httpClient.GetAsync(githubApiUrl); + } + else + { + response = await httpClient.PostAsync(githubApiUrl, body); + } + + if (response != null) + { + responseStatus = response.StatusCode; + var githubRequestId = GetGitHubRequestId(response.Headers); + + if (response.IsSuccessStatusCode) + { + Trace.Info($"Http response code: {response.StatusCode} from '{requestType.ToString()} {githubApiUrl}' ({githubRequestId})"); + var jsonResponse = await response.Content.ReadAsStringAsync(); + return StringUtil.ConvertFromJson(jsonResponse); + } + else + { + _term.WriteError($"Http response code: {response.StatusCode} from '{requestType.ToString()} {githubApiUrl}' (Request Id: {githubRequestId})"); + var errorResponse = await response.Content.ReadAsStringAsync(); + _term.WriteError(errorResponse); + response.EnsureSuccessStatusCode(); + } + } + + } + catch (Exception ex) when (retry < maxRetryAttemptsCount && responseStatus != System.Net.HttpStatusCode.NotFound) + { + Trace.Error($"{errorMessage} -- Atempt: {retry}"); + Trace.Error(ex); + } + } + var backOff = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(1), TimeSpan.FromSeconds(5)); + Trace.Info($"Retrying in {backOff.Seconds} seconds"); + await Task.Delay(backOff); + } + } + + public string GetGitHubRequestId(HttpResponseHeaders headers) + { + if (headers.TryGetValues("x-github-request-id", out var headerValues)) + { + return headerValues.FirstOrDefault(); + } + return string.Empty; + } + } +} diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 392eb0e3c3b..492bea04213 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -31,12 +31,14 @@ public sealed class ConfigurationManager : RunnerService, IConfigurationManager { private IConfigurationStore _store; private IRunnerServer _runnerServer; + private IRunnerDotcomServer _dotcomServer; private ITerminal _term; public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _runnerServer = HostContext.GetService(); + _dotcomServer = HostContext.GetService(); Trace.Verbose("Creating _store"); _store = hostContext.GetService(); Trace.Verbose("store created"); @@ -113,6 +115,7 @@ public async Task ConfigureAsync(CommandSettings command) ICredentialProvider credProvider = null; VssCredentials creds = null; _term.WriteSection("Authentication"); + string registerToken = string.Empty; while (true) { // When testing against a dev deployment of Actions Service, set this environment variable @@ -130,9 +133,11 @@ public async Task ConfigureAsync(CommandSettings command) else { runnerSettings.GitHubUrl = inputUrl; - var registerToken = await GetRunnerTokenAsync(command, inputUrl, "registration"); + registerToken = await GetRunnerTokenAsync(command, inputUrl, "registration"); GitHubAuthResult authResult = await GetTenantCredential(inputUrl, registerToken, Constants.RunnerEvent.Register); runnerSettings.ServerUrl = authResult.TenantUrl; + runnerSettings.UseV2Flow = authResult.UseV2Flow; + _term.WriteLine($"Using V2 flow: {runnerSettings.UseV2Flow}"); creds = authResult.ToVssCredentials(); Trace.Info("cred retrieved via GitHub auth"); } @@ -186,9 +191,17 @@ public async Task ConfigureAsync(CommandSettings command) // If we have more than one runner group available, allow the user to specify which one to be added into string poolName = null; TaskAgentPool agentPool = null; - List agentPools = await _runnerServer.GetAgentPoolsAsync(); - TaskAgentPool defaultPool = agentPools?.Where(x => x.IsInternal).FirstOrDefault(); + List agentPools; + if (runnerSettings.UseV2Flow) + { + agentPools = await _dotcomServer.GetRunnerGroupsAsync(runnerSettings.GitHubUrl, registerToken); + } + else + { + agentPools = await _runnerServer.GetAgentPoolsAsync(); + } + TaskAgentPool defaultPool = agentPools?.Where(x => x.IsInternal).FirstOrDefault(); if (agentPools?.Where(x => !x.IsHosted).Count() > 0) { poolName = command.GetRunnerGroupName(defaultPool?.Name); @@ -226,8 +239,16 @@ public async Task ConfigureAsync(CommandSettings command) var userLabels = command.GetLabels(); _term.WriteLine(); + List agents; + if (runnerSettings.UseV2Flow) + { + agents = await _dotcomServer.GetRunnersAsync(runnerSettings.PoolId, runnerSettings.GitHubUrl, registerToken, runnerSettings.AgentName); + } + else + { + agents = await _runnerServer.GetAgentsAsync(runnerSettings.PoolId, runnerSettings.AgentName); + } - var agents = await _runnerServer.GetAgentsAsync(runnerSettings.PoolId, runnerSettings.AgentName); Trace.Verbose("Returns {0} agents", agents.Count); agent = agents.FirstOrDefault(); if (agent != null) @@ -274,7 +295,15 @@ public async Task ConfigureAsync(CommandSettings command) try { - agent = await _runnerServer.AddAgentAsync(runnerSettings.PoolId, agent); + if (runnerSettings.UseV2Flow) + { + agent = await _dotcomServer.AddRunnerAsync(runnerSettings.PoolId, agent, runnerSettings.GitHubUrl, registerToken); + } + else + { + agent = await _runnerServer.AddAgentAsync(runnerSettings.PoolId, agent); + } + if (command.DisableUpdate && command.DisableUpdate != agent.DisableUpdate) { @@ -652,7 +681,7 @@ private async Task GetJITRunnerTokenAsync(string gith { var response = await httpClient.PostAsync(githubApiUrl, new StringContent(string.Empty)); responseStatus = response.StatusCode; - var githubRequestId = GetGitHubRequestId(response.Headers); + var githubRequestId = _dotcomServer.GetGitHubRequestId(response.Headers); if (response.IsSuccessStatusCode) { @@ -715,7 +744,7 @@ private async Task GetTenantCredential(string githubUrl, strin { var response = await httpClient.PostAsync(githubApiUrl, new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json")); responseStatus = response.StatusCode; - var githubRequestId = GetGitHubRequestId(response.Headers); + var githubRequestId = _dotcomServer.GetGitHubRequestId(response.Headers); if (response.IsSuccessStatusCode) { @@ -744,14 +773,5 @@ private async Task GetTenantCredential(string githubUrl, strin } return null; } - - private string GetGitHubRequestId(HttpResponseHeaders headers) - { - if (headers.TryGetValues("x-github-request-id", out var headerValues)) - { - return headerValues.FirstOrDefault(); - } - return string.Empty; - } } } diff --git a/src/Runner.Listener/Configuration/CredentialManager.cs b/src/Runner.Listener/Configuration/CredentialManager.cs index 327961794e5..80cff904cf5 100644 --- a/src/Runner.Listener/Configuration/CredentialManager.cs +++ b/src/Runner.Listener/Configuration/CredentialManager.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Runtime.Serialization; using GitHub.Runner.Common; @@ -20,8 +20,8 @@ public class CredentialManager : RunnerService, ICredentialManager { public static readonly Dictionary CredentialTypes = new(StringComparer.OrdinalIgnoreCase) { - { Constants.Configuration.OAuth, typeof(OAuthCredential)}, - { Constants.Configuration.OAuthAccessToken, typeof(OAuthAccessTokenCredential)}, + { Constants.Configuration.OAuth, typeof(OAuthCredential) }, + { Constants.Configuration.OAuthAccessToken, typeof(OAuthAccessTokenCredential) }, }; public ICredentialProvider GetCredentialProvider(string credType) @@ -93,6 +93,9 @@ public sealed class GitHubAuthResult [DataMember(Name = "token")] public string Token { get; set; } + [DataMember(Name = "use_v2_flow")] + public bool UseV2Flow { get; set; } + public VssCredentials ToVssCredentials() { ArgUtil.NotNullOrEmpty(TokenSchema, nameof(TokenSchema)); diff --git a/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs b/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs new file mode 100644 index 00000000000..292b8d66801 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs @@ -0,0 +1,50 @@ +using GitHub.Services.WebApi; +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; +using Newtonsoft.Json; +using System.Linq; + +namespace GitHub.DistributedTask.WebApi +{ + + public class ListRunnersResponse + { + public ListRunnersResponse() + { + } + + public ListRunnersResponse(ListRunnersResponse responseToBeCloned) + { + this.TotalCount = responseToBeCloned.TotalCount; + this.Runners = responseToBeCloned.Runners; + } + + [JsonProperty("total_count")] + public int TotalCount + { + get; + set; + } + + [JsonProperty("runners")] + public List Runners + { + get; + set; + } + + public ListRunnersResponse Clone() + { + return new ListRunnersResponse(this); + } + + public List ToTaskAgents() + { + List taskAgents = new List(); + + return Runners.Select(runner => new TaskAgent() { Name = runner.Name }).ToList(); + } + } + +} diff --git a/src/Sdk/DTWebApi/WebApi/Runner.cs b/src/Sdk/DTWebApi/WebApi/Runner.cs new file mode 100644 index 00000000000..2103bf3a210 --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/Runner.cs @@ -0,0 +1,18 @@ +using Newtonsoft.Json; + +namespace GitHub.DistributedTask.WebApi +{ + public class Runner + { + /// + /// Name of the agent + /// + [JsonProperty("name")] + public string Name + { + get; + internal set; + } + + } +} diff --git a/src/Sdk/DTWebApi/WebApi/RunnerGroup.cs b/src/Sdk/DTWebApi/WebApi/RunnerGroup.cs new file mode 100644 index 00000000000..e1171bb663b --- /dev/null +++ b/src/Sdk/DTWebApi/WebApi/RunnerGroup.cs @@ -0,0 +1,98 @@ +using GitHub.Services.WebApi; +using System; +using System.Runtime.Serialization; +using System.ComponentModel; +using System.Collections.Generic; +using Newtonsoft.Json; +using System.Linq; + +namespace GitHub.DistributedTask.WebApi +{ + /// + /// An organization-level grouping of runners. + /// + [DataContract] + public class RunnerGroup + { + internal RunnerGroup() + { + } + + public RunnerGroup(String name) + { + this.Name = name; + } + + private RunnerGroup(RunnerGroup poolToBeCloned) + { + this.Id = poolToBeCloned.Id; + this.IsHosted = poolToBeCloned.IsHosted; + this.Name = poolToBeCloned.Name; + this.IsDefault = poolToBeCloned.IsDefault; + } + + [DataMember(EmitDefaultValue = false)] + [JsonProperty("id")] + public Int32 Id + { + get; + set; + } + + [DataMember(EmitDefaultValue = false)] + [JsonProperty("name")] + public String Name + { + get; + set; + } + + /// + /// Gets or sets a value indicating whether or not this pool is internal and can't be modified by users + /// + [DataMember] + [JsonProperty("default")] + public bool IsDefault + { + get; + set; + } + + /// + /// Gets or sets a value indicating whether or not this pool is managed by the service. + /// + [DataMember] + [JsonProperty("is_hosted")] + public Boolean IsHosted + { + get; + set; + } + } + + public class RunnerGroupList + { + public RunnerGroupList() + { + this.RunnerGroups = new List(); + } + + public List ToAgentPoolList() + { + var agentPools = this.RunnerGroups.Select(x => new TaskAgentPool(x.Name) + { + Id = x.Id, + IsHosted = x.IsHosted, + IsInternal = x.IsDefault + }).ToList(); + + return agentPools; + } + + [JsonProperty("runner_groups")] + public List RunnerGroups { get; set; } + + [JsonProperty("total_count")] + public int Count { get; set; } + } +} diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index ca6e90b6bf7..19bf0250c79 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -19,6 +19,7 @@ namespace GitHub.Runner.Common.Tests.Listener.Configuration public class ConfigurationManagerL0 { private Mock _runnerServer; + private Mock _dotcomServer; private Mock _locationServer; private Mock _credMgr; private Mock _promptManager; @@ -55,6 +56,7 @@ public ConfigurationManagerL0() _store = new Mock(); _extnMgr = new Mock(); _rsaKeyManager = new Mock(); + _dotcomServer = new Mock(); #if OS_WINDOWS _serviceControlManager = new Mock(); @@ -106,6 +108,10 @@ public ConfigurationManagerL0() _runnerServer.Setup(x => x.AddAgentAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedAgent)); _runnerServer.Setup(x => x.ReplaceAgentAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedAgent)); + _dotcomServer.Setup(x => x.GetRunnersAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedAgents)); + _dotcomServer.Setup(x => x.GetRunnerGroupsAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedPools)); + _dotcomServer.Setup(x => x.AddRunnerAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedAgent)); + rsa = new RSACryptoServiceProvider(2048); _rsaKeyManager.Setup(x => x.CreateKey()).Returns(rsa); @@ -119,6 +125,7 @@ private TestHostContext CreateTestContext([CallerMemberName] String testName = " tc.SetSingleton(_store.Object); tc.SetSingleton(_extnMgr.Object); tc.SetSingleton(_runnerServer.Object); + tc.SetSingleton(_dotcomServer.Object); tc.SetSingleton(_locationServer.Object); #if OS_WINDOWS From 5c6679706f96d07c8453934b6b8fa43b5edd4871 Mon Sep 17 00:00:00 2001 From: Matisse Hack Date: Tue, 21 Mar 2023 12:04:50 -0700 Subject: [PATCH 114/145] Fix JIT configurations on Windows (#2497) * Fix JIT configurations on Windows * Update src/Runner.Listener/Runner.cs --- src/Runner.Listener/Runner.cs | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 30d60b2fc65..c727f1b3865 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -4,6 +4,7 @@ using System.Linq; using System.Reflection; using System.Runtime.CompilerServices; +using System.Security.Cryptography; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -210,10 +211,16 @@ public async Task ExecuteCommand(CommandSettings command) foreach (var config in jitConfig) { var configFile = Path.Combine(HostContext.GetDirectory(WellKnownDirectory.Root), config.Key); - var configContent = Encoding.UTF8.GetString(Convert.FromBase64String(config.Value)); - File.WriteAllText(configFile, configContent, Encoding.UTF8); + var configContent = Convert.FromBase64String(config.Value); +#if OS_WINDOWS + if (configFile == HostContext.GetConfigFile(WellKnownConfigFile.RSACredentials)) + { + configContent = ProtectedData.Protect(configContent, null, DataProtectionScope.LocalMachine); + } +#endif + File.WriteAllBytes(configFile, configContent); File.SetAttributes(configFile, File.GetAttributes(configFile) | FileAttributes.Hidden); - Trace.Info($"Save {configContent.Length} chars to '{configFile}'."); + Trace.Info($"Saved {configContent.Length} bytes to '{configFile}'."); } } catch (Exception ex) From f905dde4fc80a50399ff540d6d3786dcd396d925 Mon Sep 17 00:00:00 2001 From: Bassem Dghaidi <568794+Link-@users.noreply.github.com> Date: Thu, 23 Mar 2023 17:09:10 +0100 Subject: [PATCH 115/145] ADR: Runner Image Tags (#2494) * WIP * WIP * Add context * Add 2494-runner-image-tags ADR draft * Fix ADR title * Add more information to option 2 * Add decision * Fix status --- docs/adrs/2494-runner-image-tags.md | 65 +++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 docs/adrs/2494-runner-image-tags.md diff --git a/docs/adrs/2494-runner-image-tags.md b/docs/adrs/2494-runner-image-tags.md new file mode 100644 index 00000000000..e5bde453a0d --- /dev/null +++ b/docs/adrs/2494-runner-image-tags.md @@ -0,0 +1,65 @@ +# ADR 2494: Runner Image Tags + +**Date**: 2023-03-17 + +**Status**: Accepted + +## Context + +Following the [adoption of actions-runner-controller by GitHub](https://github.com/actions/actions-runner-controller/discussions/2072) and the introduction of the new runner scale set autoscaling mode, we needed to provide a basic runner image that could be used off the shelf without much friction. + +The [current runner image](https://github.com/actions/runner/pkgs/container/actions-runner) is published to GHCR. Each release of this image is tagged with the runner version and the most recent release is also tagged with `latest`. + +While the use of `latest` is common practice, we recommend that users pin a specific version of the runner image for a predictable runtime and improved security posture. However, we still notice that a large number of end users are relying on the `latest` tag & raising issues when they encounter problems. + +Add to that, the community actions-runner-controller maintainers have issued a [deprecation notice](https://github.com/actions/actions-runner-controller/issues/2056) of the `latest` tag for the existing runner images (https://github.com/orgs/actions-runner-controller/packages). + +## Decision + +Proceed with Option 2, keeping the `latest` tag and adding the `NOTES.txt` file to our helm charts with the notice. + +### Option 1: Remove the `latest` tag + +By removing the `latest` tag, we have to proceed with either of these options: + +1. Remove the runner image reference in the `values.yaml` provided with the `gha-runner-scale-set` helm chart and mark these fields as required so that users have to explicitly specify a runner image and a specific tag. This will obviously introduce more friction for users who want to start using actions-runner-controller for the first time. + +```yaml + spec: + containers: + - name: runner + image: "" + tag: "" + command: ["/home/runner/run.sh"] +``` + +1. Pin a specific runner image tag in the `values.yaml` provided with the `gha-runner-scale-set` helm chart. This will reduce friction for users who want to start using actions-runner-controller for the first time but will require us to update the `values.yaml` with every new runner release. + +```yaml + spec: + containers: + - name: runner + image: "ghcr.io/actions/actions-runner" + tag: "v2.300.0" + command: ["/home/runner/run.sh"] +``` + +### Option 2: Keep the `latest` tag + +Keeping the `latest` tag is also a reasonable option especially if we don't expect to make any breaking changes to the runner image. We could enhance this by adding a [NOTES.txt](https://helm.sh/docs/chart_template_guide/notes_files/) to the helm chart which will be displayed to the user after a successful helm install/upgrade. This will help users understand the implications of using the `latest` tag and how to pin a specific version of the runner image. + +The runner image release workflow will need to be updated so that the image is pushed to GHCR and tagged only when the runner rollout has reached all scale units. + +## Consequences + +Proceeding with **option 1** means: + +1. We will enhance the runtime predictability and security posture of our end users +1. We will have to update the `values.yaml` with every new runner release (that can be automated) +1. We will introduce friction for users who want to start using actions-runner-controller for the first time + +Proceeding with **option 2** means: + +1. We will have to continue to maintain the `latest` tag +1. We will assume that end users will be able to handle the implications of using the `latest` tag +1. Runner image release workflow needs to be updated From 5bb25e8f9380f54d8d4203cba69b14b28f1905d5 Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Fri, 24 Mar 2023 14:00:34 -0400 Subject: [PATCH 116/145] Listen directly to Broker for messages (#2500) --- src/Runner.Common/BrokerServer.cs | 56 +++++ src/Runner.Common/ConfigurationStore.cs | 3 + src/Runner.Listener/BrokerMessageListener.cs | 204 +++++++++++++++++++ src/Runner.Listener/Runner.cs | 14 +- src/Sdk/WebApi/WebApi/BrokerHttpClient.cs | 84 ++++++++ 5 files changed, 360 insertions(+), 1 deletion(-) create mode 100644 src/Runner.Common/BrokerServer.cs create mode 100644 src/Runner.Listener/BrokerMessageListener.cs create mode 100644 src/Sdk/WebApi/WebApi/BrokerHttpClient.cs diff --git a/src/Runner.Common/BrokerServer.cs b/src/Runner.Common/BrokerServer.cs new file mode 100644 index 00000000000..bb0691771a0 --- /dev/null +++ b/src/Runner.Common/BrokerServer.cs @@ -0,0 +1,56 @@ +using System; +using System.Collections.Generic; +using System.Threading; +using System.Threading.Tasks; +using GitHub.Actions.RunService.WebApi; +using GitHub.DistributedTask.Pipelines; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; +using Sdk.RSWebApi.Contracts; +using Sdk.WebApi.WebApi.RawClient; + +namespace GitHub.Runner.Common +{ + [ServiceLocator(Default = typeof(BrokerServer))] + public interface IBrokerServer : IRunnerService + { + Task ConnectAsync(Uri serverUrl, VssCredentials credentials); + + Task GetRunnerMessageAsync(CancellationToken token, TaskAgentStatus status, string version); + } + + public sealed class BrokerServer : RunnerService, IBrokerServer + { + private bool _hasConnection; + private Uri _brokerUri; + private RawConnection _connection; + private BrokerHttpClient _brokerHttpClient; + + public async Task ConnectAsync(Uri serverUri, VssCredentials credentials) + { + _brokerUri = serverUri; + + _connection = VssUtil.CreateRawConnection(serverUri, credentials); + _brokerHttpClient = await _connection.GetClientAsync(); + _hasConnection = true; + } + + private void CheckConnection() + { + if (!_hasConnection) + { + throw new InvalidOperationException($"SetConnection"); + } + } + + public Task GetRunnerMessageAsync(CancellationToken cancellationToken, TaskAgentStatus status, string version) + { + CheckConnection(); + var jobMessage = RetryRequest( + async () => await _brokerHttpClient.GetRunnerMessageAsync(version, status, cancellationToken), cancellationToken); + + return jobMessage; + } + } +} diff --git a/src/Runner.Common/ConfigurationStore.cs b/src/Runner.Common/ConfigurationStore.cs index 49c4229daf4..3dd5a803208 100644 --- a/src/Runner.Common/ConfigurationStore.cs +++ b/src/Runner.Common/ConfigurationStore.cs @@ -53,6 +53,9 @@ public sealed class RunnerSettings [DataMember(EmitDefaultValue = false)] public bool UseV2Flow { get; set; } + [DataMember(EmitDefaultValue = false)] + public string ServerUrlV2 { get; set; } + [IgnoreDataMember] public bool IsHostedServer { diff --git a/src/Runner.Listener/BrokerMessageListener.cs b/src/Runner.Listener/BrokerMessageListener.cs new file mode 100644 index 00000000000..93259c8e56f --- /dev/null +++ b/src/Runner.Listener/BrokerMessageListener.cs @@ -0,0 +1,204 @@ +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Runtime.InteropServices; +using System.Security.Cryptography; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Common; +using GitHub.Runner.Listener.Configuration; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; +using GitHub.Runner.Common.Util; +using GitHub.Services.OAuth; + +namespace GitHub.Runner.Listener +{ + public sealed class BrokerMessageListener : RunnerService, IMessageListener + { + private RunnerSettings _settings; + private ITerminal _term; + private TimeSpan _getNextMessageRetryInterval; + private TaskAgentStatus runnerStatus = TaskAgentStatus.Online; + private CancellationTokenSource _getMessagesTokenSource; + private IBrokerServer _brokerServer; + + public override void Initialize(IHostContext hostContext) + { + base.Initialize(hostContext); + + _term = HostContext.GetService(); + _brokerServer = HostContext.GetService(); + } + + public async Task CreateSessionAsync(CancellationToken token) + { + await RefreshBrokerConnection(); + return await Task.FromResult(true); + } + + public async Task DeleteSessionAsync() + { + await Task.CompletedTask; + } + + public void OnJobStatus(object sender, JobStatusEventArgs e) + { + Trace.Info("Received job status event. JobState: {0}", e.Status); + runnerStatus = e.Status; + try + { + _getMessagesTokenSource?.Cancel(); + } + catch (ObjectDisposedException) + { + Trace.Info("_getMessagesTokenSource is already disposed."); + } + } + + public async Task GetNextMessageAsync(CancellationToken token) + { + bool encounteringError = false; + int continuousError = 0; + Stopwatch heartbeat = new(); + heartbeat.Restart(); + var maxRetryCount = 10; + + while (true) + { + TaskAgentMessage message = null; + _getMessagesTokenSource = CancellationTokenSource.CreateLinkedTokenSource(token); + try + { + message = await _brokerServer.GetRunnerMessageAsync(_getMessagesTokenSource.Token, runnerStatus, BuildConstants.RunnerPackage.Version); + + if (message == null) + { + continue; + } + + return message; + } + catch (OperationCanceledException) when (_getMessagesTokenSource.Token.IsCancellationRequested && !token.IsCancellationRequested) + { + Trace.Info("Get messages has been cancelled using local token source. Continue to get messages with new status."); + continue; + } + catch (OperationCanceledException) when (token.IsCancellationRequested) + { + Trace.Info("Get next message has been cancelled."); + throw; + } + catch (TaskAgentAccessTokenExpiredException) + { + Trace.Info("Runner OAuth token has been revoked. Unable to pull message."); + throw; + } + catch (AccessDeniedException e) when (e.InnerException is InvalidTaskAgentVersionException) + { + throw; + } + catch (Exception ex) + { + Trace.Error("Catch exception during get next message."); + Trace.Error(ex); + + if (!IsGetNextMessageExceptionRetriable(ex)) + { + throw; + } + else + { + continuousError++; + //retry after a random backoff to avoid service throttling + //in case of there is a service error happened and all agents get kicked off of the long poll and all agent try to reconnect back at the same time. + if (continuousError <= 5) + { + // random backoff [15, 30] + _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(15), TimeSpan.FromSeconds(30), _getNextMessageRetryInterval); + } + else if (continuousError >= maxRetryCount) + { + throw; + } + else + { + // more aggressive backoff [30, 60] + _getNextMessageRetryInterval = BackoffTimerHelper.GetRandomBackoff(TimeSpan.FromSeconds(30), TimeSpan.FromSeconds(60), _getNextMessageRetryInterval); + } + + if (!encounteringError) + { + //print error only on the first consecutive error + _term.WriteError($"{DateTime.UtcNow:u}: Runner connect error: {ex.Message}. Retrying until reconnected."); + encounteringError = true; + } + + // re-create VssConnection before next retry + await RefreshBrokerConnection(); + + Trace.Info("Sleeping for {0} seconds before retrying.", _getNextMessageRetryInterval.TotalSeconds); + await HostContext.Delay(_getNextMessageRetryInterval, token); + } + } + finally + { + _getMessagesTokenSource.Dispose(); + } + + if (message == null) + { + if (heartbeat.Elapsed > TimeSpan.FromMinutes(30)) + { + Trace.Info($"No message retrieved within last 30 minutes."); + heartbeat.Restart(); + } + else + { + Trace.Verbose($"No message retrieved."); + } + + continue; + } + + Trace.Info($"Message '{message.MessageId}' received."); + } + } + + public async Task DeleteMessageAsync(TaskAgentMessage message) + { + await Task.CompletedTask; + } + + private bool IsGetNextMessageExceptionRetriable(Exception ex) + { + if (ex is TaskAgentNotFoundException || + ex is TaskAgentPoolNotFoundException || + ex is TaskAgentSessionExpiredException || + ex is AccessDeniedException || + ex is VssUnauthorizedException) + { + Trace.Info($"Non-retriable exception: {ex.Message}"); + return false; + } + else + { + Trace.Info($"Retriable exception: {ex.Message}"); + return true; + } + } + + private async Task RefreshBrokerConnection() + { + var configManager = HostContext.GetService(); + _settings = configManager.LoadSettings(); + + var credMgr = HostContext.GetService(); + VssCredentials creds = credMgr.LoadCredentials(); + await _brokerServer.ConnectAsync(new Uri(_settings.ServerUrlV2), creds); + } + } +} diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index c727f1b3865..e95379ad385 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -339,13 +339,25 @@ private void CtrlCHandler(object sender, EventArgs e) } } + private IMessageListener GetMesageListener(RunnerSettings settings) + { + if (settings.UseV2Flow) + { + var brokerListener = new BrokerMessageListener(); + brokerListener.Initialize(HostContext); + return brokerListener; + } + + return HostContext.GetService(); + } + //create worker manager, create message listener and start listening to the queue private async Task RunAsync(RunnerSettings settings, bool runOnce = false) { try { Trace.Info(nameof(RunAsync)); - _listener = HostContext.GetService(); + _listener = GetMesageListener(settings); if (!await _listener.CreateSessionAsync(HostContext.RunnerShutdownToken)) { return Constants.Runner.ReturnCode.TerminatedError; diff --git a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs new file mode 100644 index 00000000000..8f9b22e7541 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs @@ -0,0 +1,84 @@ +using System; +using System.Collections.Generic; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.Pipelines; +using GitHub.DistributedTask.WebApi; +using GitHub.Services.Common; +using GitHub.Services.OAuth; +using GitHub.Services.WebApi; +using Sdk.RSWebApi.Contracts; +using Sdk.WebApi.WebApi; + +namespace GitHub.Actions.RunService.WebApi +{ + public class BrokerHttpClient : RawHttpClientBase + { + public BrokerHttpClient( + Uri baseUrl, + VssOAuthCredential credentials) + : base(baseUrl, credentials) + { + } + + public BrokerHttpClient( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings) + : base(baseUrl, credentials, settings) + { + } + + public BrokerHttpClient( + Uri baseUrl, + VssOAuthCredential credentials, + params DelegatingHandler[] handlers) + : base(baseUrl, credentials, handlers) + { + } + + public BrokerHttpClient( + Uri baseUrl, + VssOAuthCredential credentials, + RawClientHttpRequestSettings settings, + params DelegatingHandler[] handlers) + : base(baseUrl, credentials, settings, handlers) + { + } + + public BrokerHttpClient( + Uri baseUrl, + HttpMessageHandler pipeline, + Boolean disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + } + + public Task GetRunnerMessageAsync( + string runnerVersion, + TaskAgentStatus? status, + CancellationToken cancellationToken = default + ) + { + var requestUri = new Uri(Client.BaseAddress, "message"); + + List> queryParams = new List>(); + + if (status != null) + { + queryParams.Add("status", status.Value.ToString()); + } + if (runnerVersion != null) + { + queryParams.Add("runnerVersion", runnerVersion); + } + + return SendAsync( + new HttpMethod("GET"), + requestUri: requestUri, + queryParameters: queryParams, + cancellationToken: cancellationToken); + } + } +} From b486802e89b1918c8cdd7b957e7597edec7c8b57 Mon Sep 17 00:00:00 2001 From: Luke Tomlinson Date: Tue, 28 Mar 2023 15:45:00 -0400 Subject: [PATCH 117/145] Register Runners against V2 servers (#2505) * Parse runners and send publicKey * wip * Fix conflicts * Cleanup * Cleanup * fix test * fix test * Add trace for broker message listener * Feedback * refactor * remove dead code * Remove old comment --- src/Runner.Common/RunnerDotcomServer.cs | 10 ++-- src/Runner.Listener/BrokerMessageListener.cs | 5 ++ .../Configuration/ConfigurationManager.cs | 50 +++++++++++------- src/Runner.Listener/Runner.cs | 1 + .../DTWebApi/WebApi/ListRunnersResponse.cs | 2 - src/Sdk/DTWebApi/WebApi/Runner.cs | 51 +++++++++++++++++-- .../Configuration/ConfigurationManagerL0.cs | 9 +++- 7 files changed, 99 insertions(+), 29 deletions(-) diff --git a/src/Runner.Common/RunnerDotcomServer.cs b/src/Runner.Common/RunnerDotcomServer.cs index 510877e2e17..41cc0816d6f 100644 --- a/src/Runner.Common/RunnerDotcomServer.cs +++ b/src/Runner.Common/RunnerDotcomServer.cs @@ -17,7 +17,7 @@ public interface IRunnerDotcomServer : IRunnerService { Task> GetRunnersAsync(int runnerGroupId, string githubUrl, string githubToken, string agentName); - Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, string githubUrl, string githubToken); + Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, string githubUrl, string githubToken, string publicKey); Task> GetRunnerGroupsAsync(string githubUrl, string githubToken); string GetGitHubRequestId(HttpResponseHeaders headers); @@ -136,7 +136,7 @@ public async Task> GetRunnerGroupsAsync(string githubUrl, st return agentPools?.ToAgentPoolList(); } - public async Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, string githubUrl, string githubToken) + public async Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, string githubUrl, string githubToken, string publicKey) { var gitHubUrlBuilder = new UriBuilder(githubUrl); var path = gitHubUrlBuilder.Path.Split('/', '\\', StringSplitOptions.RemoveEmptyEntries); @@ -159,12 +159,12 @@ public async Task AddRunnerAsync(int runnerGroupId, TaskAgent agent, {"updates_disabled", agent.DisableUpdate}, {"ephemeral", agent.Ephemeral}, {"labels", agent.Labels}, + {"public_key", publicKey} }; var body = new StringContent(StringUtil.ConvertToJson(bodyObject), null, "application/json"); - var responseAgent = await RetryRequest(githubApiUrl, githubToken, RequestType.Post, 3, "Failed to add agent", body); - agent.Id = responseAgent.Id; - return agent; + + return await RetryRequest(githubApiUrl, githubToken, RequestType.Post, 3, "Failed to add agent", body); } private async Task RetryRequest(string githubApiUrl, string githubToken, RequestType requestType, int maxRetryAttemptsCount = 5, string errorMessage = null, StringContent body = null) diff --git a/src/Runner.Listener/BrokerMessageListener.cs b/src/Runner.Listener/BrokerMessageListener.cs index 93259c8e56f..aa3e8188647 100644 --- a/src/Runner.Listener/BrokerMessageListener.cs +++ b/src/Runner.Listener/BrokerMessageListener.cs @@ -196,6 +196,11 @@ private async Task RefreshBrokerConnection() var configManager = HostContext.GetService(); _settings = configManager.LoadSettings(); + if (_settings.ServerUrlV2 == null) + { + throw new InvalidOperationException("ServerUrlV2 is not set"); + } + var credMgr = HostContext.GetService(); VssCredentials creds = credMgr.LoadCredentials(); await _brokerServer.ConnectAsync(new Uri(_settings.ServerUrlV2), creds); diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 492bea04213..0c835504a00 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -181,9 +181,11 @@ public async Task ConfigureAsync(CommandSettings command) // We want to use the native CSP of the platform for storage, so we use the RSACSP directly RSAParameters publicKey; var keyManager = HostContext.GetService(); + string publicKeyXML; using (var rsa = keyManager.CreateKey()) { publicKey = rsa.ExportParameters(false); + publicKeyXML = rsa.ToXmlString(includePrivateParameters: false); } _term.WriteSection("Runner Registration"); @@ -297,7 +299,15 @@ public async Task ConfigureAsync(CommandSettings command) { if (runnerSettings.UseV2Flow) { - agent = await _dotcomServer.AddRunnerAsync(runnerSettings.PoolId, agent, runnerSettings.GitHubUrl, registerToken); + var runner = await _dotcomServer.AddRunnerAsync(runnerSettings.PoolId, agent, runnerSettings.GitHubUrl, registerToken, publicKeyXML); + runnerSettings.ServerUrlV2 = runner.RunnerAuthorization.ServerUrl; + + agent.Id = runner.Id; + agent.Authorization = new TaskAgentAuthorization() + { + AuthorizationUrl = runner.RunnerAuthorization.AuthorizationUrl, + ClientId = new Guid(runner.RunnerAuthorization.ClientId) + }; } else { @@ -354,24 +364,28 @@ public async Task ConfigureAsync(CommandSettings command) } // Testing agent connection, detect any potential connection issue, like local clock skew that cause OAuth token expired. - var credMgr = HostContext.GetService(); - VssCredentials credential = credMgr.LoadCredentials(); - try - { - await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), credential); - // ConnectAsync() hits _apis/connectionData which is an anonymous endpoint - // Need to hit an authenticate endpoint to trigger OAuth token exchange. - await _runnerServer.GetAgentPoolsAsync(); - _term.WriteSuccessMessage("Runner connection is good"); - } - catch (VssOAuthTokenRequestException ex) when (ex.Message.Contains("Current server time is")) + + if (!runnerSettings.UseV2Flow) { - // there are two exception messages server send that indicate clock skew. - // 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}. - // 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}. - Trace.Error("Catch exception during test agent connection."); - Trace.Error(ex); - throw new Exception("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again."); + var credMgr = HostContext.GetService(); + VssCredentials credential = credMgr.LoadCredentials(); + try + { + await _runnerServer.ConnectAsync(new Uri(runnerSettings.ServerUrl), credential); + // ConnectAsync() hits _apis/connectionData which is an anonymous endpoint + // Need to hit an authenticate endpoint to trigger OAuth token exchange. + await _runnerServer.GetAgentPoolsAsync(); + _term.WriteSuccessMessage("Runner connection is good"); + } + catch (VssOAuthTokenRequestException ex) when (ex.Message.Contains("Current server time is")) + { + // there are two exception messages server send that indicate clock skew. + // 1. The bearer token expired on {jwt.ValidTo}. Current server time is {DateTime.UtcNow}. + // 2. The bearer token is not valid until {jwt.ValidFrom}. Current server time is {DateTime.UtcNow}. + Trace.Error("Catch exception during test agent connection."); + Trace.Error(ex); + throw new Exception("The local machine's clock may be out of sync with the server time by more than five minutes. Please sync your clock with your domain or internet time and try again."); + } } _term.WriteSection("Runner settings"); diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index e95379ad385..865c596a91e 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -343,6 +343,7 @@ private IMessageListener GetMesageListener(RunnerSettings settings) { if (settings.UseV2Flow) { + Trace.Info($"Using BrokerMessageListener"); var brokerListener = new BrokerMessageListener(); brokerListener.Initialize(HostContext); return brokerListener; diff --git a/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs b/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs index 292b8d66801..ddc83813f62 100644 --- a/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs +++ b/src/Sdk/DTWebApi/WebApi/ListRunnersResponse.cs @@ -41,8 +41,6 @@ public ListRunnersResponse Clone() public List ToTaskAgents() { - List taskAgents = new List(); - return Runners.Select(runner => new TaskAgent() { Name = runner.Name }).ToList(); } } diff --git a/src/Sdk/DTWebApi/WebApi/Runner.cs b/src/Sdk/DTWebApi/WebApi/Runner.cs index 2103bf3a210..dc871c1ca14 100644 --- a/src/Sdk/DTWebApi/WebApi/Runner.cs +++ b/src/Sdk/DTWebApi/WebApi/Runner.cs @@ -1,12 +1,44 @@ +using System; using Newtonsoft.Json; namespace GitHub.DistributedTask.WebApi { public class Runner { - /// - /// Name of the agent - /// + + public class Authorization + { + /// + /// The url to refresh tokens + /// + [JsonProperty("authorization_url")] + public Uri AuthorizationUrl + { + get; + internal set; + } + + /// + /// The url to connect to to poll for messages + /// + [JsonProperty("server_url")] + public string ServerUrl + { + get; + internal set; + } + + /// + /// The client id to use when connecting to the authorization_url + /// + [JsonProperty("client_id")] + public string ClientId + { + get; + internal set; + } + } + [JsonProperty("name")] public string Name { @@ -14,5 +46,18 @@ public string Name internal set; } + [JsonProperty("id")] + public Int32 Id + { + get; + internal set; + } + + [JsonProperty("authorization")] + public Authorization RunnerAuthorization + { + get; + internal set; + } } } diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index 19bf0250c79..12918390073 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -73,6 +73,13 @@ public ConfigurationManagerL0() AuthorizationUrl = new Uri("http://localhost:8080/pipelines"), }; + var expectedRunner = new GitHub.DistributedTask.WebApi.Runner() { Name = expectedAgent.Name, Id = 1 }; + expectedRunner.RunnerAuthorization = new GitHub.DistributedTask.WebApi.Runner.Authorization + { + ClientId = expectedAgent.Authorization.ClientId.ToString(), + AuthorizationUrl = new Uri("http://localhost:8080/pipelines"), + }; + var connectionData = new ConnectionData() { InstanceId = Guid.NewGuid(), @@ -110,7 +117,7 @@ public ConfigurationManagerL0() _dotcomServer.Setup(x => x.GetRunnersAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedAgents)); _dotcomServer.Setup(x => x.GetRunnerGroupsAsync(It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedPools)); - _dotcomServer.Setup(x => x.AddRunnerAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedAgent)); + _dotcomServer.Setup(x => x.AddRunnerAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())).Returns(Task.FromResult(expectedRunner)); rsa = new RSACryptoServiceProvider(2048); From fb6c3794b1495c2a1897b98d40dcdeeb4a84be6c Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Thu, 30 Mar 2023 15:39:09 +0200 Subject: [PATCH 118/145] Guard against NullReference while creating HostContext (#2343) * Guard against NullReference while creating HostContext * Throw on IOUtil loading object if content is empty or object is null * removed unused dependency * Changed exceptions to ArgumentNullException and ArgumentException * fixed my mistake on LoadObject * fixed tests * trigger workflows * trigger workflows * encode files using utf8 --- src/Runner.Common/HostContext.cs | 4 ++-- src/Runner.Sdk/Util/IOUtil.cs | 13 +++++++++++-- src/Test/L0/Util/IOUtilL0.cs | 31 ++++++++++++++++++++++++++++++- 3 files changed, 43 insertions(+), 5 deletions(-) diff --git a/src/Runner.Common/HostContext.cs b/src/Runner.Common/HostContext.cs index 4a78da83fd2..4d9ff24c53e 100644 --- a/src/Runner.Common/HostContext.cs +++ b/src/Runner.Common/HostContext.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; @@ -220,7 +220,7 @@ public HostContext(string hostType, string logFile = null) var runnerFile = GetConfigFile(WellKnownConfigFile.Runner); if (File.Exists(runnerFile)) { - var runnerSettings = IOUtil.LoadObject(runnerFile); + var runnerSettings = IOUtil.LoadObject(runnerFile, true); _userAgents.Add(new ProductInfoHeaderValue("RunnerId", runnerSettings.AgentId.ToString(CultureInfo.InvariantCulture))); _userAgents.Add(new ProductInfoHeaderValue("GroupId", runnerSettings.PoolId.ToString(CultureInfo.InvariantCulture))); } diff --git a/src/Runner.Sdk/Util/IOUtil.cs b/src/Runner.Sdk/Util/IOUtil.cs index d956988192a..2ecca678774 100644 --- a/src/Runner.Sdk/Util/IOUtil.cs +++ b/src/Runner.Sdk/Util/IOUtil.cs @@ -40,10 +40,19 @@ public static void SaveObject(object obj, string path) File.WriteAllText(path, StringUtil.ConvertToJson(obj), Encoding.UTF8); } - public static T LoadObject(string path) + public static T LoadObject(string path, bool required = false) { string json = File.ReadAllText(path, Encoding.UTF8); - return StringUtil.ConvertFromJson(json); + if (required && string.IsNullOrEmpty(json)) + { + throw new ArgumentNullException($"File {path} is empty"); + } + T result = StringUtil.ConvertFromJson(json); + if (required && result == null) + { + throw new ArgumentException("Converting json to object resulted in a null value"); + } + return result; } public static string GetSha256Hash(string path) diff --git a/src/Test/L0/Util/IOUtilL0.cs b/src/Test/L0/Util/IOUtilL0.cs index 6a504d19221..a3901c90de2 100644 --- a/src/Test/L0/Util/IOUtilL0.cs +++ b/src/Test/L0/Util/IOUtilL0.cs @@ -1,4 +1,3 @@ -using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.IO; @@ -931,6 +930,36 @@ public void ValidateExecutePermission_ExceedsFailsafe() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void LoadObject_ThrowsOnRequiredLoadObject() + { + using (TestHostContext hc = new(this)) + { + Tracing trace = hc.GetTrace(); + + // Arrange: Create a directory with a file. + string directory = Path.Combine(hc.GetDirectory(WellKnownDirectory.Bin), Path.GetRandomFileName()); + + string file = Path.Combine(directory, "empty file"); + Directory.CreateDirectory(directory); + + File.WriteAllText(path: file, contents: ""); + Assert.Throws(() => IOUtil.LoadObject(file, true)); + + file = Path.Combine(directory, "invalid type file"); + File.WriteAllText(path: file, contents: " "); + Assert.Throws(() => IOUtil.LoadObject(file, true)); + + // Cleanup. + if (Directory.Exists(directory)) + { + Directory.Delete(directory, recursive: true); + } + } + } + private static async Task CreateDirectoryReparsePoint(IHostContext context, string link, string target) { #if OS_WINDOWS From 56cb9c1d2d2f42623231effb70a996e3fd05e948 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Mon, 3 Apr 2023 16:31:13 -0400 Subject: [PATCH 119/145] Update Runner to send step updates to Results (#2510) * Also send Steps update to Results service * Refactor to separate results server from current job server * If hit any error while uploading to Results, skip Results upload * Add proxy authentication and buffer request for WinHttpHandler * Remove unnecessary null guard * Also send Results telemetry when step update fails * IResultsServer is not disposable --- src/Runner.Common/JobServer.cs | 37 ------- src/Runner.Common/JobServerQueue.cs | 61 +++++++++--- src/Runner.Common/ResultsServer.cs | 98 +++++++++++++++++++ .../Common/RawClientHttpRequestSettings.cs | 24 +++++ .../Common/Common/RawHttpMessageHandler.cs | 43 +++++++- .../Common/Common/VssHttpMessageHandler.cs | 2 +- .../Common/Common/VssHttpRequestSettings.cs | 2 +- src/Sdk/WebApi/WebApi/Contracts.cs | 41 ++++++++ src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 74 ++++++++++++-- 9 files changed, 314 insertions(+), 68 deletions(-) create mode 100644 src/Runner.Common/ResultsServer.cs diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 9d845037868..04c244ad8a3 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -24,15 +24,11 @@ public interface IJobServer : IRunnerService, IAsyncDisposable Task ConnectAsync(VssConnection jobConnection); void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint); - void InitializeResultsClient(Uri uri, string token); // logging and console Task AppendLogContentAsync(Guid scopeIdentifier, string hubName, Guid planId, int logId, Stream uploadStream, CancellationToken cancellationToken); Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); Task CreateAttachmentAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, String type, String name, Stream uploadStream, CancellationToken cancellationToken); - Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken); - Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); - Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken); Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken); Task CreateTimelineAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, CancellationToken cancellationToken); Task> UpdateTimelineRecordsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, IEnumerable records, CancellationToken cancellationToken); @@ -46,7 +42,6 @@ public sealed class JobServer : RunnerService, IJobServer private bool _hasConnection; private VssConnection _connection; private TaskHttpClient _taskClient; - private ResultsHttpClient _resultsClient; private ClientWebSocket _websocketClient; private ServiceEndpoint _serviceEndpoint; @@ -150,12 +145,6 @@ public void InitializeWebsocketClient(ServiceEndpoint serviceEndpoint) InitializeWebsocketClient(TimeSpan.Zero); } - public void InitializeResultsClient(Uri uri, string token) - { - var httpMessageHandler = HostContext.CreateHttpClientHandler(); - this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true); - } - public ValueTask DisposeAsync() { CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None); @@ -318,32 +307,6 @@ public Task CreateAttachmentAsync(Guid scopeIdentifier, string h return _taskClient.CreateAttachmentAsync(scopeIdentifier, hubName, planId, timelineId, timelineRecordId, type, name, uploadStream, cancellationToken: cancellationToken); } - public Task CreateStepSummaryAsync(string planId, string jobId, Guid stepId, string file, CancellationToken cancellationToken) - { - if (_resultsClient != null) - { - return _resultsClient.UploadStepSummaryAsync(planId, jobId, stepId, file, cancellationToken: cancellationToken); - } - throw new InvalidOperationException("Results client is not initialized."); - } - - public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) - { - if (_resultsClient != null) - { - return _resultsClient.UploadResultsStepLogAsync(planId, jobId, stepId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken); - } - throw new InvalidOperationException("Results client is not initialized."); - } - - public Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) - { - if (_resultsClient != null) - { - return _resultsClient.UploadResultsJobLogAsync(planId, jobId, file, finalize, firstBlock, lineCount, cancellationToken: cancellationToken); - } - throw new InvalidOperationException("Results client is not initialized."); - } public Task CreateLogAsync(Guid scopeIdentifier, string hubName, Guid planId, TaskLog log, CancellationToken cancellationToken) { diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 1fadd99a8df..20ad27a89fb 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -65,6 +65,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue // common private IJobServer _jobServer; + private IResultsServer _resultsServer; private Task[] _allDequeueTasks; private readonly TaskCompletionSource _jobCompletionSource = new(); private readonly TaskCompletionSource _jobRecordUpdated = new(); @@ -91,6 +92,7 @@ public override void Initialize(IHostContext hostContext) { base.Initialize(hostContext); _jobServer = hostContext.GetService(); + _resultsServer = hostContext.GetService(); } public void Start(Pipelines.AgentJobRequestMessage jobRequest) @@ -111,7 +113,7 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) !string.IsNullOrEmpty(resultsReceiverEndpoint)) { Trace.Info("Initializing results client"); - _jobServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken); + _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken); _resultsClientInitiated = true; } @@ -512,19 +514,14 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) } catch (Exception ex) { - var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception during file upload to results. {ex.Message}" }; - issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; - - var telemetryRecord = new TimelineRecord() - { - Id = Constants.Runner.TelemetryRecordId, - }; - telemetryRecord.Issues.Add(issue); - QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord); - Trace.Info("Catch exception during file upload to results, keep going since the process is best effort."); Trace.Error(ex); errorCount++; + + // If we hit any exceptions uploading to Results, let's skip any additional uploads to Results + _resultsClientInitiated = false; + + SendResultsTelemetry(ex); } } @@ -542,6 +539,19 @@ private async Task ProcessResultsUploadQueueAsync(bool runOnce = false) } } + private void SendResultsTelemetry(Exception ex) + { + var issue = new Issue() { Type = IssueType.Warning, Message = $"Caught exception with results. {ex.Message}" }; + issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.ResultsUploadFailure; + + var telemetryRecord = new TimelineRecord() + { + Id = Constants.Runner.TelemetryRecordId, + }; + telemetryRecord.Issues.Add(issue); + QueueTimelineRecordUpdate(_jobTimelineId, telemetryRecord); + } + private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) { while (!_jobCompletionSource.Task.IsCompleted || runOnce) @@ -612,6 +622,22 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) try { await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken)); + try + { + if (_resultsClientInitiated) + { + await _resultsServer.UpdateResultsWorkflowStepsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken)); + } + } + catch (Exception e) + { + Trace.Info("Catch exception during update steps, skip update Results."); + Trace.Error(e); + _resultsClientInitiated = false; + + SendResultsTelemetry(e); + } + if (_bufferedRetryRecords.Remove(update.TimelineId)) { Trace.Verbose("Cleanup buffered timeline record for timeline: {0}.", update.TimelineId); @@ -819,7 +845,7 @@ private async Task UploadSummaryFile(ResultsUploadFileInfo file) Trace.Info($"Starting to upload summary file to results service {file.Name}, {file.Path}"); ResultsFileUploadHandler summaryHandler = async (file) => { - await _jobServer.CreateStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, CancellationToken.None); + await _resultsServer.CreateResultsStepSummaryAsync(file.PlanId, file.JobId, file.RecordId, file.Path, CancellationToken.None); }; await UploadResultsFile(file, summaryHandler); @@ -830,7 +856,7 @@ private async Task UploadResultsStepLogFile(ResultsUploadFileInfo file) Trace.Info($"Starting upload of step log file to results service {file.Name}, {file.Path}"); ResultsFileUploadHandler stepLogHandler = async (file) => { - await _jobServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); + await _resultsServer.CreateResultsStepLogAsync(file.PlanId, file.JobId, file.RecordId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); }; await UploadResultsFile(file, stepLogHandler); @@ -841,7 +867,7 @@ private async Task UploadResultsJobLogFile(ResultsUploadFileInfo file) Trace.Info($"Starting upload of job log file to results service {file.Name}, {file.Path}"); ResultsFileUploadHandler jobLogHandler = async (file) => { - await _jobServer.CreateResultsJobLogAsync(file.PlanId, file.JobId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); + await _resultsServer.CreateResultsJobLogAsync(file.PlanId, file.JobId, file.Path, file.Finalize, file.FirstBlock, file.TotalLines, CancellationToken.None); }; await UploadResultsFile(file, jobLogHandler); @@ -849,6 +875,11 @@ private async Task UploadResultsJobLogFile(ResultsUploadFileInfo file) private async Task UploadResultsFile(ResultsUploadFileInfo file, ResultsFileUploadHandler uploadHandler) { + if (!_resultsClientInitiated) + { + return; + } + bool uploadSucceed = false; try { @@ -903,8 +934,6 @@ internal class ResultsUploadFileInfo public long TotalLines { get; set; } } - - internal class ConsoleLineInfo { public ConsoleLineInfo(Guid recordId, string line, long? lineNumber) diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs new file mode 100644 index 00000000000..4d24d55c86a --- /dev/null +++ b/src/Runner.Common/ResultsServer.cs @@ -0,0 +1,98 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Services.Results.Client; + +namespace GitHub.Runner.Common +{ + [ServiceLocator(Default = typeof(ResultServer))] + public interface IResultsServer : IRunnerService + { + void InitializeResultsClient(Uri uri, string token); + + // logging and console + Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file, + CancellationToken cancellationToken); + + Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, + bool firstBlock, long lineCount, CancellationToken cancellationToken); + + Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, + long lineCount, CancellationToken cancellationToken); + + Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, + IEnumerable records, CancellationToken cancellationToken); + } + + public sealed class ResultServer : RunnerService, IResultsServer + { + private ResultsHttpClient _resultsClient; + + public void InitializeResultsClient(Uri uri, string token) + { + var httpMessageHandler = HostContext.CreateHttpClientHandler(); + this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true); + } + + public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file, + CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadStepSummaryAsync(planId, jobId, stepId, file, + cancellationToken: cancellationToken); + } + + throw new InvalidOperationException("Results client is not initialized."); + } + + public Task CreateResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, + bool firstBlock, long lineCount, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadResultsStepLogAsync(planId, jobId, stepId, file, finalize, firstBlock, + lineCount, cancellationToken: cancellationToken); + } + + throw new InvalidOperationException("Results client is not initialized."); + } + + public Task CreateResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, + long lineCount, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + return _resultsClient.UploadResultsJobLogAsync(planId, jobId, file, finalize, firstBlock, lineCount, + cancellationToken: cancellationToken); + } + + throw new InvalidOperationException("Results client is not initialized."); + } + + public Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, + IEnumerable records, CancellationToken cancellationToken) + { + if (_resultsClient != null) + { + try + { + var timelineRecords = records.ToList(); + return _resultsClient.UpdateWorkflowStepsAsync(planId, new List(timelineRecords), + cancellationToken: cancellationToken); + } + catch (Exception ex) + { + // Log error, but continue as this call is best-effort + Trace.Info($"Failed to update steps status due to {ex.GetType().Name}"); + Trace.Error(ex); + } + } + + throw new InvalidOperationException("Results client is not initialized."); + } + } +} diff --git a/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs b/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs index 60fbbe093fa..c6956f29faa 100644 --- a/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs +++ b/src/Sdk/Common/Common/RawClientHttpRequestSettings.cs @@ -184,9 +184,33 @@ private static RawClientHttpRequestSettings ConstructDefaultSettings() return settings; } + /// + /// Gets or sets the maximum size allowed for response content buffering. + /// + [DefaultValue(c_defaultContentBufferSize)] + public Int32 MaxContentBufferSize + { + get + { + return m_maxContentBufferSize; + } + set + { + ArgumentUtility.CheckForOutOfRange(value, nameof(value), 0, c_maxAllowedContentBufferSize); + m_maxContentBufferSize = value; + } + } + private static Lazy s_defaultSettings = new Lazy(ConstructDefaultSettings); + private Int32 m_maxContentBufferSize; + // We will buffer a maximum of 1024MB in the message handler + private const Int32 c_maxAllowedContentBufferSize = 1024 * 1024 * 1024; + + // We will buffer, by default, up to 512MB in the message handler + private const Int32 c_defaultContentBufferSize = 1024 * 1024 * 512; + private const Int32 c_defaultMaxRetry = 3; private static readonly TimeSpan s_defaultTimeout = TimeSpan.FromSeconds(100); //default WebAPI timeout private ICollection m_acceptLanguages = new List(); diff --git a/src/Sdk/Common/Common/RawHttpMessageHandler.cs b/src/Sdk/Common/Common/RawHttpMessageHandler.cs index 8b3e75e1844..6774d371218 100644 --- a/src/Sdk/Common/Common/RawHttpMessageHandler.cs +++ b/src/Sdk/Common/Common/RawHttpMessageHandler.cs @@ -9,7 +9,7 @@ namespace GitHub.Services.Common { - public class RawHttpMessageHandler: HttpMessageHandler + public class RawHttpMessageHandler : HttpMessageHandler { public RawHttpMessageHandler( FederatedCredential credentials) @@ -120,6 +120,7 @@ protected override async Task SendAsync( Boolean succeeded = false; HttpResponseMessageWrapper responseWrapper; + Boolean lastResponseDemandedProxyAuth = false; Int32 retries = m_maxAuthRetries; try { @@ -138,7 +139,13 @@ protected override async Task SendAsync( // Let's start with sending a token IssuedToken token = await m_tokenProvider.GetTokenAsync(null, tokenSource.Token).ConfigureAwait(false); - ApplyToken(request, token); + ApplyToken(request, token, applyICredentialsToWebProxy: lastResponseDemandedProxyAuth); + + // The WinHttpHandler will chunk any content that does not have a computed length which is + // not what we want. By loading into a buffer up-front we bypass this behavior and there is + // no difference in the normal HttpClientHandler behavior here since this is what they were + // already doing. + await BufferRequestContentAsync(request, tokenSource.Token).ConfigureAwait(false); // ConfigureAwait(false) enables the continuation to be run outside any captured // SyncronizationContext (such as ASP.NET's) which keeps things from deadlocking... @@ -147,7 +154,8 @@ protected override async Task SendAsync( responseWrapper = new HttpResponseMessageWrapper(response); var isUnAuthorized = responseWrapper.StatusCode == HttpStatusCode.Unauthorized; - if (!isUnAuthorized) + lastResponseDemandedProxyAuth = responseWrapper.StatusCode == HttpStatusCode.ProxyAuthenticationRequired; + if (!isUnAuthorized && !lastResponseDemandedProxyAuth) { // Validate the token after it has been successfully authenticated with the server. m_tokenProvider?.ValidateToken(token, responseWrapper); @@ -211,15 +219,42 @@ protected override async Task SendAsync( } } + private static async Task BufferRequestContentAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + if (request.Content != null && + request.Headers.TransferEncodingChunked != true) + { + Int64? contentLength = request.Content.Headers.ContentLength; + if (contentLength == null) + { + await request.Content.LoadIntoBufferAsync().EnforceCancellation(cancellationToken).ConfigureAwait(false); + } + + // Explicitly turn off chunked encoding since we have computed the request content size + request.Headers.TransferEncodingChunked = false; + } + } + private void ApplyToken( HttpRequestMessage request, - IssuedToken token) + IssuedToken token, + bool applyICredentialsToWebProxy = false) { switch (token) { case null: return; case ICredentials credentialsToken: + if (applyICredentialsToWebProxy) + { + HttpClientHandler httpClientHandler = m_transportHandler as HttpClientHandler; + if (httpClientHandler != null && httpClientHandler.Proxy != null) + { + httpClientHandler.Proxy.Credentials = credentialsToken; + } + } m_credentialWrapper.InnerCredentials = credentialsToken; break; default: diff --git a/src/Sdk/Common/Common/VssHttpMessageHandler.cs b/src/Sdk/Common/Common/VssHttpMessageHandler.cs index b40f7ea21a2..39be573b8aa 100644 --- a/src/Sdk/Common/Common/VssHttpMessageHandler.cs +++ b/src/Sdk/Common/Common/VssHttpMessageHandler.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Net; diff --git a/src/Sdk/Common/Common/VssHttpRequestSettings.cs b/src/Sdk/Common/Common/VssHttpRequestSettings.cs index 06c1554ce6d..8a86eea0822 100644 --- a/src/Sdk/Common/Common/VssHttpRequestSettings.cs +++ b/src/Sdk/Common/Common/VssHttpRequestSettings.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.ComponentModel; using System.Globalization; diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index 9165d832fa9..8da6bf236ff 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -1,3 +1,4 @@ +using System.Collections.Generic; using System.Runtime.Serialization; using Newtonsoft.Json; using Newtonsoft.Json.Serialization; @@ -126,6 +127,46 @@ public class CreateMetadataResponse public bool Ok; } + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class StepsUpdateRequest + { + [DataMember] + public IEnumerable Steps; + [DataMember] + public long ChangeOrder; + [DataMember] + public string WorkflowJobRunBackendId; + [DataMember] + public string WorkflowRunBackendId; + } + + [DataContract] + [JsonObject(NamingStrategyType = typeof(SnakeCaseNamingStrategy))] + public class Step + { + [DataMember] + public string ExternalId; + [DataMember] + public int Number; + [DataMember] + public string Name; + [DataMember] + public Status Status; + [DataMember] + public string StartedAt; + [DataMember] + public string CompletedAt; + } + + public enum Status + { + StatusUnknown = 0, + StatusInProgress = 3, + StatusPending = 5, + StatusCompleted = 6 + } + public static class BlobStorageTypes { public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE"; diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index b80ce4db3bc..b80740f44a5 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -1,11 +1,16 @@ using System; +using System.Collections.Generic; +using System.Diagnostics; using System.IO; +using System.Linq; using System.Net.Http; using System.Net.Http.Headers; using System.Threading; using System.Threading.Tasks; -using GitHub.Services.Results.Contracts; using System.Net.Http.Formatting; +using GitHub.DistributedTask.WebApi; +using GitHub.Services.Common; +using GitHub.Services.Results.Contracts; using Sdk.WebApi.WebApi; namespace GitHub.Services.Results.Client @@ -22,6 +27,7 @@ public ResultsHttpClient( m_token = token; m_resultsServiceUrl = baseUrl; m_formatter = new JsonMediaTypeFormatter(); + m_changeIdCounter = 1; } // Get Sas URL calls @@ -86,7 +92,7 @@ private async Task GetJobLogUploadUrlAsync(string p // Create metadata calls - private async Task CreateMetadata(Uri uri, CancellationToken cancellationToken, R request, string timestamp) + private async Task SendRequest(Uri uri, CancellationToken cancellationToken, R request, string timestamp) { using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri)) { @@ -121,7 +127,7 @@ private async Task StepSummaryUploadCompleteAsync(string planId, string jobId, G }; var createStepSummaryMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepSummaryMetadata); - await CreateMetadata(createStepSummaryMetadataEndpoint, cancellationToken, request, timestamp); + await SendRequest(createStepSummaryMetadataEndpoint, cancellationToken, request, timestamp); } private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid stepId, long lineCount, CancellationToken cancellationToken) @@ -137,7 +143,7 @@ private async Task StepLogUploadCompleteAsync(string planId, string jobId, Guid }; var createStepLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateStepLogsMetadata); - await CreateMetadata(createStepLogsMetadataEndpoint, cancellationToken, request, timestamp); + await SendRequest(createStepLogsMetadataEndpoint, cancellationToken, request, timestamp); } private async Task JobLogUploadCompleteAsync(string planId, string jobId, long lineCount, CancellationToken cancellationToken) @@ -152,7 +158,7 @@ private async Task JobLogUploadCompleteAsync(string planId, string jobId, long l }; var createJobLogsMetadataEndpoint = new Uri(m_resultsServiceUrl, Constants.CreateJobLogsMetadata); - await CreateMetadata(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp); + await SendRequest(createJobLogsMetadataEndpoint, cancellationToken, request, timestamp); } private async Task UploadBlockFileAsync(string url, string blobStorageType, FileStream file, CancellationToken cancellationToken) @@ -252,7 +258,7 @@ public async Task UploadStepSummaryAsync(string planId, string jobId, Guid stepI await StepSummaryUploadCompleteAsync(planId, jobId, stepId, fileSize, cancellationToken); } - // Handle file upload for step log + // Handle file upload for step log public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid stepId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) { // Get the upload url @@ -262,7 +268,7 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st throw new Exception("Failed to get step log upload url"); } - // Create the Append blob + // Create the Append blob if (firstBlock) { await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); @@ -283,7 +289,7 @@ public async Task UploadResultsStepLogAsync(string planId, string jobId, Guid st } } - // Handle file upload for job log + // Handle file upload for job log public async Task UploadResultsJobLogAsync(string planId, string jobId, string file, bool finalize, bool firstBlock, long lineCount, CancellationToken cancellationToken) { // Get the upload url @@ -293,7 +299,7 @@ public async Task UploadResultsJobLogAsync(string planId, string jobId, string f throw new Exception("Failed to get job log upload url"); } - // Create the Append blob + // Create the Append blob if (firstBlock) { await CreateAppendFileAsync(uploadUrlResponse.LogsUrl, uploadUrlResponse.BlobStorageType, cancellationToken); @@ -314,9 +320,57 @@ public async Task UploadResultsJobLogAsync(string planId, string jobId, string f } } + private Step ConvertTimelineRecordToStep(TimelineRecord r) + { + return new Step() + { + ExternalId = r.Id.ToString(), + Number = r.Order.GetValueOrDefault(), + Name = r.Name, + Status = ConvertStateToStatus(r.State.GetValueOrDefault()), + StartedAt = r.StartTime?.ToString(Constants.TimestampFormat), + CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat) + }; + } + + private Status ConvertStateToStatus(TimelineRecordState s) + { + switch (s) + { + case TimelineRecordState.Completed: + return Status.StatusCompleted; + case TimelineRecordState.Pending: + return Status.StatusPending; + case TimelineRecordState.InProgress: + return Status.StatusInProgress; + default: + return Status.StatusUnknown; + } + } + + public async Task UpdateWorkflowStepsAsync(Guid planId, IEnumerable records, CancellationToken cancellationToken) + { + var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat); + var stepRecords = records.Where(r => String.Equals(r.RecordType, "Task", StringComparison.Ordinal)); + var stepUpdateRequests = stepRecords.GroupBy(r => r.ParentId).Select(sg => new StepsUpdateRequest() + { + WorkflowRunBackendId = planId.ToString(), + WorkflowJobRunBackendId = sg.Key.ToString(), + ChangeOrder = m_changeIdCounter++, + Steps = sg.Select(ConvertTimelineRecordToStep) + }); + + var stepUpdateEndpoint = new Uri(m_resultsServiceUrl, Constants.WorkflowStepsUpdate); + foreach (var request in stepUpdateRequests) + { + await SendRequest(stepUpdateEndpoint, cancellationToken, request, timestamp); + } + } + private MediaTypeFormatter m_formatter; private Uri m_resultsServiceUrl; private string m_token; + private int m_changeIdCounter; } // Constants specific to results @@ -331,6 +385,8 @@ public static class Constants public static readonly string CreateStepLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateStepLogsMetadata"; public static readonly string GetJobLogsSignedBlobURL = ResultsReceiverTwirpEndpoint + "GetJobLogsSignedBlobURL"; public static readonly string CreateJobLogsMetadata = ResultsReceiverTwirpEndpoint + "CreateJobLogsMetadata"; + public static readonly string ResultsProtoApiV1Endpoint = "twirp/github.actions.results.api.v1.WorkflowStepUpdateService/"; + public static readonly string WorkflowStepsUpdate = ResultsProtoApiV1Endpoint + "WorkflowStepsUpdate"; public static readonly string AzureBlobSealedHeader = "x-ms-blob-sealed"; public static readonly string AzureBlobTypeHeader = "x-ms-blob-type"; From 17675528eeae1e296570db7850c409a3ad336880 Mon Sep 17 00:00:00 2001 From: Eli Entelis <42981948+eli-entelis@users.noreply.github.com> Date: Fri, 7 Apr 2023 05:15:11 +0300 Subject: [PATCH 120/145] which handles broken symlink & unit test added (#2150) (#2196) * which handles broken symlink & unit test added (#2150) * Update src/Runner.Sdk/Util/WhichUtil.cs Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> * fix pr comments * trace log added, in case we find a symlink that is broken * add check in case the target of the symlink is a relative path; added test that check symlink that targets a full path; added test that check symlink that targets a relative path; * fix tests * fix tests for linux --------- Co-authored-by: Eli Entelis Co-authored-by: Eli Entelis <42981948+Eliminator1999@users.noreply.github.com> Co-authored-by: Ferenc Hammerl <31069338+fhammerl@users.noreply.github.com> --- src/Runner.Sdk/Util/WhichUtil.cs | 16 +++- src/Test/L0/Util/WhichUtilL0.cs | 125 ++++++++++++++++++++++++++++++- 2 files changed, 137 insertions(+), 4 deletions(-) diff --git a/src/Runner.Sdk/Util/WhichUtil.cs b/src/Runner.Sdk/Util/WhichUtil.cs index 1a12a889ab2..d1bf5dba778 100644 --- a/src/Runner.Sdk/Util/WhichUtil.cs +++ b/src/Runner.Sdk/Util/WhichUtil.cs @@ -60,7 +60,7 @@ public static string Which(string command, bool require = false, ITraceWriter tr trace?.Verbose(ex.ToString()); } - if (matches != null && matches.Length > 0) + if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace)) { trace?.Info($"Location: '{matches.First()}'"); return matches.First(); @@ -86,7 +86,7 @@ public static string Which(string command, bool require = false, ITraceWriter tr for (int i = 0; i < pathExtSegments.Length; i++) { string fullPath = Path.Combine(pathSegment, $"{command}{pathExtSegments[i]}"); - if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase))) + if (matches.Any(p => p.Equals(fullPath, StringComparison.OrdinalIgnoreCase)) && IsPathValid(fullPath, trace)) { trace?.Info($"Location: '{fullPath}'"); return fullPath; @@ -105,7 +105,7 @@ public static string Which(string command, bool require = false, ITraceWriter tr trace?.Verbose(ex.ToString()); } - if (matches != null && matches.Length > 0) + if (matches != null && matches.Length > 0 && IsPathValid(matches.First(), trace)) { trace?.Info($"Location: '{matches.First()}'"); return matches.First(); @@ -128,5 +128,15 @@ public static string Which(string command, bool require = false, ITraceWriter tr return null; } + + // checks if the file is a symlink and if the symlink`s target exists. + private static bool IsPathValid(string path, ITraceWriter trace = null) + { + var fileInfo = new FileInfo(path); + var linkTargetFullPath = fileInfo.Directory?.FullName + Path.DirectorySeparatorChar + fileInfo.LinkTarget; + if(fileInfo.LinkTarget == null || File.Exists(linkTargetFullPath) || File.Exists(fileInfo.LinkTarget)) return true; + trace?.Info($"the target '{fileInfo.LinkTarget}' of the symbolic link '{path}', does not exist"); + return false; + } } } diff --git a/src/Test/L0/Util/WhichUtilL0.cs b/src/Test/L0/Util/WhichUtilL0.cs index c6a05714442..180ff6ab053 100644 --- a/src/Test/L0/Util/WhichUtilL0.cs +++ b/src/Test/L0/Util/WhichUtilL0.cs @@ -1,4 +1,4 @@ -using GitHub.Runner.Common.Util; +using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using System; using System.IO; @@ -89,5 +89,128 @@ public void WhichHandleFullyQualifiedPath() Assert.Equal(gitPath, gitPath2); } } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void WhichHandlesSymlinkToTargetFullPath() + { + // Arrange + using TestHostContext hc = new TestHostContext(this); + Tracing trace = hc.GetTrace(); + string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable); +#if OS_WINDOWS + string newValue = oldValue + @$";{Path.GetTempPath()}"; + string symlinkName = $"symlink-{Guid.NewGuid()}"; + string symlink = Path.GetTempPath() + $"{symlinkName}.exe"; + string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}.exe"; +#else + string newValue = oldValue + @$":{Path.GetTempPath()}"; + string symlinkName = $"symlink-{Guid.NewGuid()}"; + string symlink = Path.GetTempPath() + $"{symlinkName}"; + string target = Path.GetTempPath() + $"target-{Guid.NewGuid()}"; +#endif + + Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue); + + + using (File.Create(target)) + { + File.CreateSymbolicLink(symlink, target); + + // Act. + var result = WhichUtil.Which(symlinkName, require: true, trace: trace); + + // Assert + Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find symlink through: {nameof(WhichUtil.Which)}"); + + } + + + // Cleanup + File.Delete(symlink); + File.Delete(target); + Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); + + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void WhichHandlesSymlinkToTargetRelativePath() + { + // Arrange + using TestHostContext hc = new TestHostContext(this); + Tracing trace = hc.GetTrace(); + string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable); +#if OS_WINDOWS + string newValue = oldValue + @$";{Path.GetTempPath()}"; + string symlinkName = $"symlink-{Guid.NewGuid()}"; + string symlink = Path.GetTempPath() + $"{symlinkName}.exe"; + string targetName = $"target-{Guid.NewGuid()}.exe"; + string target = Path.GetTempPath() + targetName; +#else + string newValue = oldValue + @$":{Path.GetTempPath()}"; + string symlinkName = $"symlink-{Guid.NewGuid()}"; + string symlink = Path.GetTempPath() + $"{symlinkName}"; + string targetName = $"target-{Guid.NewGuid()}"; + string target = Path.GetTempPath() + targetName; +#endif + Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue); + + + using (File.Create(target)) + { + File.CreateSymbolicLink(symlink, targetName); + + // Act. + var result = WhichUtil.Which(symlinkName, require: true, trace: trace); + + // Assert + Assert.True(!string.IsNullOrEmpty(result) && File.Exists(result), $"Unable to find {symlinkName} through: {nameof(WhichUtil.Which)}"); + } + + // Cleanup + File.Delete(symlink); + File.Delete(target); + Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); + + } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void WhichThrowsWhenSymlinkBroken() + { + // Arrange + using TestHostContext hc = new TestHostContext(this); + Tracing trace = hc.GetTrace(); + string oldValue = Environment.GetEnvironmentVariable(PathUtil.PathVariable); + +#if OS_WINDOWS + string newValue = oldValue + @$";{Path.GetTempPath()}"; + string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}"; + string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}.exe"; +#else + string newValue = oldValue + @$":{Path.GetTempPath()}"; + string brokenSymlinkName = $"broken-symlink-{Guid.NewGuid()}"; + string brokenSymlink = Path.GetTempPath() + $"{brokenSymlinkName}"; +#endif + + + string target = "no-such-file-cf7e351f"; + Environment.SetEnvironmentVariable(PathUtil.PathVariable, newValue); + + File.CreateSymbolicLink(brokenSymlink, target); + + // Act. + var exception = Assert.Throws(()=>WhichUtil.Which(brokenSymlinkName, require: true, trace: trace)); + + // Assert + Assert.Equal(brokenSymlinkName, exception.FileName); + + // Cleanup + File.Delete(brokenSymlink); + Environment.SetEnvironmentVariable(PathUtil.PathVariable, oldValue); + } } } From a7105697262912da22115526033c468c66546a2b Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Thu, 13 Apr 2023 09:19:00 -0400 Subject: [PATCH 121/145] Also send in conclusion for steps (#2531) --- src/Sdk/WebApi/WebApi/Contracts.cs | 11 +++++++++++ src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 21 ++++++++++++++++++++- 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/Sdk/WebApi/WebApi/Contracts.cs b/src/Sdk/WebApi/WebApi/Contracts.cs index 8da6bf236ff..45cd20fdde5 100644 --- a/src/Sdk/WebApi/WebApi/Contracts.cs +++ b/src/Sdk/WebApi/WebApi/Contracts.cs @@ -157,6 +157,8 @@ public class Step public string StartedAt; [DataMember] public string CompletedAt; + [DataMember] + public Conclusion Conclusion; } public enum Status @@ -167,6 +169,15 @@ public enum Status StatusCompleted = 6 } + public enum Conclusion + { + ConclusionUnknown = 0, + ConclusionSuccess = 2, + ConclusionFailure = 3, + ConclusionCancelled = 4, + ConclusionSkipped = 7, + } + public static class BlobStorageTypes { public static readonly string AzureBlobStorage = "BLOB_STORAGE_TYPE_AZURE"; diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index b80740f44a5..42721c0e8ed 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -329,7 +329,8 @@ private Step ConvertTimelineRecordToStep(TimelineRecord r) Name = r.Name, Status = ConvertStateToStatus(r.State.GetValueOrDefault()), StartedAt = r.StartTime?.ToString(Constants.TimestampFormat), - CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat) + CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat), + Conclusion = ConvertResultToConclusion(r.Result.GetValueOrDefault()) }; } @@ -348,6 +349,24 @@ private Status ConvertStateToStatus(TimelineRecordState s) } } + private Conclusion ConvertResultToConclusion(TaskResult r) + { + switch (r) + { + case TaskResult.Succeeded: + case TaskResult.SucceededWithIssues: + return Conclusion.ConclusionSuccess; + case TaskResult.Canceled: + return Conclusion.ConclusionCancelled; + case TaskResult.Skipped: + return Conclusion.ConclusionSkipped; + case TaskResult.Failed: + return Conclusion.ConclusionFailure; + default: + return Conclusion.ConclusionUnknown; + } + } + public async Task UpdateWorkflowStepsAsync(Guid planId, IEnumerable records, CancellationToken cancellationToken) { var timestamp = DateTime.UtcNow.ToString(Constants.TimestampFormat); From 3cf0bda259fb117f57b37f5d1a82b77a37e91458 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Thu, 13 Apr 2023 13:03:57 -0400 Subject: [PATCH 122/145] If conclusion is not set, we cannot get default value (#2535) --- src/Sdk/WebApi/WebApi/ResultsHttpClient.cs | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs index 42721c0e8ed..eb9afc96b79 100644 --- a/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/ResultsHttpClient.cs @@ -330,7 +330,7 @@ private Step ConvertTimelineRecordToStep(TimelineRecord r) Status = ConvertStateToStatus(r.State.GetValueOrDefault()), StartedAt = r.StartTime?.ToString(Constants.TimestampFormat), CompletedAt = r.FinishTime?.ToString(Constants.TimestampFormat), - Conclusion = ConvertResultToConclusion(r.Result.GetValueOrDefault()) + Conclusion = ConvertResultToConclusion(r.Result) }; } @@ -349,8 +349,13 @@ private Status ConvertStateToStatus(TimelineRecordState s) } } - private Conclusion ConvertResultToConclusion(TaskResult r) + private Conclusion ConvertResultToConclusion(TaskResult? r) { + if (!r.HasValue) + { + return Conclusion.ConclusionUnknown; + } + switch (r) { case TaskResult.Succeeded: From 5b7d827472f54261213a398d69a6832b58991bd7 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Fri, 14 Apr 2023 16:17:20 -0400 Subject: [PATCH 123/145] Handle non success raw http responses (#2541) * handle non success responses * fix format * nits Co-authored-by: Tingluo Huang --------- Co-authored-by: Tingluo Huang --- src/Runner.Common/RunServer.cs | 16 +----- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 51 +++++++++++++++++--- src/Sdk/WebApi/WebApi/BrokerHttpClient.cs | 12 ++++- src/Sdk/WebApi/WebApi/RawHttpClientBase.cs | 19 ++++++-- src/Sdk/WebApi/WebApi/RawHttpClientResult.cs | 33 +++++++++++++ 5 files changed, 104 insertions(+), 27 deletions(-) create mode 100644 src/Sdk/WebApi/WebApi/RawHttpClientResult.cs diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 32c9f063fc0..4f344c0b070 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -51,14 +51,8 @@ private void CheckConnection() public Task GetJobMessageAsync(string id, CancellationToken cancellationToken) { CheckConnection(); - var jobMessage = RetryRequest( + return RetryRequest( async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken); - if (jobMessage == null) - { - throw new TaskOrchestrationJobNotFoundException(id); - } - - return jobMessage; } public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken cancellationToken) @@ -71,14 +65,8 @@ public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Diction public Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) { CheckConnection(); - var renewJobResponse = RetryRequest( + return RetryRequest( async () => await _runServiceHttpClient.RenewJobAsync(requestUri, planId, jobId, cancellationToken), cancellationToken); - if (renewJobResponse == null) - { - throw new TaskOrchestrationJobNotFoundException(jobId.ToString()); - } - - return renewJobResponse; } } } diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 2730e66657a..e1e416d274a 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -55,7 +56,7 @@ public RunServiceHttpClient( { } - public Task GetJobMessageAsync( + public async Task GetJobMessageAsync( Uri requestUri, string messageId, CancellationToken cancellationToken = default) @@ -69,14 +70,27 @@ public Task GetJobMessageAsync( requestUri = new Uri(requestUri, "acquirejob"); var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); - return SendAsync( + var result = await SendAsync( httpMethod, requestUri: requestUri, content: requestContent, cancellationToken: cancellationToken); + + if (result.IsSuccess) + { + return result.Value; + } + + switch (result.StatusCode) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}"); + default: + throw new Exception($"Failed to get job message: {result.Error}"); + } } - public Task CompleteJobAsync( + public async Task CompleteJobAsync( Uri requestUri, Guid planId, Guid jobId, @@ -98,14 +112,26 @@ public Task CompleteJobAsync( requestUri = new Uri(requestUri, "completejob"); var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); - return SendAsync( + var response = await SendAsync( httpMethod, requestUri, content: requestContent, cancellationToken: cancellationToken); + if (response.IsSuccessStatusCode) + { + return; + } + + switch (response.StatusCode) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); + default: + throw new Exception($"Failed to complete job: {response.ReasonPhrase}"); + } } - public Task RenewJobAsync( + public async Task RenewJobAsync( Uri requestUri, Guid planId, Guid jobId, @@ -121,11 +147,24 @@ public Task RenewJobAsync( requestUri = new Uri(requestUri, "renewjob"); var requestContent = new ObjectContent(payload, new VssJsonMediaTypeFormatter(true)); - return SendAsync( + var result = await SendAsync( httpMethod, requestUri, content: requestContent, cancellationToken: cancellationToken); + + if (result.IsSuccess) + { + return result.Value; + } + + switch (result.StatusCode) + { + case HttpStatusCode.NotFound: + throw new TaskOrchestrationJobNotFoundException($"Job not found: {jobId}"); + default: + throw new Exception($"Failed to renew job: {result.Error}"); + } } } } diff --git a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs index 8f9b22e7541..e0254186672 100644 --- a/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs +++ b/src/Sdk/WebApi/WebApi/BrokerHttpClient.cs @@ -1,5 +1,6 @@ using System; using System.Collections.Generic; +using System.Net; using System.Net.Http; using System.Threading; using System.Threading.Tasks; @@ -55,7 +56,7 @@ public BrokerHttpClient( { } - public Task GetRunnerMessageAsync( + public async Task GetRunnerMessageAsync( string runnerVersion, TaskAgentStatus? status, CancellationToken cancellationToken = default @@ -74,11 +75,18 @@ public Task GetRunnerMessageAsync( queryParams.Add("runnerVersion", runnerVersion); } - return SendAsync( + var result = await SendAsync( new HttpMethod("GET"), requestUri: requestUri, queryParameters: queryParams, cancellationToken: cancellationToken); + + if (result.IsSuccess) + { + return result.Value; + } + + throw new Exception($"Failed to get job message: {result.Error}"); } } } diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs index 8054a81145a..aa602f760bb 100644 --- a/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs +++ b/src/Sdk/WebApi/WebApi/RawHttpClientBase.cs @@ -21,7 +21,7 @@ namespace Sdk.WebApi.WebApi { - public class RawHttpClientBase: IDisposable + public class RawHttpClientBase : IDisposable { protected RawHttpClientBase( Uri baseUrl, @@ -101,7 +101,7 @@ protected async Task SendAsync( } } - protected Task SendAsync( + protected Task> SendAsync( HttpMethod method, Uri requestUri, HttpContent content = null, @@ -112,7 +112,7 @@ protected Task SendAsync( return SendAsync(method, null, requestUri, content, queryParameters, userState, cancellationToken); } - protected async Task SendAsync( + protected async Task> SendAsync( HttpMethod method, IEnumerable> additionalHeaders, Uri requestUri, @@ -128,7 +128,7 @@ protected async Task SendAsync( } } - protected async Task SendAsync( + protected async Task> SendAsync( HttpRequestMessage message, Object userState = null, CancellationToken cancellationToken = default(CancellationToken)) @@ -138,7 +138,16 @@ protected async Task SendAsync( //from deadlocking... using (HttpResponseMessage response = await this.SendAsync(message, userState, cancellationToken).ConfigureAwait(false)) { - return await ReadContentAsAsync(response, cancellationToken).ConfigureAwait(false); + if (response.IsSuccessStatusCode) + { + T data = await ReadContentAsAsync(response, cancellationToken).ConfigureAwait(false); + return RawHttpClientResult.Ok(data); + } + else + { + string errorMessage = $"Error: {response.ReasonPhrase}"; + return RawHttpClientResult.Fail(errorMessage, response.StatusCode); + } } } diff --git a/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs new file mode 100644 index 00000000000..1b00d61c280 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/RawHttpClientResult.cs @@ -0,0 +1,33 @@ +using System.Net; + +namespace Sdk.WebApi.WebApi +{ + public class RawHttpClientResult + { + public bool IsSuccess { get; protected set; } + public string Error { get; protected set; } + public HttpStatusCode StatusCode { get; protected set; } + public bool IsFailure => !IsSuccess; + + protected RawHttpClientResult(bool isSuccess, string error, HttpStatusCode statusCode) + { + IsSuccess = isSuccess; + Error = error; + StatusCode = statusCode; + } + } + + public class RawHttpClientResult : RawHttpClientResult + { + public T Value { get; private set; } + + protected internal RawHttpClientResult(T value, bool isSuccess, string error, HttpStatusCode statusCode) + : base(isSuccess, error, statusCode) + { + Value = value; + } + + public static RawHttpClientResult Fail(string message, HttpStatusCode statusCode) => new RawHttpClientResult(default(T), false, message, statusCode); + public static RawHttpClientResult Ok(T value) => new RawHttpClientResult(value, true, string.Empty, HttpStatusCode.OK); + } +} From d3ca7cbda2d67f9da53a26a8a2538c4190deeb12 Mon Sep 17 00:00:00 2001 From: Maggie Spletzer Date: Tue, 18 Apr 2023 14:31:46 -0400 Subject: [PATCH 124/145] Adding curl retry for external tool downloads (#2552) * adding retry for flaky downloads * adding comment * Update src/Misc/externals.sh Co-authored-by: Tingluo Huang --------- Co-authored-by: Tingluo Huang --- src/Misc/externals.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index f9c9c95df39..4a1254f3318 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -60,7 +60,8 @@ function acquireExternalTool() { # -S Show error. With -s, make curl show errors when they occur # -L Follow redirects (H) # -o FILE Write to FILE instead of stdout - curl -fkSL -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl' + # --retry 3 Retries transient errors 3 times (timeouts, 5xx) + curl -fkSL --retry 3 -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl' # Move the partial file to the download target. mv "$partial_target" "$download_target" || checkRC 'mv' From d28dd8029ea55b49f0f4b3296383eb4c86d8b7a3 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 19 Apr 2023 10:20:00 -0400 Subject: [PATCH 125/145] Limit the time we wait for waiting websocket to connect. (#2554) --- src/Runner.Common/JobServer.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index 04c244ad8a3..c0719f61572 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -199,13 +199,15 @@ private async Task ConnectWebSocketClient(string feedStreamUrl, TimeSpan delay) { Trace.Info($"Attempting to start websocket client with delay {delay}."); await Task.Delay(delay); - await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), default(CancellationToken)); + using var connectTimeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), connectTimeoutTokenSource.Token); Trace.Info($"Successfully started websocket client."); } catch (Exception ex) { Trace.Info("Exception caught during websocket client connect, fallback of HTTP would be used now instead of websocket."); Trace.Error(ex); + this._websocketClient = null; } } From 601990b63464669d0f4ad6bf687394f2a9897110 Mon Sep 17 00:00:00 2001 From: Maggie Spletzer Date: Thu, 20 Apr 2023 17:01:11 -0400 Subject: [PATCH 126/145] Adding curl --retry-all-errors for external tool downloads (#2557) * adding retry-all-errors based on curl version >= 7.71.0 * adding logging * fixing conditional logic --- src/Misc/externals.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/Misc/externals.sh b/src/Misc/externals.sh index 4a1254f3318..5e86a75d3e6 100755 --- a/src/Misc/externals.sh +++ b/src/Misc/externals.sh @@ -55,13 +55,23 @@ function acquireExternalTool() { # Download from source to the partial file. echo "Downloading $download_source" mkdir -p "$(dirname "$download_target")" || checkRC 'mkdir' + + CURL_VERSION=$(curl --version | awk 'NR==1{print $2}') + echo "Curl version: $CURL_VERSION" + # curl -f Fail silently (no output at all) on HTTP errors (H) # -k Allow connections to SSL sites without certs (H) # -S Show error. With -s, make curl show errors when they occur # -L Follow redirects (H) # -o FILE Write to FILE instead of stdout # --retry 3 Retries transient errors 3 times (timeouts, 5xx) - curl -fkSL --retry 3 -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl' + if [[ "$(printf '%s\n' "7.71.0" "$CURL_VERSION" | sort -V | head -n1)" != "7.71.0" ]]; then + # Curl version is less than or equal to 7.71.0, skipping retry-all-errors flag + curl -fkSL --retry 3 -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl' + else + # Curl version is greater than 7.71.0, running curl with --retry-all-errors flag + curl -fkSL --retry 3 --retry-all-errors -o "$partial_target" "$download_source" 2>"${download_target}_download.log" || checkRC 'curl' + fi # Move the partial file to the download target. mv "$partial_target" "$download_target" || checkRC 'mv' From a3cc9e2aad98a44675f090889a885ae0b7edca2e Mon Sep 17 00:00:00 2001 From: Erez Testiler Date: Mon, 24 Apr 2023 12:52:28 -0400 Subject: [PATCH 127/145] Add *.ghe.localhost domains to hosted server check (#2536) --- src/Runner.Sdk/Util/UrlUtil.cs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/Runner.Sdk/Util/UrlUtil.cs b/src/Runner.Sdk/Util/UrlUtil.cs index 2c932e8c1f1..91c3a522e99 100644 --- a/src/Runner.Sdk/Util/UrlUtil.cs +++ b/src/Runner.Sdk/Util/UrlUtil.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace GitHub.Runner.Sdk { @@ -11,10 +11,11 @@ public static bool IsHostedServer(UriBuilder gitHubUrl) return false; } - return + return string.Equals(gitHubUrl.Host, "github.com", StringComparison.OrdinalIgnoreCase) || string.Equals(gitHubUrl.Host, "www.github.com", StringComparison.OrdinalIgnoreCase) || string.Equals(gitHubUrl.Host, "github.localhost", StringComparison.OrdinalIgnoreCase) || + gitHubUrl.Host.EndsWith(".ghe.localhost", StringComparison.OrdinalIgnoreCase) || gitHubUrl.Host.EndsWith(".ghe.com", StringComparison.OrdinalIgnoreCase); } From 552bdff8ab55911b36960434c0e9b7f618af3dfb Mon Sep 17 00:00:00 2001 From: John Wesley Walker III <81404201+jww3@users.noreply.github.com> Date: Wed, 26 Apr 2023 18:31:41 +0200 Subject: [PATCH 128/145] Rename AcquireJobRequest::StreamID to AcquireJobRequest::JobMessageID (#2547) * Rename AcquireJobRequest::StreamID to AcquireJobRequest::JobMessageID to match corresponding server-side update. * appeased the linter * Added unit tests to prove AcquireJobRequest serialization/deserialization is as-expected. * Distinguish unit test variations. * Incorporated PR Feedback. --- .../RSWebApi/Contracts/AcquireJobRequest.cs | 11 ++-- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 3 +- .../L0/Sdk/RSWebApi/AcquireJobRequestL0.cs | 63 +++++++++++++++++++ 3 files changed, 72 insertions(+), 5 deletions(-) create mode 100644 src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs diff --git a/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs b/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs index 022f34b42f1..895e29afa68 100644 --- a/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs +++ b/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs @@ -1,13 +1,16 @@ -using System.Collections.Generic; using System.Runtime.Serialization; -using GitHub.DistributedTask.WebApi; namespace GitHub.Actions.RunService.WebApi { [DataContract] public class AcquireJobRequest { + [DataMember(Name = "jobMessageId", EmitDefaultValue = false)] + public string JobMessageId { get; set; } + + // This field will be removed in an upcoming Runner release. + // It's left here temporarily to facilitate the transition to the new field name, JobMessageId. [DataMember(Name = "streamId", EmitDefaultValue = false)] - public string StreamID { get; set; } + public string StreamId { get; set; } } -} \ No newline at end of file +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index e1e416d274a..65259c5ba22 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -64,7 +64,8 @@ public async Task GetJobMessageAsync( HttpMethod httpMethod = new HttpMethod("POST"); var payload = new AcquireJobRequest { - StreamID = messageId + JobMessageId = messageId, + StreamId = messageId, }; requestUri = new Uri(requestUri, "acquirejob"); diff --git a/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs b/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs new file mode 100644 index 00000000000..c8883b07172 --- /dev/null +++ b/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs @@ -0,0 +1,63 @@ +using System.Collections.Generic; +using System.IO; +using System.Runtime.Serialization.Json; +using System.Text; +using Xunit; + +namespace GitHub.Actions.RunService.WebApi.Tests; + +public sealed class AcquireJobRequestL0 +{ + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void VerifySerialization() + { + var jobMessageId = "1526919030369-33"; + var request = new AcquireJobRequest + { + JobMessageId = jobMessageId, + StreamId = jobMessageId + }; + var serializer = new DataContractJsonSerializer(typeof(AcquireJobRequest)); + using var stream = new MemoryStream(); + serializer.WriteObject(stream, request); + + stream.Position = 0; + using var reader = new StreamReader(stream, Encoding.UTF8); + string json = reader.ReadToEnd(); + string expected = DoubleQuotify(string.Format("{{'jobMessageId':'{0}','streamId':'{0}'}}", request.JobMessageId)); + Assert.Equal(expected, json); + + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Common")] + public void VerifyDeserialization() + { + var serializer = new DataContractJsonSerializer(typeof(AcquireJobRequest)); + var variations = new Dictionary() + { + ["{'streamId': 'legacy', 'jobMessageId': 'new-1'}"] = "new-1", + ["{'jobMessageId': 'new-2', 'streamId': 'legacy'}"] = "new-2", + ["{'jobMessageId': 'new-3'}"] = "new-3" + }; + + foreach (var (source, expected) in variations) + { + using var stream = new MemoryStream(); + stream.Write(Encoding.UTF8.GetBytes(DoubleQuotify(source))); + stream.Position = 0; + var recoveredRecord = serializer.ReadObject(stream) as AcquireJobRequest; + Assert.NotNull(recoveredRecord); + Assert.Equal(expected, recoveredRecord.JobMessageId); + } + } + + private static string DoubleQuotify(string text) + { + return text.Replace('\'', '"'); + } +} From 9b8f24445ca688139818e857b8f99e40f6672a96 Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Wed, 26 Apr 2023 12:51:56 -0400 Subject: [PATCH 129/145] Bypass job server when run service is enabled (#2516) * Only upload to Results with new job message type * No need to have separate websocketFeedServer * Linting fix * Update src/Runner.Common/JobServerQueue.cs Co-authored-by: Tingluo Huang * add connection timeout * Consolidate initializing webclient to result client * Retry websocket delivery for console logs * Linter fix * Do not give up for good, reconnect again in 10 minutes * Has to reset delivered * Only first time retry 3 times to connect to websocket --------- Co-authored-by: Tingluo Huang --- src/Runner.Common/JobServerQueue.cs | 78 +++++++++---- src/Runner.Common/ResultsServer.cs | 170 +++++++++++++++++++++++++++- src/Runner.Worker/JobRunner.cs | 3 + 3 files changed, 224 insertions(+), 27 deletions(-) diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 20ad27a89fb..57ba3eb23f6 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -17,7 +17,7 @@ public interface IJobServerQueue : IRunnerService, IThrottlingReporter TaskCompletionSource JobRecordUpdated { get; } event EventHandler JobServerQueueThrottling; Task ShutdownAsync(); - void Start(Pipelines.AgentJobRequestMessage jobRequest); + void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultServiceOnly = false); void QueueWebConsoleLine(Guid stepRecordId, string line, long? lineNumber = null); void QueueFileUpload(Guid timelineId, Guid timelineRecordId, string type, string name, string path, bool deleteSource); void QueueResultsUpload(Guid timelineRecordId, string name, string path, string type, bool deleteSource, bool finalize, bool firstBlock, long totalLines); @@ -70,6 +70,7 @@ public sealed class JobServerQueue : RunnerService, IJobServerQueue private readonly TaskCompletionSource _jobCompletionSource = new(); private readonly TaskCompletionSource _jobRecordUpdated = new(); private bool _queueInProcess = false; + private bool _resultsServiceOnly = false; public TaskCompletionSource JobRecordUpdated => _jobRecordUpdated; @@ -95,13 +96,17 @@ public override void Initialize(IHostContext hostContext) _resultsServer = hostContext.GetService(); } - public void Start(Pipelines.AgentJobRequestMessage jobRequest) + public void Start(Pipelines.AgentJobRequestMessage jobRequest, bool resultServiceOnly = false) { Trace.Entering(); + _resultsServiceOnly = resultServiceOnly; var serviceEndPoint = jobRequest.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - _jobServer.InitializeWebsocketClient(serviceEndPoint); + if (!resultServiceOnly) + { + _jobServer.InitializeWebsocketClient(serviceEndPoint); + } // This code is usually wrapped by an instance of IExecutionContext which isn't available here. jobRequest.Variables.TryGetValue("system.github.results_endpoint", out VariableValue resultsEndpointVariable); @@ -112,8 +117,16 @@ public void Start(Pipelines.AgentJobRequestMessage jobRequest) !string.IsNullOrEmpty(accessToken) && !string.IsNullOrEmpty(resultsReceiverEndpoint)) { + string liveConsoleFeedUrl = null; Trace.Info("Initializing results client"); - _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), accessToken); + if (resultServiceOnly + && serviceEndPoint.Data.TryGetValue("FeedStreamUrl", out var feedStreamUrl) + && !string.IsNullOrEmpty(feedStreamUrl)) + { + liveConsoleFeedUrl = feedStreamUrl; + } + + _resultsServer.InitializeResultsClient(new Uri(resultsReceiverEndpoint), liveConsoleFeedUrl, accessToken); _resultsClientInitiated = true; } @@ -194,6 +207,9 @@ public async Task ShutdownAsync() Trace.Info($"Disposing job server ..."); await _jobServer.DisposeAsync(); + Trace.Info($"Disposing results server ..."); + await _resultsServer.DisposeAsync(); + Trace.Info("All queue process tasks have been stopped, and all queues are drained."); } @@ -372,7 +388,14 @@ private async Task ProcessWebConsoleLinesQueueAsync(bool runOnce = false) // Give at most 60s for each request. using (var timeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(60))) { - await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token); + if (_resultsServiceOnly) + { + await _resultsServer.AppendLiveConsoleFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token); + } + else + { + await _jobServer.AppendTimelineRecordFeedAsync(_scopeIdentifier, _hubName, _planId, _jobTimelineId, _jobTimelineRecordId, stepRecordId, batch.Select(logLine => logLine.Line).ToList(), batch[0].LineNumber, timeoutTokenSource.Token); + } } if (_firstConsoleOutputs) @@ -599,7 +622,7 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) foreach (var detailTimeline in update.PendingRecords.Where(r => r.Details != null)) { - if (!_allTimelines.Contains(detailTimeline.Details.Id)) + if (!_resultsServiceOnly && !_allTimelines.Contains(detailTimeline.Details.Id)) { try { @@ -621,7 +644,11 @@ private async Task ProcessTimelinesUpdateQueueAsync(bool runOnce = false) try { - await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken)); + if (!_resultsServiceOnly) + { + await _jobServer.UpdateTimelineRecordsAsync(_scopeIdentifier, _hubName, _planId, update.TimelineId, update.PendingRecords, default(CancellationToken)); + } + try { if (_resultsClientInitiated) @@ -797,27 +824,30 @@ private async Task UploadFile(UploadFileInfo file) bool uploadSucceed = false; try { - if (String.Equals(file.Type, CoreAttachmentType.Log, StringComparison.OrdinalIgnoreCase)) + if (!_resultsServiceOnly) { - // Create the log - var taskLog = await _jobServer.CreateLogAsync(_scopeIdentifier, _hubName, _planId, new TaskLog(String.Format(@"logs\{0:D}", file.TimelineRecordId)), default(CancellationToken)); - - // Upload the contents - using (FileStream fs = File.Open(file.Path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + if (String.Equals(file.Type, CoreAttachmentType.Log, StringComparison.OrdinalIgnoreCase)) { - var logUploaded = await _jobServer.AppendLogContentAsync(_scopeIdentifier, _hubName, _planId, taskLog.Id, fs, default(CancellationToken)); - } + // Create the log + var taskLog = await _jobServer.CreateLogAsync(_scopeIdentifier, _hubName, _planId, new TaskLog(String.Format(@"logs\{0:D}", file.TimelineRecordId)), default(CancellationToken)); - // Create a new record and only set the Log field - var attachmentUpdataRecord = new TimelineRecord() { Id = file.TimelineRecordId, Log = taskLog }; - QueueTimelineRecordUpdate(file.TimelineId, attachmentUpdataRecord); - } - else - { - // Create attachment - using (FileStream fs = File.Open(file.Path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + // Upload the contents + using (FileStream fs = File.Open(file.Path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + var logUploaded = await _jobServer.AppendLogContentAsync(_scopeIdentifier, _hubName, _planId, taskLog.Id, fs, default(CancellationToken)); + } + + // Create a new record and only set the Log field + var attachmentUpdataRecord = new TimelineRecord() { Id = file.TimelineRecordId, Log = taskLog }; + QueueTimelineRecordUpdate(file.TimelineId, attachmentUpdataRecord); + } + else { - var result = await _jobServer.CreateAttachmentAsync(_scopeIdentifier, _hubName, _planId, file.TimelineId, file.TimelineRecordId, file.Type, file.Name, fs, default(CancellationToken)); + // Create attachment + using (FileStream fs = File.Open(file.Path, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + var result = await _jobServer.CreateAttachmentAsync(_scopeIdentifier, _hubName, _planId, file.TimelineId, file.TimelineRecordId, file.Type, file.Name, fs, default(CancellationToken)); + } } } diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs index 4d24d55c86a..b36a5efbd9e 100644 --- a/src/Runner.Common/ResultsServer.cs +++ b/src/Runner.Common/ResultsServer.cs @@ -1,17 +1,26 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net.Http.Headers; +using System.Net.WebSockets; +using System.Security; +using System.Text; using System.Threading; using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; +using GitHub.Runner.Sdk; +using GitHub.Services.Common; using GitHub.Services.Results.Client; +using GitHub.Services.WebApi.Utilities.Internal; namespace GitHub.Runner.Common { [ServiceLocator(Default = typeof(ResultServer))] - public interface IResultsServer : IRunnerService + public interface IResultsServer : IRunnerService, IAsyncDisposable { - void InitializeResultsClient(Uri uri, string token); + void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token); + + Task AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken); // logging and console Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file, @@ -31,10 +40,26 @@ public sealed class ResultServer : RunnerService, IResultsServer { private ResultsHttpClient _resultsClient; - public void InitializeResultsClient(Uri uri, string token) + private ClientWebSocket _websocketClient; + private DateTime? _lastConnectionFailure; + + private static readonly TimeSpan MinDelayForWebsocketReconnect = TimeSpan.FromMilliseconds(100); + private static readonly TimeSpan MaxDelayForWebsocketReconnect = TimeSpan.FromMilliseconds(500); + + private Task _websocketConnectTask; + private String _liveConsoleFeedUrl; + private string _token; + + public void InitializeResultsClient(Uri uri, string liveConsoleFeedUrl, string token) { var httpMessageHandler = HostContext.CreateHttpClientHandler(); this._resultsClient = new ResultsHttpClient(uri, httpMessageHandler, token, disposeHandler: true); + _token = token; + if (!string.IsNullOrEmpty(liveConsoleFeedUrl)) + { + _liveConsoleFeedUrl = liveConsoleFeedUrl; + InitializeWebsocketClient(liveConsoleFeedUrl, token, TimeSpan.Zero, retryConnection: true); + } } public Task CreateResultsStepSummaryAsync(string planId, string jobId, Guid stepId, string file, @@ -94,5 +119,144 @@ public Task UpdateResultsWorkflowStepsAsync(Guid scopeIdentifier, string hubName throw new InvalidOperationException("Results client is not initialized."); } + + public ValueTask DisposeAsync() + { + CloseWebSocket(WebSocketCloseStatus.NormalClosure, CancellationToken.None); + + GC.SuppressFinalize(this); + + return ValueTask.CompletedTask; + } + + private void InitializeWebsocketClient(string liveConsoleFeedUrl, string accessToken, TimeSpan delay, bool retryConnection = false) + { + if (!string.IsNullOrEmpty(accessToken)) + { + Trace.Info($"No access token from server"); + return; + } + + if (!string.IsNullOrEmpty(liveConsoleFeedUrl)) + { + Trace.Info($"No live console feed url from server"); + return; + } + + Trace.Info($"Creating websocket client ..." + liveConsoleFeedUrl); + this._websocketClient = new ClientWebSocket(); + this._websocketClient.Options.SetRequestHeader("Authorization", $"Bearer {accessToken}"); + var userAgentValues = new List(); + userAgentValues.AddRange(UserAgentUtility.GetDefaultRestUserAgent()); + userAgentValues.AddRange(HostContext.UserAgents); + this._websocketClient.Options.SetRequestHeader("User-Agent", string.Join(" ", userAgentValues.Select(x => x.ToString()))); + + // during initialization, retry upto 3 times to setup connection + this._websocketConnectTask = ConnectWebSocketClient(liveConsoleFeedUrl, delay, retryConnection); + } + + private async Task ConnectWebSocketClient(string feedStreamUrl, TimeSpan delay, bool retryConnection = false) + { + bool connected = false; + int retries = 0; + + do + { + try + { + Trace.Info($"Attempting to start websocket client with delay {delay}."); + await Task.Delay(delay); + using var connectTimeoutTokenSource = new CancellationTokenSource(TimeSpan.FromSeconds(30)); + await this._websocketClient.ConnectAsync(new Uri(feedStreamUrl), connectTimeoutTokenSource.Token); + Trace.Info($"Successfully started websocket client."); + connected = true; + } + catch (Exception ex) + { + Trace.Info("Exception caught during websocket client connect, retry connection."); + Trace.Error(ex); + retries++; + this._websocketClient = null; + _lastConnectionFailure = DateTime.Now; + } + } while (retryConnection && !connected && retries < 3); + } + + public async Task AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string hubName, Guid planId, Guid timelineId, Guid timelineRecordId, Guid stepId, IList lines, long? startLine, CancellationToken cancellationToken) + { + if (_websocketConnectTask != null) + { + await _websocketConnectTask; + } + + bool delivered = false; + int retries = 0; + + // "_websocketClient != null" implies either: We have a successful connection OR we have to attempt sending again and then reconnect + // ...in other words, if websocket client is null, we will skip sending to websocket + if (_websocketClient != null) + { + var linesWrapper = startLine.HasValue + ? new TimelineRecordFeedLinesWrapper(stepId, lines, startLine.Value) + : new TimelineRecordFeedLinesWrapper(stepId, lines); + var jsonData = StringUtil.ConvertToJson(linesWrapper); + var jsonDataBytes = Encoding.UTF8.GetBytes(jsonData); + // break the message into chunks of 1024 bytes + for (var i = 0; i < jsonDataBytes.Length; i += 1 * 1024) + { + var lastChunk = i + (1 * 1024) >= jsonDataBytes.Length; + var chunk = new ArraySegment(jsonDataBytes, i, Math.Min(1 * 1024, jsonDataBytes.Length - i)); + + delivered = false; + while (!delivered && retries < 3) + { + try + { + if (_websocketClient != null) + { + await _websocketClient.SendAsync(chunk, WebSocketMessageType.Text, endOfMessage: lastChunk, cancellationToken); + delivered = true; + } + } + catch (Exception ex) + { + var delay = BackoffTimerHelper.GetRandomBackoff(MinDelayForWebsocketReconnect, MaxDelayForWebsocketReconnect); + Trace.Info($"Websocket is not open, let's attempt to connect back again with random backoff {delay} ms."); + Trace.Error(ex); + retries++; + InitializeWebsocketClient(_liveConsoleFeedUrl, _token, delay); + } + } + } + } + + if (!delivered) + { + // Giving up for now, so next invocation of this method won't attempt to reconnect + _websocketClient = null; + + // however if 10 minutes have already passed, let's try reestablish connection again + if (_lastConnectionFailure.HasValue && DateTime.Now > _lastConnectionFailure.Value.AddMinutes(10)) + { + // Some minutes passed since we retried last time, try connection again + InitializeWebsocketClient(_liveConsoleFeedUrl, _token, TimeSpan.Zero); + } + } + + return delivered; + } + + private void CloseWebSocket(WebSocketCloseStatus closeStatus, CancellationToken cancellationToken) + { + try + { + _websocketClient?.CloseOutputAsync(closeStatus, "Closing websocket", cancellationToken); + } + catch (Exception websocketEx) + { + // In some cases this might be okay since the websocket might be open yet, so just close and don't trace exceptions + Trace.Info($"Failed to close websocket gracefully {websocketEx.GetType().Name}"); + } + } } } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index f63fbf71776..3911bd340f9 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -49,6 +49,9 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat VssCredentials jobServerCredential = VssUtil.GetVssCredential(systemConnection); await runServer.ConnectAsync(systemConnection.Url, jobServerCredential); server = runServer; + + _jobServerQueue = HostContext.GetService(); + _jobServerQueue.Start(message, resultServiceOnly: true); } else { From c28320421fd9b9dcbb01f36950c414510b0781b0 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 26 Apr 2023 15:15:19 -0400 Subject: [PATCH 130/145] Add orchestrationId to useragent for better correlation. (#2568) * Add orchestrationId to useragent for better correlation. * . --- src/Runner.Common/Constants.cs | 3 ++- src/Runner.Worker/JobRunner.cs | 8 ++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index 86599ada0fa..ceddd42116f 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace GitHub.Runner.Common { @@ -261,6 +261,7 @@ public static class System public static readonly string AccessToken = "system.accessToken"; public static readonly string Culture = "system.culture"; public static readonly string PhaseDisplayName = "system.phaseDisplayName"; + public static readonly string OrchestrationId = "system.orchestrationId"; } } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 3911bd340f9..77a93bf9d34 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -3,6 +3,7 @@ using System.IO; using System.Linq; using System.Net.Http; +using System.Net.Http.Headers; using System.Text; using System.Threading; using System.Threading.Tasks; @@ -42,6 +43,13 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat DateTime jobStartTimeUtc = DateTime.UtcNow; IRunnerService server = null; + // add orchestration id to useragent for better correlation. + if (message.Variables.TryGetValue(Constants.Variables.System.OrchestrationId, out VariableValue orchestrationId) && + !string.IsNullOrEmpty(orchestrationId.Value)) + { + HostContext.UserAgents.Add(new ProductInfoHeaderValue("OrchestrationId", orchestrationId.Value)); + } + ServiceEndpoint systemConnection = message.Resources.Endpoints.Single(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); if (MessageUtil.IsRunServiceJob(message.MessageType)) { From 4c3438c2ca90390fa640fb9952912a2864184fb0 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Wed, 26 Apr 2023 15:48:13 -0400 Subject: [PATCH 131/145] Prepare 2.304.0 runner release. (#2569) --- releaseNote.md | 19 ++++++++++--------- src/runnerversion | 2 +- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/releaseNote.md b/releaseNote.md index 7ed7306633e..1c046efdf03 100644 --- a/releaseNote.md +++ b/releaseNote.md @@ -1,17 +1,18 @@ ## Features -- Support matrix context in output keys (#2477) -- Add update certificates to `./run.sh` if `RUNNER_UPDATE_CA_CERTS` env is set (#2471) -- Bypass all proxies for all hosts if `no_proxy='*'` is set (#2395) -- Change runner image to make user/folder align with `ubuntu-latest` hosted runner. (#2469) +- Runner changes for communication with Results service (#2510, #2531, #2535, #2516) +- Add `*.ghe.localhost` domains to hosted server check (#2536) +- Add `OrchestrationId` to user-agent for better telemetry correlation. (#2568) ## Bugs -- Exit on runner version deprecation error (#2299) -- Runner service exit after consecutive re-try exits (#2426) +- Fix JIT configurations on Windows (#2497) +- Guard against NullReference while creating HostContext (#2343) +- Handles broken symlink in `Which` (#2150, #2196) +- Adding curl retry for external tool downloads (#2552, #2557) +- Limit the time we wait for waiting websocket to connect. (#2554) ## Misc -- Replace deprecated command with environment file (#2429) -- Make requests to `Run` service to renew job request (#2461) -- Add job/step log upload to Result service (#2447, #2439) +- Bump container hooks version to 0.3.1 in runner image (#2496) +- Runner changes to communicate with vNext services (#2487, #2500, #2505, #2541, #2547) _Note: Actions Runner follows a progressive release policy, so the latest release might not be available to your enterprise, organization, or repository yet. To confirm which version of the Actions Runner you should expect, please view the download instructions for your enterprise, organization, or repository. diff --git a/src/runnerversion b/src/runnerversion index f67bb997a56..971440973d7 100644 --- a/src/runnerversion +++ b/src/runnerversion @@ -1 +1 @@ -2.303.0 +2.304.0 From 737927e5eaedfa755f03cc69e4aaf54fe86d483f Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Thu, 27 Apr 2023 13:59:12 -0400 Subject: [PATCH 132/145] handle conflict errors from run service (#2570) * handle conflict errors from run service * nit: formatting * fix formatting --- src/Runner.Common/RunServer.cs | 3 ++- src/Runner.Common/RunnerService.cs | 9 +++++---- src/Runner.Listener/Runner.cs | 12 +++++++++++- src/Sdk/DTWebApi/WebApi/Exceptions.cs | 22 +++++++++++++++++++++- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 2 ++ 5 files changed, 41 insertions(+), 7 deletions(-) diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 4f344c0b070..3cc53f3d894 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -52,7 +52,8 @@ public Task GetJobMessageAsync(string id, CancellationTo { CheckConnection(); return RetryRequest( - async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken); + async () => await _runServiceHttpClient.GetJobMessageAsync(requestUri, id, cancellationToken), cancellationToken, + shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException); } public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken cancellationToken) diff --git a/src/Runner.Common/RunnerService.cs b/src/Runner.Common/RunnerService.cs index a0cb96a0ecf..4694af1ce2e 100644 --- a/src/Runner.Common/RunnerService.cs +++ b/src/Runner.Common/RunnerService.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Threading; using System.Threading.Tasks; using GitHub.Runner.Sdk; @@ -80,10 +80,11 @@ async Task wrappedFunc() } await RetryRequest(wrappedFunc, cancellationToken, maxRetryAttemptsCount); } - + protected async Task RetryRequest(Func> func, CancellationToken cancellationToken, - int maxRetryAttemptsCount = 5 + int maxRetryAttemptsCount = 5, + Func shouldRetry = null ) { var retryCount = 0; @@ -96,7 +97,7 @@ protected async Task RetryRequest(Func> func, return await func(); } // TODO: Add handling of non-retriable exceptions: https://github.com/github/actions-broker/issues/122 - catch (Exception ex) when (retryCount < maxRetryAttemptsCount) + catch (Exception ex) when (retryCount < maxRetryAttemptsCount && (shouldRetry == null || shouldRetry(ex))) { Trace.Error("Catch exception during request"); Trace.Error(ex); diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index 865c596a91e..e71ba9cb5b0 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -549,7 +549,17 @@ private async Task RunAsync(RunnerSettings settings, bool runOnce = false) { var runServer = HostContext.CreateService(); await runServer.ConnectAsync(new Uri(messageRef.RunServiceUrl), creds); - jobRequestMessage = await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, messageQueueLoopTokenSource.Token); + try + { + jobRequestMessage = + await runServer.GetJobMessageAsync(messageRef.RunnerRequestId, + messageQueueLoopTokenSource.Token); + } + catch (TaskOrchestrationJobAlreadyAcquiredException) + { + Trace.Info("Job is already acquired, skip this message."); + continue; + } } jobDispatcher.Run(jobRequestMessage, runOnce); diff --git a/src/Sdk/DTWebApi/WebApi/Exceptions.cs b/src/Sdk/DTWebApi/WebApi/Exceptions.cs index 3690bbe6bb0..aed702b7e5c 100644 --- a/src/Sdk/DTWebApi/WebApi/Exceptions.cs +++ b/src/Sdk/DTWebApi/WebApi/Exceptions.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Runtime.Serialization; using GitHub.Services.Common; @@ -1519,6 +1519,26 @@ private TaskOrchestrationJobNotFoundException(SerializationInfo info, StreamingC } } + [Serializable] + [ExceptionMapping("0.0", "3.0", "TaskOrchestrationJobAlreadyAcquiredException", "GitHub.DistributedTask.WebApi.TaskOrchestrationJobAlreadyAcquiredException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")] + public sealed class TaskOrchestrationJobAlreadyAcquiredException : DistributedTaskException + { + public TaskOrchestrationJobAlreadyAcquiredException(String message) + : base(message) + { + } + + public TaskOrchestrationJobAlreadyAcquiredException(String message, Exception innerException) + : base(message, innerException) + { + } + + private TaskOrchestrationJobAlreadyAcquiredException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } + [Serializable] [ExceptionMapping("0.0", "3.0", "TaskOrchestrationPlanSecurityException", "GitHub.DistributedTask.WebApi.TaskOrchestrationPlanSecurityException, GitHub.DistributedTask.WebApi, Version=14.0.0.0, Culture=neutral, PublicKeyToken=b03f5f7f11d50a3a")] public sealed class TaskOrchestrationPlanSecurityException : DistributedTaskException diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 65259c5ba22..8ddd1672b04 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -86,6 +86,8 @@ public async Task GetJobMessageAsync( { case HttpStatusCode.NotFound: throw new TaskOrchestrationJobNotFoundException($"Job message not found: {messageId}"); + case HttpStatusCode.Conflict: + throw new TaskOrchestrationJobAlreadyAcquiredException($"Job message already acquired: {messageId}"); default: throw new Exception($"Failed to get job message: {result.Error}"); } From 70d90a05ae5efb280387bfc381c4f8c69fd88085 Mon Sep 17 00:00:00 2001 From: Per Lundberg Date: Thu, 27 Apr 2023 23:19:56 +0300 Subject: [PATCH 133/145] contribute.md: Fix link to style guidelines (#2560) --- docs/contribute.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/contribute.md b/docs/contribute.md index a350b846b19..56301b1d25b 100644 --- a/docs/contribute.md +++ b/docs/contribute.md @@ -157,7 +157,7 @@ cat (Runner/Worker)_TIMESTAMP.log # view your log file ## Styling We use the .NET Foundation and CoreCLR style guidelines [located here]( -https://github.com/dotnet/corefx/blob/master/Documentation/coding-guidelines/coding-style.md) +https://github.com/dotnet/runtime/blob/main/docs/coding-guidelines/coding-style.md) ### Format C# Code @@ -165,4 +165,4 @@ To format both staged and unstaged .cs files ``` cd ./src ./dev.(cmd|sh) format -``` \ No newline at end of file +``` From 0db9b07a8ba8a4a8ec5f60f3c7317a68b71f91ff Mon Sep 17 00:00:00 2001 From: Yang Cao Date: Fri, 28 Apr 2023 16:56:10 -0400 Subject: [PATCH 134/145] Fix null guard bug (#2576) --- src/Runner.Common/ResultsServer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs index b36a5efbd9e..905d717086b 100644 --- a/src/Runner.Common/ResultsServer.cs +++ b/src/Runner.Common/ResultsServer.cs @@ -131,13 +131,13 @@ public ValueTask DisposeAsync() private void InitializeWebsocketClient(string liveConsoleFeedUrl, string accessToken, TimeSpan delay, bool retryConnection = false) { - if (!string.IsNullOrEmpty(accessToken)) + if (string.IsNullOrEmpty(accessToken)) { Trace.Info($"No access token from server"); return; } - if (!string.IsNullOrEmpty(liveConsoleFeedUrl)) + if (string.IsNullOrEmpty(liveConsoleFeedUrl)) { Trace.Info($"No live console feed url from server"); return; From 7b629c880490a9cf6436aa0b9c7711a24eff6139 Mon Sep 17 00:00:00 2001 From: Yashwanth Anantharaju Date: Mon, 1 May 2023 08:33:03 -0400 Subject: [PATCH 135/145] send annotations to run-service (#2574) * send annotations to run-service * skip message deletion * actually don't skip deletion * enum as numbers * fix enum * linting * remove unncessary file * feedback --- src/Runner.Common/RunServer.cs | 20 +++- src/Runner.Listener/JobDispatcher.cs | 77 ++++++++-------- src/Runner.Worker/ExecutionContext.cs | 32 +++++-- src/Runner.Worker/GlobalContext.cs | 4 +- src/Runner.Worker/JobRunner.cs | 2 +- src/Sdk/RSWebApi/Contracts/Annotation.cs | 35 +++++++ src/Sdk/RSWebApi/Contracts/AnnotationLevel.cs | 20 ++++ .../RSWebApi/Contracts/CompleteJobRequest.cs | 16 ++-- src/Sdk/RSWebApi/Contracts/IssueExtensions.cs | 91 +++++++++++++++++++ src/Sdk/RSWebApi/Contracts/IssueKeys.cs | 13 +++ src/Sdk/RSWebApi/Contracts/StepResult.cs | 7 +- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 4 +- src/Test/L0/Sdk/RSWebApi/AnnotationsL0.cs | 70 ++++++++++++++ 13 files changed, 332 insertions(+), 59 deletions(-) create mode 100644 src/Sdk/RSWebApi/Contracts/Annotation.cs create mode 100644 src/Sdk/RSWebApi/Contracts/AnnotationLevel.cs create mode 100644 src/Sdk/RSWebApi/Contracts/IssueExtensions.cs create mode 100644 src/Sdk/RSWebApi/Contracts/IssueKeys.cs create mode 100644 src/Test/L0/Sdk/RSWebApi/AnnotationsL0.cs diff --git a/src/Runner.Common/RunServer.cs b/src/Runner.Common/RunServer.cs index 3cc53f3d894..47cd62b63c3 100644 --- a/src/Runner.Common/RunServer.cs +++ b/src/Runner.Common/RunServer.cs @@ -19,7 +19,14 @@ public interface IRunServer : IRunnerService Task GetJobMessageAsync(string id, CancellationToken token); - Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken token); + Task CompleteJobAsync( + Guid planId, + Guid jobId, + TaskResult result, + Dictionary outputs, + IList stepResults, + IList jobAnnotations, + CancellationToken token); Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken token); } @@ -56,11 +63,18 @@ public Task GetJobMessageAsync(string id, CancellationTo shouldRetry: ex => ex is not TaskOrchestrationJobAlreadyAcquiredException); } - public Task CompleteJobAsync(Guid planId, Guid jobId, TaskResult result, Dictionary outputs, IList stepResults, CancellationToken cancellationToken) + public Task CompleteJobAsync( + Guid planId, + Guid jobId, + TaskResult result, + Dictionary outputs, + IList stepResults, + IList jobAnnotations, + CancellationToken cancellationToken) { CheckConnection(); return RetryRequest( - async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, cancellationToken), cancellationToken); + async () => await _runServiceHttpClient.CompleteJobAsync(requestUri, planId, jobId, result, outputs, stepResults, jobAnnotations, cancellationToken), cancellationToken); } public Task RenewJobAsync(Guid planId, Guid jobId, CancellationToken cancellationToken) diff --git a/src/Runner.Listener/JobDispatcher.cs b/src/Runner.Listener/JobDispatcher.cs index 53380b9b220..9402cd2d7c1 100644 --- a/src/Runner.Listener/JobDispatcher.cs +++ b/src/Runner.Listener/JobDispatcher.cs @@ -15,6 +15,7 @@ using GitHub.Services.Common; using GitHub.Services.WebApi; using GitHub.Services.WebApi.Jwt; +using Sdk.RSWebApi.Contracts; using Pipelines = GitHub.DistributedTask.Pipelines; namespace GitHub.Runner.Listener @@ -372,6 +373,8 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc TaskCompletionSource firstJobRequestRenewed = new(); var notification = HostContext.GetService(); + var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); + // lock renew cancellation token. using (var lockRenewalTokenSource = new CancellationTokenSource()) using (var workerProcessCancelTokenSource = new CancellationTokenSource()) @@ -379,8 +382,6 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc long requestId = message.RequestId; Guid lockToken = Guid.Empty; // lockToken has never been used, keep this here of compat - var systemConnection = message.Resources.Endpoints.SingleOrDefault(x => string.Equals(x.Name, WellKnownServiceEndpointNames.SystemVssConnection, StringComparison.OrdinalIgnoreCase)); - // start renew job request Trace.Info($"Start renew job request {requestId} for job {message.JobId}."); Task renewJobRequest = RenewJobRequestAsync(message, systemConnection, _poolId, requestId, lockToken, orchestrationId, firstJobRequestRenewed, lockRenewalTokenSource.Token); @@ -405,7 +406,7 @@ private async Task RunAsync(Pipelines.AgentJobRequestMessage message, string orc await renewJobRequest; // complete job request with result Cancelled - await CompleteJobRequestAsync(_poolId, message, lockToken, TaskResult.Canceled); + await CompleteJobRequestAsync(_poolId, message, systemConnection, lockToken, TaskResult.Canceled); return; } @@ -544,7 +545,6 @@ await processChannel.SendAsync( detailInfo = string.Join(Environment.NewLine, workerOutput); Trace.Info($"Return code {returnCode} indicate worker encounter an unhandled exception or app crash, attach worker stdout/stderr to JobRequest result."); - var jobServer = await InitializeJobServerAsync(systemConnection); await LogWorkerProcessUnhandledException(jobServer, message, detailInfo); @@ -552,7 +552,7 @@ await processChannel.SendAsync( if (detailInfo.Contains(typeof(System.IO.IOException).ToString(), StringComparison.OrdinalIgnoreCase)) { Trace.Info($"Finish job with result 'Failed' due to IOException."); - await ForceFailJob(jobServer, message); + await ForceFailJob(jobServer, message, detailInfo); } } @@ -567,7 +567,7 @@ await processChannel.SendAsync( await renewJobRequest; // complete job request - await CompleteJobRequestAsync(_poolId, message, lockToken, result, detailInfo); + await CompleteJobRequestAsync(_poolId, message, systemConnection, lockToken, result, detailInfo); // print out unhandled exception happened in worker after we complete job request. // when we run out of disk space, report back to server has higher priority. @@ -664,7 +664,7 @@ await processChannel.SendAsync( await renewJobRequest; // complete job request - await CompleteJobRequestAsync(_poolId, message, lockToken, resultOnAbandonOrCancel); + await CompleteJobRequestAsync(_poolId, message, systemConnection, lockToken, resultOnAbandonOrCancel); } finally { @@ -1065,7 +1065,7 @@ private async Task TryUploadUnfinishedLogs(Pipelines.AgentJobRequestMessage mess } } - private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, Guid lockToken, TaskResult result, string detailInfo = null) + private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequestMessage message, ServiceEndpoint systemConnection, Guid lockToken, TaskResult result, string detailInfo = null) { Trace.Entering(); @@ -1077,7 +1077,23 @@ private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequest if (this._isRunServiceJob) { - Trace.Verbose($"Skip FinishAgentRequest call from Listener because MessageType is {message.MessageType}"); + var runServer = await GetRunServerAsync(systemConnection); + var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo }; + var unhandledAnnotation = unhandledExceptionIssue.ToAnnotation(); + var jobAnnotations = new List(); + if (unhandledAnnotation.HasValue) + { + jobAnnotations.Add(unhandledAnnotation.Value); + } + try + { + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, CancellationToken.None); + } + catch (Exception ex) + { + Trace.Error("Fail to raise job completion back to service."); + Trace.Error(ex); + } return; } @@ -1117,7 +1133,7 @@ private async Task CompleteJobRequestAsync(int poolId, Pipelines.AgentJobRequest } // log an error issue to job level timeline record - private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string errorMessage) + private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo) { if (server is IJobServer jobServer) { @@ -1129,34 +1145,11 @@ private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pip TimelineRecord jobRecord = timeline.Records.FirstOrDefault(x => x.Id == message.JobId && x.RecordType == "Job"); ArgUtil.NotNull(jobRecord, nameof(jobRecord)); - try - { - if (!string.IsNullOrEmpty(errorMessage) && - message.Variables.TryGetValue("DistributedTask.EnableRunnerIPCDebug", out var enableRunnerIPCDebug) && - StringUtil.ConvertToBoolean(enableRunnerIPCDebug.Value)) - { - // the trace should be best effort and not affect any job result - var match = _invalidJsonRegex.Match(errorMessage); - if (match.Success && - match.Groups.Count == 2) - { - var jsonPosition = int.Parse(match.Groups[1].Value); - var serializedJobMessage = JsonUtility.ToString(message); - var originalJson = serializedJobMessage.Substring(jsonPosition - 10, 20); - errorMessage = $"Runner sent Json at position '{jsonPosition}': {originalJson} ({Convert.ToBase64String(Encoding.UTF8.GetBytes(originalJson))})\n{errorMessage}"; - } - } - } - catch (Exception ex) - { - Trace.Error(ex); - errorMessage = $"Fail to check json IPC error: {ex.Message}\n{errorMessage}"; - } - - var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = errorMessage }; + var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo }; unhandledExceptionIssue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.WorkerCrash; jobRecord.ErrorCount++; jobRecord.Issues.Add(unhandledExceptionIssue); + await jobServer.UpdateTimelineRecordsAsync(message.Plan.ScopeIdentifier, message.Plan.PlanType, message.Plan.PlanId, message.Timeline.Id, new TimelineRecord[] { jobRecord }, CancellationToken.None); } catch (Exception ex) @@ -1167,13 +1160,13 @@ private async Task LogWorkerProcessUnhandledException(IRunnerService server, Pip } else { - Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", errorMessage); + Trace.Info("Job server does not support handling unhandled exception yet, error message: {0}", detailInfo); return; } } // raise job completed event to fail the job. - private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message) + private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequestMessage message, string detailInfo) { if (server is IJobServer jobServer) { @@ -1192,7 +1185,15 @@ private async Task ForceFailJob(IRunnerService server, Pipelines.AgentJobRequest { try { - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, CancellationToken.None); + var unhandledExceptionIssue = new Issue() { Type = IssueType.Error, Message = detailInfo }; + var unhandledAnnotation = unhandledExceptionIssue.ToAnnotation(); + var jobAnnotations = new List(); + if (unhandledAnnotation.HasValue) + { + jobAnnotations.Add(unhandledAnnotation.Value); + } + + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, TaskResult.Failed, outputs: null, stepResults: null, jobAnnotations: jobAnnotations, CancellationToken.None); } catch (Exception ex) { diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 8d981c5492c..660883d73e0 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -18,6 +18,7 @@ using GitHub.Runner.Worker.Container; using GitHub.Runner.Worker.Handlers; using Newtonsoft.Json; +using Sdk.RSWebApi.Contracts; using ObjectTemplating = GitHub.DistributedTask.ObjectTemplating; using Pipelines = GitHub.DistributedTask.Pipelines; @@ -438,14 +439,26 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = PublishStepTelemetry(); - var stepResult = new StepResult(); - stepResult.ExternalID = _record.Id; - stepResult.Conclusion = _record.Result ?? TaskResult.Succeeded; - stepResult.Status = _record.State; - stepResult.Number = _record.Order; - stepResult.Name = _record.Name; - stepResult.StartedAt = _record.StartTime; - stepResult.CompletedAt = _record.FinishTime; + var stepResult = new StepResult + { + ExternalID = _record.Id, + Conclusion = _record.Result ?? TaskResult.Succeeded, + Status = _record.State, + Number = _record.Order, + Name = _record.Name, + StartedAt = _record.StartTime, + CompletedAt = _record.FinishTime, + Annotations = new List() + }; + + _record.Issues?.ForEach(issue => + { + var annotation = issue.ToAnnotation(); + if (annotation != null) + { + stepResult.Annotations.Add(annotation.Value); + } + }); Global.StepsResult.Add(stepResult); @@ -725,6 +738,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // Steps results for entire job Global.StepsResult = new List(); + // Job level annotations + Global.JobAnnotations = new List(); + // Job Outputs JobOutputs = new Dictionary(StringComparer.OrdinalIgnoreCase); diff --git a/src/Runner.Worker/GlobalContext.cs b/src/Runner.Worker/GlobalContext.cs index 9ca9cc3a7c6..3307401d036 100644 --- a/src/Runner.Worker/GlobalContext.cs +++ b/src/Runner.Worker/GlobalContext.cs @@ -1,10 +1,11 @@ -using System; +using System; using System.Collections.Generic; using GitHub.Actions.RunService.WebApi; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Common.Util; using GitHub.Runner.Worker.Container; using Newtonsoft.Json.Linq; +using Sdk.RSWebApi.Contracts; namespace GitHub.Runner.Worker { @@ -19,6 +20,7 @@ public sealed class GlobalContext public IDictionary> CompositeDefaults { get; set; } public List StepsTelemetry { get; set; } public List StepsResult { get; set; } + public List JobAnnotations { get; set; } public List JobTelemetry { get; set; } public TaskOrchestrationPlanReference Plan { get; set; } public List PrependPath { get; set; } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index 77a93bf9d34..f2edb873d8b 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -272,7 +272,7 @@ private async Task CompleteJobAsync(IRunServer runServer, IExecution { try { - await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, default); + await runServer.CompleteJobAsync(message.Plan.PlanId, message.JobId, result, jobContext.JobOutputs, jobContext.Global.StepsResult, jobContext.Global.JobAnnotations, default); return result; } catch (Exception ex) diff --git a/src/Sdk/RSWebApi/Contracts/Annotation.cs b/src/Sdk/RSWebApi/Contracts/Annotation.cs new file mode 100644 index 00000000000..522dccad961 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/Annotation.cs @@ -0,0 +1,35 @@ +using System.Runtime.Serialization; + +namespace Sdk.RSWebApi.Contracts +{ + [DataContract] + public struct Annotation + { + [DataMember(Name = "level", EmitDefaultValue = false)] + public AnnotationLevel Level; + + [DataMember(Name = "message", EmitDefaultValue = false)] + public string Message; + + [DataMember(Name = "rawDetails", EmitDefaultValue = false)] + public string RawDetails; + + [DataMember(Name = "path", EmitDefaultValue = false)] + public string Path; + + [DataMember(Name = "isInfrastructureIssue", EmitDefaultValue = false)] + public bool IsInfrastructureIssue; + + [DataMember(Name = "startLine", EmitDefaultValue = false)] + public long StartLine; + + [DataMember(Name = "endLine", EmitDefaultValue = false)] + public long EndLine; + + [DataMember(Name = "startColumn", EmitDefaultValue = false)] + public long StartColumn; + + [DataMember(Name = "endColumn", EmitDefaultValue = false)] + public long EndColumn; + } +} diff --git a/src/Sdk/RSWebApi/Contracts/AnnotationLevel.cs b/src/Sdk/RSWebApi/Contracts/AnnotationLevel.cs new file mode 100644 index 00000000000..826ac40d7a0 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/AnnotationLevel.cs @@ -0,0 +1,20 @@ +using System.Runtime.Serialization; + +namespace Sdk.RSWebApi.Contracts +{ + [DataContract] + public enum AnnotationLevel + { + [EnumMember] + UNKNOWN = 0, + + [EnumMember] + NOTICE = 1, + + [EnumMember] + WARNING = 2, + + [EnumMember] + FAILURE = 3 + } +} diff --git a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs index 27aa3f963b3..5c8813f9813 100644 --- a/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs +++ b/src/Sdk/RSWebApi/Contracts/CompleteJobRequest.cs @@ -2,6 +2,7 @@ using System.Collections.Generic; using System.Runtime.Serialization; using GitHub.DistributedTask.WebApi; +using Sdk.RSWebApi.Contracts; namespace GitHub.Actions.RunService.WebApi { @@ -10,17 +11,20 @@ public class CompleteJobRequest { [DataMember(Name = "planId", EmitDefaultValue = false)] public Guid PlanID { get; set; } - + [DataMember(Name = "jobId", EmitDefaultValue = false)] public Guid JobID { get; set; } - + [DataMember(Name = "conclusion")] public TaskResult Conclusion { get; set; } - + [DataMember(Name = "outputs", EmitDefaultValue = false)] - public Dictionary Outputs { get; set; } - + public Dictionary Outputs { get; set; } + [DataMember(Name = "stepResults", EmitDefaultValue = false)] public IList StepResults { get; set; } + + [DataMember(Name = "annotations", EmitDefaultValue = false)] + public IList Annotations { get; set; } } -} \ No newline at end of file +} diff --git a/src/Sdk/RSWebApi/Contracts/IssueExtensions.cs b/src/Sdk/RSWebApi/Contracts/IssueExtensions.cs new file mode 100644 index 00000000000..88272cca104 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/IssueExtensions.cs @@ -0,0 +1,91 @@ +using GitHub.DistributedTask.WebApi; + +namespace Sdk.RSWebApi.Contracts +{ + public static class IssueExtensions + { + public static Annotation? ToAnnotation(this Issue issue) + { + var issueMessage = issue.Message; + if (string.IsNullOrWhiteSpace(issueMessage)) + { + if (!issue.Data.TryGetValue(RunIssueKeys.Message, out issueMessage) || string.IsNullOrWhiteSpace(issueMessage)) + { + return null; + } + } + + var annotationLevel = GetAnnotationLevel(issue.Type); + var path = GetFilePath(issue); + var lineNumber = GetAnnotationNumber(issue, RunIssueKeys.Line) ?? 0; + var endLineNumber = GetAnnotationNumber(issue, RunIssueKeys.EndLine) ?? lineNumber; + var columnNumber = GetAnnotationNumber(issue, RunIssueKeys.Col) ?? 0; + var endColumnNumber = GetAnnotationNumber(issue, RunIssueKeys.EndColumn) ?? columnNumber; + var logLineNumber = GetAnnotationNumber(issue, RunIssueKeys.LogLineNumber) ?? 0; + + if (path == null && lineNumber == 0 && logLineNumber != 0) + { + lineNumber = logLineNumber; + endLineNumber = logLineNumber; + } + + return new Annotation + { + Level = annotationLevel, + Message = issueMessage, + Path = path, + StartLine = lineNumber, + EndLine = endLineNumber, + StartColumn = columnNumber, + EndColumn = endColumnNumber, + }; + } + + private static AnnotationLevel GetAnnotationLevel(IssueType issueType) + { + switch (issueType) + { + case IssueType.Error: + return AnnotationLevel.FAILURE; + case IssueType.Warning: + return AnnotationLevel.WARNING; + case IssueType.Notice: + return AnnotationLevel.NOTICE; + default: + return AnnotationLevel.UNKNOWN; + } + } + + private static int? GetAnnotationNumber(Issue issue, string key) + { + if (issue.Data.TryGetValue(key, out var numberString) && + int.TryParse(numberString, out var number)) + { + return number; + } + + return null; + } + + private static string GetAnnotationField(Issue issue, string key) + { + if (issue.Data.TryGetValue(key, out var value)) + { + return value; + } + + return null; + } + + private static string GetFilePath(Issue issue) + { + if (issue.Data.TryGetValue(RunIssueKeys.File, out var path) && + !string.IsNullOrWhiteSpace(path)) + { + return path; + } + + return null; + } + } +} diff --git a/src/Sdk/RSWebApi/Contracts/IssueKeys.cs b/src/Sdk/RSWebApi/Contracts/IssueKeys.cs new file mode 100644 index 00000000000..d7a14a847d9 --- /dev/null +++ b/src/Sdk/RSWebApi/Contracts/IssueKeys.cs @@ -0,0 +1,13 @@ +namespace Sdk.RSWebApi.Contracts +{ + public static class RunIssueKeys + { + public const string Message = "message"; + public const string File = "file"; + public const string Line = "line"; + public const string Col = "col"; + public const string EndLine = "endLine"; + public const string EndColumn = "endColumn"; + public const string LogLineNumber = "logFileLineNumber"; + } +} diff --git a/src/Sdk/RSWebApi/Contracts/StepResult.cs b/src/Sdk/RSWebApi/Contracts/StepResult.cs index e7df8d6d41e..e24489817d3 100644 --- a/src/Sdk/RSWebApi/Contracts/StepResult.cs +++ b/src/Sdk/RSWebApi/Contracts/StepResult.cs @@ -1,7 +1,9 @@ using System; +using System.Collections.Generic; using System.Runtime.Serialization; using System.Threading.Tasks; using GitHub.DistributedTask.WebApi; +using Sdk.RSWebApi.Contracts; namespace GitHub.Actions.RunService.WebApi { @@ -34,5 +36,8 @@ public class StepResult [DataMember(Name = "completed_log_lines", EmitDefaultValue = false)] public long? CompletedLogLines { get; set; } + + [DataMember(Name = "annotations", EmitDefaultValue = false)] + public List Annotations { get; set; } } -} \ No newline at end of file +} diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 8ddd1672b04..3b910403dbc 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -100,6 +100,7 @@ public async Task CompleteJobAsync( TaskResult result, Dictionary outputs, IList stepResults, + IList jobAnnotations, CancellationToken cancellationToken = default) { HttpMethod httpMethod = new HttpMethod("POST"); @@ -109,7 +110,8 @@ public async Task CompleteJobAsync( JobID = jobId, Conclusion = result, Outputs = outputs, - StepResults = stepResults + StepResults = stepResults, + Annotations = jobAnnotations }; requestUri = new Uri(requestUri, "completejob"); diff --git a/src/Test/L0/Sdk/RSWebApi/AnnotationsL0.cs b/src/Test/L0/Sdk/RSWebApi/AnnotationsL0.cs new file mode 100644 index 00000000000..0970e67c286 --- /dev/null +++ b/src/Test/L0/Sdk/RSWebApi/AnnotationsL0.cs @@ -0,0 +1,70 @@ +using System.Collections.Generic; +using GitHub.DistributedTask.WebApi; +using Sdk.RSWebApi.Contracts; +using Xunit; + +namespace GitHub.Actions.RunService.WebApi.Tests; + +public sealed class AnnotationsL0 +{ + [Fact] + public void ToAnnotation_ValidIssueWithMessage_ReturnsAnnotation() + { + var issue = new Issue + { + Type = IssueType.Error, + Message = "An error occurred", + IsInfrastructureIssue = true + }; + + issue.Data.Add(RunIssueKeys.File, "test.txt"); + issue.Data.Add(RunIssueKeys.Line, "5"); + issue.Data.Add(RunIssueKeys.Col, "10"); + issue.Data.Add(RunIssueKeys.EndLine, "8"); + issue.Data.Add(RunIssueKeys.EndColumn, "20"); + issue.Data.Add(RunIssueKeys.LogLineNumber, "2"); + + var annotation = issue.ToAnnotation(); + + Assert.NotNull(annotation); + Assert.Equal(AnnotationLevel.FAILURE, annotation.Value.Level); + Assert.Equal("An error occurred", annotation.Value.Message); + Assert.Equal("test.txt", annotation.Value.Path); + Assert.Equal(5, annotation.Value.StartLine); + Assert.Equal(8, annotation.Value.EndLine); + Assert.Equal(10, annotation.Value.StartColumn); + Assert.Equal(20, annotation.Value.EndColumn); + } + + [Fact] + public void ToAnnotation_ValidIssueWithEmptyMessage_ReturnsNull() + { + var issue = new Issue + { + Type = IssueType.Warning, + Message = string.Empty + }; + + var annotation = issue.ToAnnotation(); + + Assert.Null(annotation); + } + + [Fact] + public void ToAnnotation_ValidIssueWithMessageInData_ReturnsAnnotation() + { + var issue = new Issue + { + Type = IssueType.Warning, + Message = string.Empty, + }; + + issue.Data.Add(RunIssueKeys.Message, "A warning occurred"); + + var annotation = issue.ToAnnotation(); + + Assert.NotNull(annotation); + Assert.Equal(AnnotationLevel.WARNING, annotation.Value.Level); + Assert.Equal("A warning occurred", annotation.Value.Message); + } +} From 195f0704703ce57caac6fdf2eafef786f574c1eb Mon Sep 17 00:00:00 2001 From: John Wesley Walker III <81404201+jww3@users.noreply.github.com> Date: Wed, 3 May 2023 15:29:19 +0200 Subject: [PATCH 136/145] Remove Temporary Serialization Shim (#2549) --- src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs | 5 ----- src/Sdk/RSWebApi/RunServiceHttpClient.cs | 1 - src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs | 6 ++---- 3 files changed, 2 insertions(+), 10 deletions(-) diff --git a/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs b/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs index 895e29afa68..d3f155cde7e 100644 --- a/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs +++ b/src/Sdk/RSWebApi/Contracts/AcquireJobRequest.cs @@ -7,10 +7,5 @@ public class AcquireJobRequest { [DataMember(Name = "jobMessageId", EmitDefaultValue = false)] public string JobMessageId { get; set; } - - // This field will be removed in an upcoming Runner release. - // It's left here temporarily to facilitate the transition to the new field name, JobMessageId. - [DataMember(Name = "streamId", EmitDefaultValue = false)] - public string StreamId { get; set; } } } diff --git a/src/Sdk/RSWebApi/RunServiceHttpClient.cs b/src/Sdk/RSWebApi/RunServiceHttpClient.cs index 3b910403dbc..8d45ece066d 100644 --- a/src/Sdk/RSWebApi/RunServiceHttpClient.cs +++ b/src/Sdk/RSWebApi/RunServiceHttpClient.cs @@ -65,7 +65,6 @@ public async Task GetJobMessageAsync( var payload = new AcquireJobRequest { JobMessageId = messageId, - StreamId = messageId, }; requestUri = new Uri(requestUri, "acquirejob"); diff --git a/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs b/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs index c8883b07172..e9889ec0945 100644 --- a/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs +++ b/src/Test/L0/Sdk/RSWebApi/AcquireJobRequestL0.cs @@ -14,11 +14,9 @@ public sealed class AcquireJobRequestL0 [Trait("Category", "Common")] public void VerifySerialization() { - var jobMessageId = "1526919030369-33"; var request = new AcquireJobRequest { - JobMessageId = jobMessageId, - StreamId = jobMessageId + JobMessageId = "1526919030369-33" }; var serializer = new DataContractJsonSerializer(typeof(AcquireJobRequest)); using var stream = new MemoryStream(); @@ -27,7 +25,7 @@ public void VerifySerialization() stream.Position = 0; using var reader = new StreamReader(stream, Encoding.UTF8); string json = reader.ReadToEnd(); - string expected = DoubleQuotify(string.Format("{{'jobMessageId':'{0}','streamId':'{0}'}}", request.JobMessageId)); + string expected = DoubleQuotify(string.Format("{{'jobMessageId':'{0}'}}", request.JobMessageId)); Assert.Equal(expected, json); } From e32d77194e382f164974571dffa747465b0750cc Mon Sep 17 00:00:00 2001 From: John Hernley Date: Wed, 3 May 2023 16:04:21 -0400 Subject: [PATCH 137/145] Resolve Actions Directly From Launch for Run Service Jobs (#2529) Co-authored-by: Tingluo Huang --- src/Runner.Common/Constants.cs | 1 + src/Runner.Common/LaunchServer.cs | 42 ++++++++ src/Runner.Worker/ActionManager.cs | 11 ++- src/Runner.Worker/ExecutionContext.cs | 3 + src/Runner.Worker/JobRunner.cs | 12 +++ src/Sdk/Sdk.csproj | 2 +- src/Sdk/WebApi/WebApi/LaunchContracts.cs | 70 +++++++++++++ src/Sdk/WebApi/WebApi/LaunchHttpClient.cs | 115 ++++++++++++++++++++++ src/Test/L0/Worker/ActionManagerL0.cs | 21 ++++ 9 files changed, 275 insertions(+), 2 deletions(-) create mode 100644 src/Runner.Common/LaunchServer.cs create mode 100644 src/Sdk/WebApi/WebApi/LaunchContracts.cs create mode 100644 src/Sdk/WebApi/WebApi/LaunchHttpClient.cs diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index ceddd42116f..d14e3e88d05 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -261,6 +261,7 @@ public static class System public static readonly string AccessToken = "system.accessToken"; public static readonly string Culture = "system.culture"; public static readonly string PhaseDisplayName = "system.phaseDisplayName"; + public static readonly string JobRequestType = "system.jobRequestType"; public static readonly string OrchestrationId = "system.orchestrationId"; } } diff --git a/src/Runner.Common/LaunchServer.cs b/src/Runner.Common/LaunchServer.cs new file mode 100644 index 00000000000..338e5b88488 --- /dev/null +++ b/src/Runner.Common/LaunchServer.cs @@ -0,0 +1,42 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; +using GitHub.DistributedTask.WebApi; +using GitHub.Services.Launch.Client; +using GitHub.Services.WebApi; + +namespace GitHub.Runner.Common +{ + [ServiceLocator(Default = typeof(LaunchServer))] + public interface ILaunchServer : IRunnerService + { + void InitializeLaunchClient(Uri uri, string token); + + Task ResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList, CancellationToken cancellationToken); + } + + public sealed class LaunchServer : RunnerService, ILaunchServer + { + private LaunchHttpClient _launchClient; + + public void InitializeLaunchClient(Uri uri, string token) + { + var httpMessageHandler = HostContext.CreateHttpClientHandler(); + this._launchClient = new LaunchHttpClient(uri, httpMessageHandler, token, disposeHandler: true); + } + + public Task ResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList, + CancellationToken cancellationToken) + { + if (_launchClient != null) + { + return _launchClient.GetResolveActionsDownloadInfoAsync(planId, jobId, actionReferenceList, + cancellationToken: cancellationToken); + } + + throw new InvalidOperationException("Launch client is not initialized."); + } + } +} diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index 6a07c9aa0d9..efacc138ab1 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -11,6 +11,7 @@ using System.Threading.Tasks; using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.Runner.Common; +using GitHub.Runner.Common.Util; using GitHub.Runner.Sdk; using GitHub.Runner.Worker.Container; using GitHub.Services.Common; @@ -648,13 +649,21 @@ private async Task BuildActionContainerAsync(IExecutionContext executionContext, } // Resolve download info + var launchServer = HostContext.GetService(); var jobServer = HostContext.GetService(); var actionDownloadInfos = default(WebApi.ActionDownloadInfoCollection); for (var attempt = 1; attempt <= 3; attempt++) { try { - actionDownloadInfos = await jobServer.ResolveActionDownloadInfoAsync(executionContext.Global.Plan.ScopeIdentifier, executionContext.Global.Plan.PlanType, executionContext.Global.Plan.PlanId, executionContext.Root.Id, new WebApi.ActionReferenceList { Actions = actionReferences }, executionContext.CancellationToken); + if (MessageUtil.IsRunServiceJob(executionContext.Global.Variables.Get(Constants.Variables.System.JobRequestType))) + { + actionDownloadInfos = await launchServer.ResolveActionsDownloadInfoAsync(executionContext.Global.Plan.PlanId, executionContext.Root.Id, new WebApi.ActionReferenceList { Actions = actionReferences }, executionContext.CancellationToken); + } + else + { + actionDownloadInfos = await jobServer.ResolveActionDownloadInfoAsync(executionContext.Global.Plan.ScopeIdentifier, executionContext.Global.Plan.PlanType, executionContext.Global.Plan.PlanId, executionContext.Root.Id, new WebApi.ActionReferenceList { Actions = actionReferences }, executionContext.CancellationToken); + } break; } catch (Exception ex) when (!executionContext.CancellationToken.IsCancellationRequested) // Do not retry if the run is cancelled. diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index 660883d73e0..a5df0379f7c 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -757,6 +757,9 @@ public void InitializeJob(Pipelines.AgentJobRequestMessage message, Cancellation // File table Global.FileTable = new List(message.FileTable ?? new string[0]); + // What type of job request is running (i.e. Run Service vs. pipelines) + Global.Variables.Set(Constants.Variables.System.JobRequestType, message.MessageType); + // Expression values if (message.ContextData?.Count > 0) { diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index f2edb873d8b..e2803dc1c9c 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -58,6 +58,18 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat await runServer.ConnectAsync(systemConnection.Url, jobServerCredential); server = runServer; + message.Variables.TryGetValue("system.github.launch_endpoint", out VariableValue launchEndpointVariable); + var launchReceiverEndpoint = launchEndpointVariable?.Value; + + if (systemConnection?.Authorization != null && + systemConnection.Authorization.Parameters.TryGetValue("AccessToken", out var accessToken) && + !string.IsNullOrEmpty(accessToken) && + !string.IsNullOrEmpty(launchReceiverEndpoint)) + { + Trace.Info("Initializing launch client"); + var launchServer = HostContext.GetService(); + launchServer.InitializeLaunchClient(new Uri(launchReceiverEndpoint), accessToken); + } _jobServerQueue = HostContext.GetService(); _jobServerQueue.Start(message, resultServiceOnly: true); } diff --git a/src/Sdk/Sdk.csproj b/src/Sdk/Sdk.csproj index 2bb85af37f7..9e73caa8b9c 100644 --- a/src/Sdk/Sdk.csproj +++ b/src/Sdk/Sdk.csproj @@ -8,7 +8,7 @@ NU1701;NU1603 $(Version) TRACE - 7.3 + 8.0 true diff --git a/src/Sdk/WebApi/WebApi/LaunchContracts.cs b/src/Sdk/WebApi/WebApi/LaunchContracts.cs new file mode 100644 index 00000000000..0815a116bf3 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/LaunchContracts.cs @@ -0,0 +1,70 @@ +using System; +using System.Collections.Generic; +using System.Runtime.Serialization; + +namespace GitHub.Services.Launch.Contracts +{ + [DataContract] + public class ActionReferenceRequest + { + [DataMember(EmitDefaultValue = false, Name = "action")] + public string Action { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "version")] + public string Version { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "path")] + public string Path { get; set; } + } + + [DataContract] + public class ActionReferenceRequestList + { + [DataMember(EmitDefaultValue = false, Name = "actions")] + public IList Actions { get; set; } + } + + [DataContract] + public class ActionDownloadInfoResponse + { + [DataMember(EmitDefaultValue = false, Name = "authentication")] + public ActionDownloadAuthenticationResponse Authentication { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "name")] + public string Name { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "resolved_name")] + public string ResolvedName { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "resolved_sha")] + public string ResolvedSha { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "tar_url")] + public string TarUrl { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "version")] + public string Version { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "zip_url")] + public string ZipUrl { get; set; } + } + + [DataContract] + public class ActionDownloadAuthenticationResponse + { + [DataMember(EmitDefaultValue = false, Name = "expires_at")] + public DateTime ExpiresAt { get; set; } + + [DataMember(EmitDefaultValue = false, Name = "token")] + public string Token { get; set; } + } + + [DataContract] + public class ActionDownloadInfoResponseCollection + { + /// A mapping of action specifications to their download information. + /// The key is the full name of the action plus version, e.g. "actions/checkout@v2". + [DataMember(EmitDefaultValue = false, Name = "actions")] + public IDictionary Actions { get; set; } + } +} diff --git a/src/Sdk/WebApi/WebApi/LaunchHttpClient.cs b/src/Sdk/WebApi/WebApi/LaunchHttpClient.cs new file mode 100644 index 00000000000..54a718b3938 --- /dev/null +++ b/src/Sdk/WebApi/WebApi/LaunchHttpClient.cs @@ -0,0 +1,115 @@ +#nullable enable + +using System; +using System.Linq; +using System.Net.Http; +using System.Net.Http.Formatting; +using System.Net.Http.Headers; +using System.Threading; +using System.Threading.Tasks; + +using GitHub.DistributedTask.WebApi; +using GitHub.Services.Launch.Contracts; + +using Sdk.WebApi.WebApi; + +namespace GitHub.Services.Launch.Client +{ + public class LaunchHttpClient : RawHttpClientBase + { + public LaunchHttpClient( + Uri baseUrl, + HttpMessageHandler pipeline, + string token, + bool disposeHandler) + : base(baseUrl, pipeline, disposeHandler) + { + m_token = token; + m_launchServiceUrl = baseUrl; + m_formatter = new JsonMediaTypeFormatter(); + } + + public async Task GetResolveActionsDownloadInfoAsync(Guid planId, Guid jobId, ActionReferenceList actionReferenceList, CancellationToken cancellationToken) + { + var GetResolveActionsDownloadInfoURLEndpoint = new Uri(m_launchServiceUrl, $"/actions/build/{planId.ToString()}/jobs/{jobId.ToString()}/runnerresolve/actions"); + return ToServerData(await GetLaunchSignedURLResponse(GetResolveActionsDownloadInfoURLEndpoint, ToGitHubData(actionReferenceList), cancellationToken)); + } + + // Resolve Actions + private async Task GetLaunchSignedURLResponse(Uri uri, R request, CancellationToken cancellationToken) + { + using (HttpRequestMessage requestMessage = new HttpRequestMessage(HttpMethod.Post, uri)) + { + requestMessage.Headers.Authorization = new AuthenticationHeaderValue("Bearer", m_token); + requestMessage.Headers.Accept.Add(MediaTypeWithQualityHeaderValue.Parse("application/json")); + + using (HttpContent content = new ObjectContent(request, m_formatter)) + { + requestMessage.Content = content; + using (var response = await SendAsync(requestMessage, HttpCompletionOption.ResponseContentRead, cancellationToken: cancellationToken)) + { + return await ReadJsonContentAsync(response, cancellationToken); + } + } + } + } + + private static ActionReferenceRequestList ToGitHubData(ActionReferenceList actionReferenceList) + { + return new ActionReferenceRequestList + { + Actions = actionReferenceList.Actions?.Select(ToGitHubData).ToList() + }; + } + + private static ActionReferenceRequest ToGitHubData(ActionReference actionReference) + { + return new ActionReferenceRequest + { + Action = actionReference.NameWithOwner, + Version = actionReference.Ref, + Path = actionReference.Path + }; + } + + private static ActionDownloadInfoCollection ToServerData(ActionDownloadInfoResponseCollection actionDownloadInfoResponseCollection) + { + return new ActionDownloadInfoCollection + { + Actions = actionDownloadInfoResponseCollection.Actions?.ToDictionary(kvp => kvp.Key, kvp => ToServerData(kvp.Value)) + }; + } + + private static ActionDownloadInfo ToServerData(ActionDownloadInfoResponse actionDownloadInfoResponse) + { + return new ActionDownloadInfo + { + Authentication = ToServerData(actionDownloadInfoResponse.Authentication), + NameWithOwner = actionDownloadInfoResponse.Name, + ResolvedNameWithOwner = actionDownloadInfoResponse.ResolvedName, + ResolvedSha = actionDownloadInfoResponse.ResolvedSha, + TarballUrl = actionDownloadInfoResponse.TarUrl, + Ref = actionDownloadInfoResponse.Version, + ZipballUrl = actionDownloadInfoResponse.ZipUrl, + }; + } + + private static ActionDownloadAuthentication? ToServerData(ActionDownloadAuthenticationResponse? actionDownloadAuthenticationResponse) + { + if (actionDownloadAuthenticationResponse == null) + { + return null; + } + + return new ActionDownloadAuthentication + { + ExpiresAt = actionDownloadAuthenticationResponse.ExpiresAt, + Token = actionDownloadAuthenticationResponse.Token + }; + } + + private MediaTypeFormatter m_formatter; + private Uri m_launchServiceUrl; + private string m_token; + } +} diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 07cf9b48700..7a1a3d7a30d 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -29,6 +29,7 @@ public sealed class ActionManagerL0 private Mock _dockerManager; private Mock _ec; private Mock _jobServer; + private Mock _launchServer; private Mock _pluginManager; private TestHostContext _hc; private ActionManager _actionManager; @@ -2175,6 +2176,25 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t return Task.FromResult(result); }); + _launchServer = new Mock(); + _launchServer.Setup(x => x.ResolveActionsDownloadInfoAsync(It.IsAny(), It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((Guid planId, Guid jobId, ActionReferenceList actions, CancellationToken cancellationToken) => + { + var result = new ActionDownloadInfoCollection { Actions = new Dictionary() }; + foreach (var action in actions.Actions) + { + var key = $"{action.NameWithOwner}@{action.Ref}"; + result.Actions[key] = new ActionDownloadInfo + { + NameWithOwner = action.NameWithOwner, + Ref = action.Ref, + TarballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/tarball/{action.Ref}", + ZipballUrl = $"https://api.github.com/repos/{action.NameWithOwner}/zipball/{action.Ref}", + }; + } + return Task.FromResult(result); + }); + _pluginManager = new Mock(); _pluginManager.Setup(x => x.GetPluginAction(It.IsAny())).Returns(new RunnerPluginActionInfo() { PluginTypeName = "plugin.class, plugin", PostPluginTypeName = "plugin.cleanup, plugin" }); @@ -2183,6 +2203,7 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t _hc.SetSingleton(_dockerManager.Object); _hc.SetSingleton(_jobServer.Object); + _hc.SetSingleton(_launchServer.Object); _hc.SetSingleton(_pluginManager.Object); _hc.SetSingleton(actionManifest); _hc.SetSingleton(new HttpClientHandlerFactory()); From a92deb244a301f6dfc2fc33e8c02242f760c59c7 Mon Sep 17 00:00:00 2001 From: Tingluo Huang Date: Mon, 8 May 2023 18:54:34 -0400 Subject: [PATCH 138/145] Trace WebSocket exception into verbose level to reduce noise in diag log. (#2591) --- src/Runner.Common/JobServer.cs | 6 +++--- src/Runner.Common/ResultsServer.cs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/Runner.Common/JobServer.cs b/src/Runner.Common/JobServer.cs index c0719f61572..a6626099985 100644 --- a/src/Runner.Common/JobServer.cs +++ b/src/Runner.Common/JobServer.cs @@ -11,10 +11,10 @@ using GitHub.DistributedTask.WebApi; using GitHub.Runner.Sdk; using GitHub.Services.Common; +using GitHub.Services.OAuth; +using GitHub.Services.Results.Client; using GitHub.Services.WebApi; using GitHub.Services.WebApi.Utilities.Internal; -using GitHub.Services.Results.Client; -using GitHub.Services.OAuth; namespace GitHub.Runner.Common { @@ -254,7 +254,7 @@ public async Task AppendTimelineRecordFeedAsync(Guid scopeIdentifier, string hub { failedAttemptsToPostBatchedLinesByWebsocket++; Trace.Info($"Caught exception during append web console line to websocket, let's fallback to sending via non-websocket call (total calls: {totalBatchedLinesAttemptedByWebsocket}, failed calls: {failedAttemptsToPostBatchedLinesByWebsocket}, websocket state: {this._websocketClient?.State})."); - Trace.Error(ex); + Trace.Verbose(ex.ToString()); if (totalBatchedLinesAttemptedByWebsocket > _minWebsocketBatchedLinesCountToConsider) { // let's consider failure percentage diff --git a/src/Runner.Common/ResultsServer.cs b/src/Runner.Common/ResultsServer.cs index 905d717086b..9ba49349dd3 100644 --- a/src/Runner.Common/ResultsServer.cs +++ b/src/Runner.Common/ResultsServer.cs @@ -222,7 +222,7 @@ public async Task AppendLiveConsoleFeedAsync(Guid scopeIdentifier, string { var delay = BackoffTimerHelper.GetRandomBackoff(MinDelayForWebsocketReconnect, MaxDelayForWebsocketReconnect); Trace.Info($"Websocket is not open, let's attempt to connect back again with random backoff {delay} ms."); - Trace.Error(ex); + Trace.Verbose(ex.ToString()); retries++; InitializeWebsocketClient(_liveConsoleFeedUrl, _token, delay); } From ff9684c48ece4c94d871aa8abd5d2fe250f31855 Mon Sep 17 00:00:00 2001 From: John Wesley Walker III <81404201+jww3@users.noreply.github.com> Date: Wed, 10 May 2023 16:24:02 +0200 Subject: [PATCH 139/145] [1742] Ensure multiple composite annoations are correctly written. (#2311) * [1742] Ensure multiple composite annoations are correctly written. This implementation uses a collector pattern to allow embedded ExecutionContexts to stash Issue objects for later processing by a non-embedded ancestor ExecutionContext. Also: - Provide explicit constructor implementations for ExecutionContext - Leverage explicit constructors to solidify immutability of several ExecutionContext class members. - Fixed erroneous call to ExecutionContext.Complete in CompositeActionHandler.cs - Use a consistent timestamp for FinishTime in ExecutionContext::Complete * Ensure collected issues are processed only by a non-embedded ExecutionContext. This was already implicit. Now, just making it explicit. * Provide a clear mechanism that allows callers to opt-in/opt-out of ExecutionContext::AddIssue's logging behavior. * Addressed deserialization inconsistencies in TimelineRecord.cs * Added TimelineRecord unit tests. * Refined unit tests related to TimelineRecord::Variables case-insensitivity * Add a unit test that verifies ExecutionContextLogOptions::LogMessageOverride has the desired effect. * Responded to PR feedback. * Don't allow embedded ExecutionContexts to add Issues to a TimelineRecord --- src/Runner.Common/JobServerQueue.cs | 10 +- src/Runner.Worker/ActionCommandManager.cs | 10 +- src/Runner.Worker/ExecutionContext.cs | 172 +++++++---- .../Handlers/CompositeActionHandler.cs | 2 +- src/Runner.Worker/Handlers/OutputManager.cs | 3 +- src/Runner.Worker/JobExtension.cs | 2 +- src/Runner.Worker/JobRunner.cs | 3 +- src/Sdk/DTWebApi/WebApi/TimelineRecord.cs | 229 +++++++++------ .../WebApi/TimelineRecordL0.cs | 270 ++++++++++++++++++ src/Test/L0/Worker/ActionCommandManagerL0.cs | 20 +- src/Test/L0/Worker/ActionManagerL0.cs | 2 +- src/Test/L0/Worker/ActionManifestManagerL0.cs | 6 +- src/Test/L0/Worker/ActionRunnerL0.cs | 6 +- .../L0/Worker/CreateStepSummaryCommandL0.cs | 16 +- src/Test/L0/Worker/ExecutionContextL0.cs | 180 ++++++++---- src/Test/L0/Worker/OutputManagerL0.cs | 11 +- src/Test/L0/Worker/SaveStateFileCommandL0.cs | 14 +- src/Test/L0/Worker/SetEnvFileCommandL0.cs | 14 +- src/Test/L0/Worker/SetOutputFileCommandL0.cs | 18 +- src/Test/L0/Worker/StepsRunnerL0.cs | 6 +- .../TestData/timelinerecord_json_samples.yml | 70 +++++ 21 files changed, 794 insertions(+), 270 deletions(-) create mode 100644 src/Test/L0/DistributedTask/WebApi/TimelineRecordL0.cs create mode 100644 src/Test/TestData/timelinerecord_json_samples.yml diff --git a/src/Runner.Common/JobServerQueue.cs b/src/Runner.Common/JobServerQueue.cs index 57ba3eb23f6..d494645f154 100644 --- a/src/Runner.Common/JobServerQueue.cs +++ b/src/Runner.Common/JobServerQueue.cs @@ -756,17 +756,17 @@ private List MergeTimelineRecords(List timelineR timelineRecord.State = rec.State ?? timelineRecord.State; timelineRecord.WorkerName = rec.WorkerName ?? timelineRecord.WorkerName; - if (rec.ErrorCount != null && rec.ErrorCount > 0) + if (rec.ErrorCount > 0) { timelineRecord.ErrorCount = rec.ErrorCount; } - if (rec.WarningCount != null && rec.WarningCount > 0) + if (rec.WarningCount > 0) { timelineRecord.WarningCount = rec.WarningCount; } - if (rec.NoticeCount != null && rec.NoticeCount > 0) + if (rec.NoticeCount > 0) { timelineRecord.NoticeCount = rec.NoticeCount; } @@ -797,7 +797,7 @@ private List MergeTimelineRecords(List timelineR foreach (var record in mergedRecords) { Trace.Verbose($" Record: t={record.RecordType}, n={record.Name}, s={record.State}, st={record.StartTime}, {record.PercentComplete}%, ft={record.FinishTime}, r={record.Result}: {record.CurrentOperation}"); - if (record.Issues != null && record.Issues.Count > 0) + if (record.Issues != null) { foreach (var issue in record.Issues) { @@ -807,7 +807,7 @@ private List MergeTimelineRecords(List timelineR } } - if (record.Variables != null && record.Variables.Count > 0) + if (record.Variables != null) { foreach (var variable in record.Variables) { diff --git a/src/Runner.Worker/ActionCommandManager.cs b/src/Runner.Worker/ActionCommandManager.cs index 33fab1f2141..66cab8f8bd1 100644 --- a/src/Runner.Worker/ActionCommandManager.cs +++ b/src/Runner.Worker/ActionCommandManager.cs @@ -1,4 +1,4 @@ -using GitHub.DistributedTask.Pipelines.ContextData; +using GitHub.DistributedTask.Pipelines.ContextData; using GitHub.DistributedTask.WebApi; using GitHub.Runner.Worker.Container; using System; @@ -276,7 +276,7 @@ public void ProcessCommand(IExecutionContext context, string line, ActionCommand Message = $"Can't update {blocked} environment variable using ::set-env:: command." }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = $"{Constants.Runner.UnsupportedCommand}_{envName}"; - context.AddIssue(issue); + context.AddIssue(issue, ExecutionContextLogOptions.Default); return; } @@ -315,7 +315,7 @@ public void ProcessCommand(IExecutionContext context, string line, ActionCommand Message = String.Format(Constants.Runner.UnsupportedCommandMessage, this.Command) }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.UnsupportedCommand; - context.AddIssue(issue); + context.AddIssue(issue, ExecutionContextLogOptions.Default); } if (!command.Properties.TryGetValue(SetOutputCommandProperties.Name, out string outputName) || string.IsNullOrEmpty(outputName)) @@ -350,7 +350,7 @@ public void ProcessCommand(IExecutionContext context, string line, ActionCommand Message = String.Format(Constants.Runner.UnsupportedCommandMessage, this.Command) }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.UnsupportedCommand; - context.AddIssue(issue); + context.AddIssue(issue, ExecutionContextLogOptions.Default); } if (!command.Properties.TryGetValue(SaveStateCommandProperties.Name, out string stateName) || string.IsNullOrEmpty(stateName)) @@ -666,7 +666,7 @@ public void ProcessCommand(IExecutionContext context, string inputLine, ActionCo } } - context.AddIssue(issue); + context.AddIssue(issue, ExecutionContextLogOptions.Default); } public static void ValidateLinesAndColumns(ActionCommand command, IExecutionContext context) diff --git a/src/Runner.Worker/ExecutionContext.cs b/src/Runner.Worker/ExecutionContext.cs index a5df0379f7c..70b78cc6353 100644 --- a/src/Runner.Worker/ExecutionContext.cs +++ b/src/Runner.Worker/ExecutionContext.cs @@ -24,10 +24,16 @@ namespace GitHub.Runner.Worker { - public class ExecutionContextType + public static class ExecutionContextType { - public static string Job = "Job"; - public static string Task = "Task"; + public const string Job = "Job"; + public const string Task = "Task"; + } + + public record ExecutionContextLogOptions(bool WriteToLog, string LogMessageOverride) + { + public static readonly ExecutionContextLogOptions None = new(false, null); + public static readonly ExecutionContextLogOptions Default = new(true, null); } [ServiceLocator(Default = typeof(ExecutionContext))] @@ -93,7 +99,7 @@ public interface IExecutionContext : IRunnerService void SetGitHubContext(string name, string value); void SetOutput(string name, string value, out string reference); void SetTimeout(TimeSpan? timeout); - void AddIssue(Issue issue, string message = null); + void AddIssue(Issue issue, ExecutionContextLogOptions logOptions); void Progress(int percentage, string currentOperation = null); void UpdateDetailTimelineRecord(TimelineRecord record); @@ -119,7 +125,7 @@ public interface IExecutionContext : IRunnerService public sealed class ExecutionContext : RunnerService, IExecutionContext { - private const int _maxIssueCount = 10; + private const int _maxCountPerIssueType = 10; private const int _throttlingDelayReportThreshold = 10 * 1000; // Don't report throttling with less than 10 seconds delay private const int _maxIssueMessageLength = 4096; // Don't send issue with huge message since we can't forward them from actions to check annotation. private const int _maxIssueCountInTelemetry = 3; // Only send the first 3 issues to telemetry @@ -127,8 +133,10 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private readonly TimelineRecord _record = new(); private readonly Dictionary _detailRecords = new(); + private readonly List _embeddedIssueCollector; private readonly object _loggerLock = new(); private readonly object _matchersLock = new(); + private readonly ExecutionContext _parentExecutionContext; private event OnMatcherChanged _onMatcherChanged; @@ -136,7 +144,6 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private IPagingLogger _logger; private IJobServerQueue _jobServerQueue; - private ExecutionContext _parentExecutionContext; private Guid _mainTimelineId; private Guid _detailTimelineId; @@ -150,6 +157,29 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext private long _totalThrottlingDelayInMilliseconds = 0; private bool _stepTelemetryPublished = false; + public ExecutionContext() + : this(parent: null, embedded: false) + { + } + + private ExecutionContext(ExecutionContext parent, bool embedded) + { + if (embedded) + { + ArgUtil.NotNull(parent, nameof(parent)); + } + + _parentExecutionContext = parent; + this.IsEmbedded = embedded; + this.StepTelemetry = new ActionsStepTelemetry + { + IsEmbedded = embedded + }; + + //Embedded Execution Contexts pseudo-inherit their parent's embeddedIssueCollector. + _embeddedIssueCollector = embedded ? parent._embeddedIssueCollector : new(); + } + public Guid Id => _record.Id; public Guid EmbeddedId { get; private set; } public string ScopeName { get; private set; } @@ -162,7 +192,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext public Dictionary JobOutputs { get; private set; } public ActionsEnvironmentReference ActionsEnvironment { get; private set; } - public ActionsStepTelemetry StepTelemetry { get; } = new ActionsStepTelemetry(); + public ActionsStepTelemetry StepTelemetry { get; private init; } public DictionaryContextData ExpressionValues { get; } = new DictionaryContextData(); public IList ExpressionFunctions { get; } = new List(); @@ -187,7 +217,7 @@ public sealed class ExecutionContext : RunnerService, IExecutionContext // An embedded execution context shares the same record ID, record name, and logger // as its enclosing execution context. - public bool IsEmbedded { get; private set; } + public bool IsEmbedded { get; private init; } public TaskResult? Result { @@ -322,7 +352,7 @@ public IExecutionContext CreateChild( { Trace.Entering(); - var child = new ExecutionContext(); + var child = new ExecutionContext(this, isEmbedded); child.Initialize(HostContext); child.Global = Global; child.ScopeName = scopeName; @@ -347,7 +377,6 @@ public IExecutionContext CreateChild( child.ExpressionFunctions.Add(item); } child._cancellationTokenSource = cancellationTokenSource ?? new CancellationTokenSource(); - child._parentExecutionContext = this; child.EchoOnActionCommand = EchoOnActionCommand; if (recordOrder != null) @@ -368,11 +397,9 @@ public IExecutionContext CreateChild( child._logger.Setup(_mainTimelineId, recordId); } - child.IsEmbedded = isEmbedded; child.StepTelemetry.StepId = recordId; child.StepTelemetry.Stage = stage.ToString(); - child.StepTelemetry.IsEmbedded = isEmbedded; - child.StepTelemetry.StepContextName = child.GetFullyQualifiedContextName(); ; + child.StepTelemetry.StepContextName = child.GetFullyQualifiedContextName(); return child; } @@ -414,13 +441,24 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = this.Warning($"The job has experienced {TimeSpan.FromMilliseconds(_totalThrottlingDelayInMilliseconds).TotalSeconds} seconds total delay caused by server throttling."); } + DateTime now = DateTime.UtcNow; _record.CurrentOperation = currentOperation ?? _record.CurrentOperation; _record.ResultCode = resultCode ?? _record.ResultCode; - _record.FinishTime = DateTime.UtcNow; + _record.FinishTime = now; _record.PercentComplete = 100; _record.Result = _record.Result ?? TaskResult.Succeeded; _record.State = TimelineRecordState.Completed; + // Before our main timeline's final QueueTimelineRecordUpdate, + // inject any issues collected by embedded ExecutionContexts. + if (!this.IsEmbedded) + { + foreach (var issue in _embeddedIssueCollector) + { + AddIssue(issue, ExecutionContextLogOptions.None); + } + } + _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); // complete all detail timeline records. @@ -428,7 +466,7 @@ public TaskResult Complete(TaskResult? result = null, string currentOperation = { foreach (var record in _detailRecords) { - record.Value.FinishTime = record.Value.FinishTime ?? DateTime.UtcNow; + record.Value.FinishTime = record.Value.FinishTime ?? now; record.Value.PercentComplete = record.Value.PercentComplete ?? 100; record.Value.Result = record.Value.Result ?? TaskResult.Succeeded; record.Value.State = TimelineRecordState.Completed; @@ -572,14 +610,10 @@ public void Progress(int percentage, string currentOperation = null) } // This is not thread safe, the caller need to take lock before calling issue() - public void AddIssue(Issue issue, string logMessage = null) + public void AddIssue(Issue issue, ExecutionContextLogOptions logOptions) { ArgUtil.NotNull(issue, nameof(issue)); - - if (string.IsNullOrEmpty(logMessage)) - { - logMessage = issue.Message; - } + ArgUtil.NotNull(logOptions, nameof(logOptions)); issue.Message = HostContext.SecretMasker.MaskSecrets(issue.Message); if (issue.Message.Length > _maxIssueMessageLength) @@ -594,53 +628,64 @@ public void AddIssue(Issue issue, string logMessage = null) issue.Data["stepNumber"] = _record.Order.ToString(); } - if (issue.Type == IssueType.Error) + string wellKnownTag = null; + Int32 previousCountForIssueType = 0; + Action incrementIssueTypeCount = NoOp; + switch (issue.Type) { - if (!string.IsNullOrEmpty(logMessage)) - { - long logLineNumber = Write(WellKnownTags.Error, logMessage); - issue.Data["logFileLineNumber"] = logLineNumber.ToString(); - } + case IssueType.Error: + wellKnownTag = WellKnownTags.Error; + previousCountForIssueType = _record.ErrorCount; + incrementIssueTypeCount = () => { _record.ErrorCount++; }; + break; + case IssueType.Warning: + wellKnownTag = WellKnownTags.Warning; + previousCountForIssueType = _record.WarningCount; + incrementIssueTypeCount = () => { _record.WarningCount++; }; + break; + case IssueType.Notice: + wellKnownTag = WellKnownTags.Notice; + previousCountForIssueType = _record.NoticeCount; + incrementIssueTypeCount = () => { _record.NoticeCount++; }; + break; + } - if (_record.ErrorCount < _maxIssueCount) + if (!string.IsNullOrEmpty(wellKnownTag)) + { + if (!this.IsEmbedded && previousCountForIssueType < _maxCountPerIssueType) { + incrementIssueTypeCount(); _record.Issues.Add(issue); } - _record.ErrorCount++; - } - else if (issue.Type == IssueType.Warning) - { - if (!string.IsNullOrEmpty(logMessage)) + if (logOptions.WriteToLog) { - long logLineNumber = Write(WellKnownTags.Warning, logMessage); - issue.Data["logFileLineNumber"] = logLineNumber.ToString(); - } + string logMessage = issue.Message; + if (!string.IsNullOrEmpty(logOptions.LogMessageOverride)) + { + logMessage = logOptions.LogMessageOverride; + } - if (_record.WarningCount < _maxIssueCount) - { - _record.Issues.Add(issue); + if (!string.IsNullOrEmpty(logMessage)) + { + // Note that ::Write() has its own secret-masking logic. + long logLineNumber = Write(wellKnownTag, logMessage); + issue.Data["logFileLineNumber"] = logLineNumber.ToString(); + } } + } - _record.WarningCount++; + // Embedded ExecutionContexts (a.k.a. Composite actions) should never upload a timeline record to the server. + // Instead, we store processed issues on a shared (psuedo-inherited) list (belonging to the closest + // non-embedded ancestor ExecutionContext) so that they can be processed when that ancestor completes. + if (this.IsEmbedded) + { + _embeddedIssueCollector.Add(issue); } - else if (issue.Type == IssueType.Notice) + else { - if (!string.IsNullOrEmpty(logMessage)) - { - long logLineNumber = Write(WellKnownTags.Notice, logMessage); - issue.Data["logFileLineNumber"] = logLineNumber.ToString(); - } - - if (_record.NoticeCount < _maxIssueCount) - { - _record.Issues.Add(issue); - } - - _record.NoticeCount++; + _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); } - - _jobServerQueue.QueueTimelineRecordUpdate(_mainTimelineId, _record); } public void UpdateDetailTimelineRecord(TimelineRecord record) @@ -1014,8 +1059,7 @@ public void PublishStepTelemetry() StepTelemetry.FinishTime = _record.FinishTime; } - if (!IsEmbedded && - _record.Issues.Count > 0) + if (!IsEmbedded) { foreach (var issue in _record.Issues) { @@ -1181,6 +1225,11 @@ public void ApplyContinueOnError(TemplateToken continueOnErrorToken) UpdateGlobalStepsContext(); } + + private static void NoOp() + { + } + } // The Error/Warning/etc methods are created as extension methods to simplify unit testing. @@ -1205,19 +1254,22 @@ public static void Error(this IExecutionContext context, Exception ex) // Do not add a format string overload. See comment on ExecutionContext.Write(). public static void Error(this IExecutionContext context, string message) { - context.AddIssue(new Issue() { Type = IssueType.Error, Message = message }); + var issue = new Issue() { Type = IssueType.Error, Message = message }; + context.AddIssue(issue, ExecutionContextLogOptions.Default); } // Do not add a format string overload. See comment on ExecutionContext.Write(). public static void InfrastructureError(this IExecutionContext context, string message) { - context.AddIssue(new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true }); + var issue = new Issue() { Type = IssueType.Error, Message = message, IsInfrastructureIssue = true }; + context.AddIssue(issue, ExecutionContextLogOptions.Default); } // Do not add a format string overload. See comment on ExecutionContext.Write(). public static void Warning(this IExecutionContext context, string message) { - context.AddIssue(new Issue() { Type = IssueType.Warning, Message = message }); + var issue = new Issue() { Type = IssueType.Warning, Message = message }; + context.AddIssue(issue, ExecutionContextLogOptions.Default); } // Do not add a format string overload. See comment on ExecutionContext.Write(). diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index b22d9399004..aa749b2fe17 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -318,7 +318,7 @@ private async Task RunStepsAsync(List embeddedSteps, ActionRunStage stage // Evaluation error Trace.Info("Caught exception from expression for embedded step.env"); step.ExecutionContext.Error(ex); - step.ExecutionContext.Complete(TaskResult.Failed); + SetStepConclusion(step, TaskResult.Failed); } // Register Callback diff --git a/src/Runner.Worker/Handlers/OutputManager.cs b/src/Runner.Worker/Handlers/OutputManager.cs index 8b97c867d95..5dd94059bed 100644 --- a/src/Runner.Worker/Handlers/OutputManager.cs +++ b/src/Runner.Worker/Handlers/OutputManager.cs @@ -143,7 +143,8 @@ public void OnDataReceived(object sender, ProcessDataReceivedEventArgs e) if (issue != null) { // Log issue - _executionContext.AddIssue(issue, stripped); + var logOptions = new ExecutionContextLogOptions(true, stripped); + _executionContext.AddIssue(issue, logOptions); return; } diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 580c9999dd7..42e6fea816b 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -683,7 +683,7 @@ private async Task CheckDiskSpaceAsync(IExecutionContext context, CancellationTo { var issue = new Issue() { Type = IssueType.Warning, Message = $"You are running out of disk space. The runner will stop working when the machine runs out of disk space. Free space left: {freeSpaceInMB} MB" }; issue.Data[Constants.Runner.InternalTelemetryIssueDataKey] = Constants.Runner.LowDiskSpace; - context.AddIssue(issue); + context.AddIssue(issue, ExecutionContextLogOptions.Default); return; } diff --git a/src/Runner.Worker/JobRunner.cs b/src/Runner.Worker/JobRunner.cs index e2803dc1c9c..737ad6d8596 100644 --- a/src/Runner.Worker/JobRunner.cs +++ b/src/Runner.Worker/JobRunner.cs @@ -120,7 +120,8 @@ public async Task RunAsync(AgentJobRequestMessage message, Cancellat default: throw new ArgumentException(HostContext.RunnerShutdownReason.ToString(), nameof(HostContext.RunnerShutdownReason)); } - jobContext.AddIssue(new Issue() { Type = IssueType.Error, Message = errorMessage }); + var issue = new Issue() { Type = IssueType.Error, Message = errorMessage }; + jobContext.AddIssue(issue, ExecutionContextLogOptions.Default); }); // Validate directory permissions. diff --git a/src/Sdk/DTWebApi/WebApi/TimelineRecord.cs b/src/Sdk/DTWebApi/WebApi/TimelineRecord.cs index 51317d1713b..4d41572237d 100644 --- a/src/Sdk/DTWebApi/WebApi/TimelineRecord.cs +++ b/src/Sdk/DTWebApi/WebApi/TimelineRecord.cs @@ -1,8 +1,7 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.Serialization; -using GitHub.Services.Common; namespace GitHub.DistributedTask.WebApi { @@ -10,69 +9,78 @@ namespace GitHub.DistributedTask.WebApi public sealed class TimelineRecord { public TimelineRecord() + : this(null) { - this.Attempt = 1; } private TimelineRecord(TimelineRecord recordToBeCloned) { - this.Attempt = recordToBeCloned.Attempt; - this.ChangeId = recordToBeCloned.ChangeId; - this.CurrentOperation = recordToBeCloned.CurrentOperation; - this.FinishTime = recordToBeCloned.FinishTime; - this.Id = recordToBeCloned.Id; - this.Identifier = recordToBeCloned.Identifier; - this.LastModified = recordToBeCloned.LastModified; - this.Location = recordToBeCloned.Location; - this.Name = recordToBeCloned.Name; - this.Order = recordToBeCloned.Order; - this.ParentId = recordToBeCloned.ParentId; - this.PercentComplete = recordToBeCloned.PercentComplete; - this.RecordType = recordToBeCloned.RecordType; - this.Result = recordToBeCloned.Result; - this.ResultCode = recordToBeCloned.ResultCode; - this.StartTime = recordToBeCloned.StartTime; - this.State = recordToBeCloned.State; - this.TimelineId = recordToBeCloned.TimelineId; - this.WorkerName = recordToBeCloned.WorkerName; - this.RefName = recordToBeCloned.RefName; - this.ErrorCount = recordToBeCloned.ErrorCount; - this.WarningCount = recordToBeCloned.WarningCount; - this.NoticeCount = recordToBeCloned.NoticeCount; - this.AgentPlatform = recordToBeCloned.AgentPlatform; - - if (recordToBeCloned.Log != null) + this.EnsureInitialized(); + + if (recordToBeCloned != null) { - this.Log = new TaskLogReference + this.Attempt = recordToBeCloned.Attempt; + this.ChangeId = recordToBeCloned.ChangeId; + this.CurrentOperation = recordToBeCloned.CurrentOperation; + this.FinishTime = recordToBeCloned.FinishTime; + this.Id = recordToBeCloned.Id; + this.Identifier = recordToBeCloned.Identifier; + this.LastModified = recordToBeCloned.LastModified; + this.Location = recordToBeCloned.Location; + this.Name = recordToBeCloned.Name; + this.Order = recordToBeCloned.Order; + this.ParentId = recordToBeCloned.ParentId; + this.PercentComplete = recordToBeCloned.PercentComplete; + this.RecordType = recordToBeCloned.RecordType; + this.Result = recordToBeCloned.Result; + this.ResultCode = recordToBeCloned.ResultCode; + this.StartTime = recordToBeCloned.StartTime; + this.State = recordToBeCloned.State; + this.TimelineId = recordToBeCloned.TimelineId; + this.WorkerName = recordToBeCloned.WorkerName; + this.RefName = recordToBeCloned.RefName; + this.ErrorCount = recordToBeCloned.ErrorCount; + this.WarningCount = recordToBeCloned.WarningCount; + this.NoticeCount = recordToBeCloned.NoticeCount; + this.AgentPlatform = recordToBeCloned.AgentPlatform; + + if (recordToBeCloned.Log != null) { - Id = recordToBeCloned.Log.Id, - Location = recordToBeCloned.Log.Location, - }; - } + this.Log = new TaskLogReference + { + Id = recordToBeCloned.Log.Id, + Location = recordToBeCloned.Log.Location, + }; + } - if (recordToBeCloned.Details != null) - { - this.Details = new TimelineReference + if (recordToBeCloned.Details != null) { - ChangeId = recordToBeCloned.Details.ChangeId, - Id = recordToBeCloned.Details.Id, - Location = recordToBeCloned.Details.Location, - }; - } + this.Details = new TimelineReference + { + ChangeId = recordToBeCloned.Details.ChangeId, + Id = recordToBeCloned.Details.Id, + Location = recordToBeCloned.Details.Location, + }; + } - if (recordToBeCloned.m_issues?.Count> 0) - { - this.Issues.AddRange(recordToBeCloned.Issues.Select(i => i.Clone())); - } + if (recordToBeCloned.m_issues?.Count > 0) + { + this.Issues.AddRange(recordToBeCloned.Issues.Select(i => i.Clone())); + } - if (recordToBeCloned.m_previousAttempts?.Count > 0) - { - this.PreviousAttempts.AddRange(recordToBeCloned.PreviousAttempts); - } + if (recordToBeCloned.m_previousAttempts?.Count > 0) + { + this.m_previousAttempts.AddRange(recordToBeCloned.m_previousAttempts); + } - if (recordToBeCloned.m_variables?.Count > 0) - { - this.m_variables = recordToBeCloned.Variables.ToDictionary(k => k.Key, v => v.Value.Clone()); + if (recordToBeCloned.m_variables?.Count > 0) + { + // Don't pave over the case-insensitive Dictionary we initialized above. + foreach (var kvp in recordToBeCloned.m_variables) + { + m_variables[kvp.Key] = kvp.Value.Clone(); + } + } } } @@ -98,14 +106,14 @@ public Guid? ParentId } [DataMember(Name = "Type", Order = 3)] - public String RecordType + public string RecordType { get; set; } [DataMember(Order = 4)] - public String Name + public string Name { get; set; @@ -126,7 +134,7 @@ public DateTime? FinishTime } [DataMember(Order = 7)] - public String CurrentOperation + public string CurrentOperation { get; set; @@ -154,7 +162,7 @@ public TaskResult? Result } [DataMember(Order = 11)] - public String ResultCode + public string ResultCode { get; set; @@ -175,7 +183,7 @@ public DateTime LastModified } [DataMember(Order = 14)] - public String WorkerName + public string WorkerName { get; set; @@ -189,7 +197,7 @@ public Int32? Order } [DataMember(Order = 16, EmitDefaultValue = false)] - public String RefName + public string RefName { get; set; @@ -209,35 +217,46 @@ public TimelineReference Details set; } - [DataMember(Order = 40)] - public Int32? ErrorCount + public Int32 ErrorCount { - get; - set; + get + { + return m_errorCount.GetValueOrDefault(0); + } + set + { + m_errorCount = value; + } } - [DataMember(Order = 50)] - public Int32? WarningCount + public Int32 WarningCount { - get; - set; + get + { + return m_warningCount.GetValueOrDefault(0); + } + set + { + m_warningCount = value; + } } - [DataMember(Order = 55)] - public Int32? NoticeCount + public Int32 NoticeCount { - get; - set; + get + { + return m_noticeCount.GetValueOrDefault(0); + } + set + { + m_noticeCount = value; + } } public List Issues { get { - if (m_issues == null) - { - m_issues = new List(); - } return m_issues; } } @@ -257,7 +276,7 @@ public Int32 Attempt } [DataMember(Order = 131)] - public String Identifier + public string Identifier { get; set; @@ -274,22 +293,14 @@ public IList PreviousAttempts { get { - if (m_previousAttempts == null) - { - m_previousAttempts = new List(); - } return m_previousAttempts; } } - public IDictionary Variables + public IDictionary Variables { get { - if (m_variables == null) - { - m_variables = new Dictionary(StringComparer.OrdinalIgnoreCase); - } return m_variables; } } @@ -299,13 +310,53 @@ public TimelineRecord Clone() return new TimelineRecord(this); } - [DataMember(Name = "Issues", EmitDefaultValue = false, Order = 60)] + [OnDeserialized] + private void OnDeserialized(StreamingContext context) + { + this.EnsureInitialized(); + } + + + /// + /// DataContractSerializer bypasses all constructor logic and inline initialization! + /// This method takes the place of a workhorse constructor for baseline initialization. + /// The expectation is for this logic to be accessible to constructors and also to the OnDeserialized helper. + /// + private void EnsureInitialized() + { + // Note that ?? is a short-circuiting operator. (??= would be preferable, but it's not supported in the .NET Framework version currently used by actions/runner.) + + // De-nullify the following historically-nullable ints. + // (After several weeks in production, it may be possible to eliminate these nullable backing fields.) + m_errorCount = m_errorCount ?? 0; + m_warningCount = m_warningCount ?? 0; + m_noticeCount = m_noticeCount ?? 0; + + m_issues = m_issues ?? new List(); + m_previousAttempts = m_previousAttempts ?? new List(); + this.Attempt = Math.Max(this.Attempt, 1); + + // Ensure whatever content may have been deserialized for m_variables is backed by a case-insensitive Dictionary. + var empty = Enumerable.Empty>(); + m_variables = new Dictionary(m_variables ?? empty, StringComparer.OrdinalIgnoreCase); + } + + [DataMember(Name = nameof(ErrorCount), Order = 40)] + private Int32? m_errorCount; + + [DataMember(Name = nameof(WarningCount), Order = 50)] + private Int32? m_warningCount; + + [DataMember(Name = nameof(NoticeCount), Order = 55)] + private Int32? m_noticeCount; + + [DataMember(Name = nameof(Issues), EmitDefaultValue = false, Order = 60)] private List m_issues; - [DataMember(Name = "Variables", EmitDefaultValue = false, Order = 80)] - private Dictionary m_variables; + [DataMember(Name = nameof(Variables), EmitDefaultValue = false, Order = 80)] + private Dictionary m_variables; - [DataMember(Name = "PreviousAttempts", EmitDefaultValue = false, Order = 120)] + [DataMember(Name = nameof(PreviousAttempts), EmitDefaultValue = false, Order = 120)] private List m_previousAttempts; } } diff --git a/src/Test/L0/DistributedTask/WebApi/TimelineRecordL0.cs b/src/Test/L0/DistributedTask/WebApi/TimelineRecordL0.cs new file mode 100644 index 00000000000..7df9cf3921d --- /dev/null +++ b/src/Test/L0/DistributedTask/WebApi/TimelineRecordL0.cs @@ -0,0 +1,270 @@ +#nullable enable + +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using System.Runtime.Serialization.Json; +using GitHub.DistributedTask.WebApi; +using Xunit; +using System.Text; + +namespace GitHub.Runner.Common.Tests.DistributedTask +{ + public sealed class TimelineRecordL0 + { + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_Defaults() + { + var tr = new TimelineRecord(); + + Assert.Equal(0, tr.ErrorCount); + Assert.Equal(0, tr.WarningCount); + Assert.Equal(0, tr.NoticeCount); + Assert.Equal(1, tr.Attempt); + Assert.NotNull(tr.Issues); + Assert.NotNull(tr.PreviousAttempts); + Assert.NotNull(tr.Variables); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_Clone() + { + var original = new TimelineRecord(); + original.ErrorCount = 100; + original.WarningCount = 200; + original.NoticeCount = 300; + original.Attempt = 3; + + // The Variables dictionary should be a case-insensitive dictionary. + original.Variables["xxx"] = new VariableValue("first", false); + original.Variables["XXX"] = new VariableValue("second", false); + + Assert.Equal(1, original.Variables.Count); + Assert.Equal("second", original.Variables.Values.First().Value); + Assert.Equal("second", original.Variables["xXx"].Value); + + var clone = original.Clone(); + + Assert.NotSame(original, clone); + Assert.NotSame(original.Variables, clone.Variables); + Assert.Equal(100, clone.ErrorCount); + Assert.Equal(200, clone.WarningCount); + Assert.Equal(300, clone.NoticeCount); + Assert.Equal(3, clone.Attempt); + + // Now, mutate the original post-clone. + original.ErrorCount++; + original.WarningCount += 10; + original.NoticeCount *= 3; + original.Attempt--; + original.Variables["a"] = new VariableValue("1", false); + + // Verify that the clone was unaffected by the changes to the original. + Assert.Equal(100, clone.ErrorCount); + Assert.Equal(200, clone.WarningCount); + Assert.Equal(300, clone.NoticeCount); + Assert.Equal(3, clone.Attempt); + Assert.Equal(1, clone.Variables.Count); + Assert.Equal("second", clone.Variables.Values.First().Value); + + // Verify that the clone's Variables dictionary is also case-sensitive. + clone.Variables["yyy"] = new VariableValue("third", false); + clone.Variables["YYY"] = new VariableValue("fourth", false); + + Assert.Equal(2, clone.Variables.Count); + Assert.Equal("second", clone.Variables["xXx"].Value); + Assert.Equal("fourth", clone.Variables["yYy"].Value); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_DeserializationEdgeCase_NonNullCollections() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // Verify that missing JSON fields don't result in null values for collection properties. + var tr = Deserialize(jsonSamples["minimal"]); + Assert.NotNull(tr); + Assert.Equal("minimal", tr!.Name); + Assert.NotNull(tr.Issues); + Assert.NotNull(tr.PreviousAttempts); + Assert.NotNull(tr.Variables); + + // Verify that explicitly-null JSON fields don't result in null values for collection properties. + // (Our deserialization logic should fix these up and instantiate an empty collection.) + tr = Deserialize(jsonSamples["explicit-null-collections"]); + Assert.NotNull(tr); + Assert.Equal("explicit-null-collections", tr!.Name); + Assert.NotNull(tr.Issues); + Assert.NotNull(tr.PreviousAttempts); + Assert.NotNull(tr.Variables); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_DeserializationEdgeCase_AttemptCannotBeLessThan1() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // Verify that 1 is the effective floor for TimelineRecord::Attempt. + var tr = Deserialize(jsonSamples["minimal"]); + Assert.NotNull(tr); + Assert.Equal("minimal", tr!.Name); + Assert.Equal(1, tr.Attempt); + + tr = Deserialize(jsonSamples["invalid-attempt-value"]); + Assert.NotNull(tr); + Assert.Equal("invalid-attempt-value", tr!.Name); + Assert.Equal(1, tr.Attempt); + + tr = Deserialize(jsonSamples["zero-attempt-value"]); + Assert.NotNull(tr); + Assert.Equal("zero-attempt-value", tr!.Name); + Assert.Equal(1, tr.Attempt); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_DeserializationEdgeCase_HandleLegacyNullsGracefully() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // Verify that nulls for ErrorCount, WarningCount, and NoticeCount are interpreted as 0. + var tr = Deserialize(jsonSamples["legacy-nulls"]); + Assert.NotNull(tr); + Assert.Equal("legacy-nulls", tr!.Name); + Assert.Equal(0, tr.ErrorCount); + Assert.Equal(0, tr.WarningCount); + Assert.Equal(0, tr.NoticeCount); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_DeserializationEdgeCase_HandleMissingCountsGracefully() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // Verify that nulls for ErrorCount, WarningCount, and NoticeCount are interpreted as 0. + var tr = Deserialize(jsonSamples["missing-counts"]); + Assert.NotNull(tr); + Assert.Equal("missing-counts", tr!.Name); + Assert.Equal(0, tr.ErrorCount); + Assert.Equal(0, tr.WarningCount); + Assert.Equal(0, tr.NoticeCount); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_DeserializationEdgeCase_NonZeroCounts() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // Verify that nulls for ErrorCount, WarningCount, and NoticeCount are interpreted as 0. + var tr = Deserialize(jsonSamples["non-zero-counts"]); + Assert.NotNull(tr); + Assert.Equal("non-zero-counts", tr!.Name); + Assert.Equal(10, tr.ErrorCount); + Assert.Equal(20, tr.WarningCount); + Assert.Equal(30, tr.NoticeCount); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_Deserialization_LeanTimelineRecord() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // Verify that a lean TimelineRecord can be deserialized. + var tr = Deserialize(jsonSamples["lean"]); + Assert.NotNull(tr); + Assert.Equal("lean", tr!.Name); + Assert.Equal(4, tr.Attempt); + Assert.Equal(1, tr.Issues.Count); + Assert.Equal(3, tr.Variables.Count); + Assert.Equal(3, tr.PreviousAttempts.Count); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_Deserialization_VariablesDictionaryIsCaseInsensitive() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + var tr = Deserialize(jsonSamples["lean"]); + Assert.NotNull(tr); + Assert.Equal("lean", tr!.Name); + Assert.Equal(3, tr.Variables.Count); + + // Verify that the Variables Dictionary is case-insensitive. + tr.Variables["X"] = new VariableValue("overwritten", false); + Assert.Equal(3, tr.Variables.Count); + + tr.Variables["new"] = new VariableValue("new.1", false); + Assert.Equal(4, tr.Variables.Count); + + tr.Variables["NEW"] = new VariableValue("new.2", false); + Assert.Equal(4, tr.Variables.Count); + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "DistributedTask")] + public void VerifyTimelineRecord_DeserializationEdgeCase_DuplicateVariableKeysThrowsException() + { + var jsonSamples = LoadJsonSamples(JsonSamplesFilePath); + + // We could be more forgiving in this case if we discover that it's not uncommon in Production for serialized TimelineRecords to: + // 1) get incorrectly instantiated with a case-sensitive Variables dictionary (in older versions, this was possible via TimelineRecord::Clone) + // 2) end up with case variations of the same key + // 3) make another serialization/deserialization round trip. + // + // If we wanted to grant clemency to such incorrectly-serialized TimelineRecords, + // the fix to TimelineRecord::EnsureInitialized would look something like the following: + // + // var seedVariables = m_variables ?? Enumerable.Empty>(); + // m_variables = new Dictionary(seedVariables.Count(), StringComparer.OrdinalIgnoreCase); + // foreach (var kvp in seedVariables) + // { + // m_variables[kvp.Key] = kvp.Value; + // } + Assert.Throws(() => Deserialize(jsonSamples["duplicate-variable-keys"])); + } + + + private static Dictionary LoadJsonSamples(string path) + { + // Embedding independent JSON samples within YML works well because JSON generally doesn't need to be escaped or otherwise mangled. + var yamlDeserializer = new YamlDotNet.Serialization.Deserializer(); + using var stream = new StreamReader(path); + return yamlDeserializer.Deserialize>(stream); + } + + private static TimelineRecord? Deserialize(string rawJson) + { + using var stream = new MemoryStream(Encoding.UTF8.GetBytes(rawJson ?? string.Empty)); + return m_jsonSerializer.ReadObject(stream) as TimelineRecord; + } + + private static string JsonSamplesFilePath + { + get + { + return Path.Combine(TestUtil.GetTestDataPath(), "timelinerecord_json_samples.yml"); + } + } + + private static readonly DataContractJsonSerializer m_jsonSerializer = new(typeof(TimelineRecord)); + } +} diff --git a/src/Test/L0/Worker/ActionCommandManagerL0.cs b/src/Test/L0/Worker/ActionCommandManagerL0.cs index cef826fd94e..78a5ebd6927 100644 --- a/src/Test/L0/Worker/ActionCommandManagerL0.cs +++ b/src/Test/L0/Worker/ActionCommandManagerL0.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.IO; using System.Runtime.CompilerServices; @@ -32,10 +32,10 @@ public void EnablePluginInternalCommand() hc.GetTrace().Info($"{tag} {line}"); return 1; }); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((Issue issue, string message) => + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((Issue issue, ExecutionContextLogOptions logOptions) => { - hc.GetTrace().Info($"{issue.Type} {issue.Message} {message ?? string.Empty}"); + hc.GetTrace().Info($"{issue.Type} {issue.Message} {logOptions.LogMessageOverride ?? string.Empty}"); }); _commandManager.EnablePluginInternalCommand(); @@ -59,10 +59,10 @@ public void DisablePluginInternalCommand() hc.GetTrace().Info($"{tag} {line}"); return 1; }); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((Issue issue, string message) => + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((Issue issue, ExecutionContextLogOptions logOptions) => { - hc.GetTrace().Info($"{issue.Type} {issue.Message} {message ?? string.Empty}"); + hc.GetTrace().Info($"{issue.Type} {issue.Message} {logOptions.LogMessageOverride ?? string.Empty}"); }); _commandManager.EnablePluginInternalCommand(); @@ -92,10 +92,10 @@ public void StopProcessCommand() return 1; }); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((Issue issue, string message) => + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((Issue issue, ExecutionContextLogOptions logOptions) => { - hc.GetTrace().Info($"{issue.Type} {issue.Message} {message ?? string.Empty}"); + hc.GetTrace().Info($"{issue.Type} {issue.Message} {logOptions.LogMessageOverride ?? string.Empty}"); }); _ec.Object.Global.EnvironmentVariables = new Dictionary(); diff --git a/src/Test/L0/Worker/ActionManagerL0.cs b/src/Test/L0/Worker/ActionManagerL0.cs index 7a1a3d7a30d..6052a848bc9 100644 --- a/src/Test/L0/Worker/ActionManagerL0.cs +++ b/src/Test/L0/Worker/ActionManagerL0.cs @@ -2148,7 +2148,7 @@ private void Setup([CallerMemberName] string name = "", bool enableComposite = t _ec.Object.Global.FileTable = new List(); _ec.Object.Global.Plan = new TaskOrchestrationPlanReference(); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _hc.GetTrace().Info($"[{issue.Type}]{logOptions.LogMessageOverride ?? issue.Message}"); }); _ec.Setup(x => x.GetGitHubContext("workspace")).Returns(Path.Combine(_workFolder, "actions", "actions")); _dockerManager = new Mock(); diff --git a/src/Test/L0/Worker/ActionManifestManagerL0.cs b/src/Test/L0/Worker/ActionManifestManagerL0.cs index b6e3f855b34..75010748b96 100644 --- a/src/Test/L0/Worker/ActionManifestManagerL0.cs +++ b/src/Test/L0/Worker/ActionManifestManagerL0.cs @@ -670,7 +670,7 @@ public void Load_PluginAction() { Teardown(); } - } + } [Fact] [Trait("Level", "L0")] @@ -715,7 +715,7 @@ public void Load_CompositeActionNoUsing() //Assert var err = Assert.Throws(() => actionManifest.Load(_ec.Object, action_path)); Assert.Contains($"Fail to load {action_path}", err.Message); - _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Missing 'using' value. 'using' requires 'composite', 'docker', 'node12' or 'node16'.")), It.IsAny()), Times.Once); + _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Missing 'using' value. 'using' requires 'composite', 'docker', 'node12' or 'node16'.")), It.IsAny()), Times.Once); } finally { @@ -860,7 +860,7 @@ private void Setup([CallerMemberName] string name = "") _ec.Setup(x => x.ExpressionValues).Returns(new DictionaryContextData()); _ec.Setup(x => x.ExpressionFunctions).Returns(new List()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"{tag}{message}"); }); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _hc.GetTrace().Info($"[{issue.Type}]{logOptions.LogMessageOverride ?? issue.Message}"); }); } private void Teardown() diff --git a/src/Test/L0/Worker/ActionRunnerL0.cs b/src/Test/L0/Worker/ActionRunnerL0.cs index 586e9dad2d4..4942f568c7d 100644 --- a/src/Test/L0/Worker/ActionRunnerL0.cs +++ b/src/Test/L0/Worker/ActionRunnerL0.cs @@ -1,4 +1,4 @@ -using GitHub.DistributedTask.Expressions2; +using GitHub.DistributedTask.Expressions2; using GitHub.DistributedTask.ObjectTemplating.Tokens; using GitHub.DistributedTask.Pipelines; using GitHub.DistributedTask.Pipelines.ContextData; @@ -366,7 +366,7 @@ public async void WarnInvalidInputs() Assert.Equal("invalid1", finialInputs["invalid1"]); Assert.Equal("invalid2", finialInputs["invalid2"]); - _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input(s) 'invalid1', 'invalid2'")), It.IsAny()), Times.Once); + _ec.Verify(x => x.AddIssue(It.Is(s => s.Message.Contains("Unexpected input(s) 'invalid1', 'invalid2'")), It.IsAny()), Times.Once); } [Fact] @@ -485,7 +485,7 @@ private void Setup([CallerMemberName] string name = "") _ec.Setup(x => x.CancellationToken).Returns(_ecTokenSource.Token); _ec.Object.Global.Variables = new Variables(_hc, new Dictionary()); _ec.Setup(x => x.Write(It.IsAny(), It.IsAny())).Callback((string tag, string message) => { _hc.GetTrace().Info($"[{tag}]{message}"); }); - _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, string message) => { _hc.GetTrace().Info($"[{issue.Type}]{issue.Message ?? message}"); }); + _ec.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())).Callback((Issue issue, ExecutionContextLogOptions logOptions) => { _hc.GetTrace().Info($"[{issue.Type}]{logOptions.LogMessageOverride ?? issue.Message}"); }); _hc.SetSingleton(_actionManager.Object); _hc.SetSingleton(_handlerFactory.Object); diff --git a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs index 67631ed4bd7..40b067afa10 100644 --- a/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs +++ b/src/Test/L0/Worker/CreateStepSummaryCommandL0.cs @@ -247,12 +247,16 @@ private TestHostContext Setup([CallerMemberName] string name = "") WriteDebug = true, Variables = _variables, }); - _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((DTWebApi.Issue issue, string logMessage) => + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, ExecutionContextLogOptions logOptions) => { - _issues.Add(new Tuple(issue, logMessage)); - var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; - _trace.Info($"Issue '{issue.Type}': {message}"); + var resolvedMessage = issue.Message; + if (logOptions.WriteToLog && !string.IsNullOrEmpty(logOptions.LogMessageOverride)) + { + resolvedMessage = logOptions.LogMessageOverride; + } + _issues.Add(new(issue, resolvedMessage)); + _trace.Info($"Issue '{issue.Type}': {resolvedMessage}"); }); _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) .Callback((string tag, string message) => @@ -268,4 +272,4 @@ private TestHostContext Setup([CallerMemberName] string name = "") return hostContext; } } -} \ No newline at end of file +} diff --git a/src/Test/L0/Worker/ExecutionContextL0.cs b/src/Test/L0/Worker/ExecutionContextL0.cs index 649e66c863d..2bdbd9f44cc 100644 --- a/src/Test/L0/Worker/ExecutionContextL0.cs +++ b/src/Test/L0/Worker/ExecutionContextL0.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Linq; using System.Runtime.CompilerServices; @@ -52,42 +52,43 @@ public void AddIssue_CountWarningsErrors() // Act. ec.InitializeJob(jobRequest, CancellationToken.None); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); + // Flood the ExecutionContext with errors and warnings (past its max capacity of 10). + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); ec.Complete(); // Assert. - jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.ErrorCount == 15)), Times.AtLeastOnce); - jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.WarningCount == 14)), Times.AtLeastOnce); + jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.ErrorCount > 0)), Times.AtLeast(10)); + jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.WarningCount > 0)), Times.AtLeast(10)); jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Type == IssueType.Error).Count() == 10)), Times.AtLeastOnce); jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Type == IssueType.Warning).Count() == 10)), Times.AtLeastOnce); } @@ -190,9 +191,9 @@ public void AddIssue_TrimMessageSize() bigMessage += "a"; } - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = bigMessage }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = bigMessage }); - ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = bigMessage }); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = bigMessage }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = bigMessage }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = bigMessage }, ExecutionContextLogOptions.Default); ec.Complete(); @@ -203,6 +204,61 @@ public void AddIssue_TrimMessageSize() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "Worker")] + public void AddIssue_OverrideLogMessage() + { + using (TestHostContext hc = CreateTestContext()) + { + // Arrange: Create a job request message. + TaskOrchestrationPlanReference plan = new(); + TimelineReference timeline = new(); + Guid jobId = Guid.NewGuid(); + string jobName = "some job name"; + var jobRequest = new Pipelines.AgentJobRequestMessage(plan, timeline, jobId, jobName, jobName, null, null, null, new Dictionary(), new List(), new Pipelines.JobResources(), new Pipelines.ContextData.DictionaryContextData(), new Pipelines.WorkspaceOptions(), new List(), null, null, null, null); + jobRequest.Resources.Repositories.Add(new Pipelines.RepositoryResource() + { + Alias = Pipelines.PipelineConstants.SelfAlias, + Id = "github", + Version = "sha1" + }); + jobRequest.ContextData["github"] = new Pipelines.ContextData.DictionaryContextData(); + + // Arrange: Setup the paging logger. + var pagingLogger = new Mock(); + var jobServerQueue = new Mock(); + + hc.EnqueueInstance(pagingLogger.Object); + hc.SetSingleton(jobServerQueue.Object); + + var ec = new Runner.Worker.ExecutionContext(); + ec.Initialize(hc); + + // Act. + ec.InitializeJob(jobRequest, CancellationToken.None); + + var issueMessage = "Message embedded in issue."; + var overrideMessage = "Message override."; + var options = new ExecutionContextLogOptions(true, overrideMessage); + + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = issueMessage }, options); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = issueMessage }, options); + ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = issueMessage }, options); + + // Finally, add a variation that DOESN'T override the message. + ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = issueMessage }, ExecutionContextLogOptions.Default); + + ec.Complete(); + + // Assert. + jobServerQueue.Verify(x => x.QueueWebConsoleLine(It.IsAny(), It.IsAny(), It.IsAny()), Times.Exactly(4)); + jobServerQueue.Verify(x => x.QueueWebConsoleLine(It.IsAny(), It.Is(text => text.EndsWith(overrideMessage)), It.IsAny()), Times.Exactly(3)); + jobServerQueue.Verify(x => x.QueueWebConsoleLine(It.IsAny(), It.Is(text => text.EndsWith(issueMessage)), It.IsAny()), Times.Exactly(1)); + } + } + + [Fact] [Trait("Level", "L0")] [Trait("Category", "Worker")] @@ -242,13 +298,15 @@ public void AddIssue_AddStepAndLineNumberInformation() var embeddedStep = ec.CreateChild(Guid.NewGuid(), "action_1_pre", "action_1_pre", null, null, ActionRunStage.Main, isEmbedded: true); embeddedStep.Start(); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error annotation that should have step and line number information" }); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning annotation that should have step and line number information" }); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice annotation that should have step and line number information" }); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error annotation that should have step and line number information" }, ExecutionContextLogOptions.Default); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning annotation that should have step and line number information" }, ExecutionContextLogOptions.Default); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice annotation that should have step and line number information" }, ExecutionContextLogOptions.Default); - jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Error).Count() == 1)), Times.AtLeastOnce); - jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Warning).Count() == 1)), Times.AtLeastOnce); - jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Notice).Count() == 1)), Times.AtLeastOnce); + jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.IsAny()), Times.AtLeastOnce); + // Verify that Error/Warning/Notice issues added to embedded steps don't get sent up to the server. + jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Error).Count() == 1)), Times.Never); + jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Warning).Count() == 1)), Times.Never); + jobServerQueue.Verify(x => x.QueueTimelineRecordUpdate(It.IsAny(), It.Is(t => t.Issues.Where(i => i.Data.ContainsKey("stepNumber") && i.Data.ContainsKey("logFileLineNumber") && i.Type == IssueType.Notice).Count() == 1)), Times.Never); } } @@ -626,12 +684,12 @@ public void PublishStepTelemetry_RegularStep() ec.StepTelemetry.StepId = Guid.NewGuid(); ec.StepTelemetry.Stage = "main"; - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }); - ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + ec.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default); ec.Complete(); @@ -692,9 +750,9 @@ public void PublishStepTelemetry_EmbeddedStep() embeddedStep.StepTelemetry.Action = "actions/checkout"; embeddedStep.StepTelemetry.Ref = "v2"; - embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default); embeddedStep.PublishStepTelemetry(); @@ -756,9 +814,9 @@ public void PublishStepResult_EmbeddedStep() embeddedStep.StepTelemetry.Action = "actions/checkout"; embeddedStep.StepTelemetry.Ref = "v2"; - embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }); - embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Error, Message = "error" }, ExecutionContextLogOptions.Default); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Warning, Message = "warning" }, ExecutionContextLogOptions.Default); + embeddedStep.AddIssue(new Issue() { Type = IssueType.Notice, Message = "notice" }, ExecutionContextLogOptions.Default); ec.Complete(); @@ -927,7 +985,7 @@ public void ActionVariables_AddedToVarsContext() inputVarsContext["VARIABLE_2"] = new StringContextData("value2"); jobRequest.ContextData["vars"] = inputVarsContext; - // Arrange: Setup the paging logger. + // Arrange: Setup the paging logger. var pagingLogger1 = new Mock(); var jobServerQueue = new Mock(); hc.EnqueueInstance(pagingLogger1.Object); @@ -941,7 +999,7 @@ public void ActionVariables_AddedToVarsContext() var expected = new DictionaryContextData(); expected["VARIABLE_1"] = new StringContextData("value1"); expected["VARIABLE_2"] = new StringContextData("value1"); - + Assert.True(ExpressionValuesAssertEqual(expected, jobContext.ExpressionValues["vars"] as DictionaryContextData)); } } @@ -972,7 +1030,7 @@ public void ActionVariables_DebugUsingVars() inputVarsContext[Constants.Variables.Actions.RunnerDebug] = new StringContextData("true"); jobRequest.ContextData["vars"] = inputVarsContext; - // Arrange: Setup the paging logger. + // Arrange: Setup the paging logger. var pagingLogger1 = new Mock(); var jobServerQueue = new Mock(); hc.EnqueueInstance(pagingLogger1.Object); @@ -983,7 +1041,7 @@ public void ActionVariables_DebugUsingVars() jobContext.InitializeJob(jobRequest, CancellationToken.None); - + Assert.Equal("true", jobContext.Global.Variables.Get(Constants.Variables.Actions.StepDebug)); Assert.Equal("true", jobContext.Global.Variables.Get(Constants.Variables.Actions.RunnerDebug)); } @@ -1018,7 +1076,7 @@ public void ActionVariables_SecretsPrecedenceForDebugUsingVars() jobRequest.Variables[Constants.Variables.Actions.StepDebug] = "false"; jobRequest.Variables[Constants.Variables.Actions.RunnerDebug] = "false"; - // Arrange: Setup the paging logger. + // Arrange: Setup the paging logger. var pagingLogger1 = new Mock(); var jobServerQueue = new Mock(); hc.EnqueueInstance(pagingLogger1.Object); @@ -1029,7 +1087,7 @@ public void ActionVariables_SecretsPrecedenceForDebugUsingVars() jobContext.InitializeJob(jobRequest, CancellationToken.None); - + Assert.Equal("false", jobContext.Global.Variables.Get(Constants.Variables.Actions.StepDebug)); Assert.Equal("false", jobContext.Global.Variables.Get(Constants.Variables.Actions.RunnerDebug)); } @@ -1061,4 +1119,4 @@ private bool ExpressionValuesAssertEqual(DictionaryContextData expect, Dictionar return true; } } -} \ No newline at end of file +} diff --git a/src/Test/L0/Worker/OutputManagerL0.cs b/src/Test/L0/Worker/OutputManagerL0.cs index 8bac9393362..cf233bb5513 100644 --- a/src/Test/L0/Worker/OutputManagerL0.cs +++ b/src/Test/L0/Worker/OutputManagerL0.cs @@ -984,10 +984,15 @@ private TestHostContext Setup( { _onMatcherChanged = handler; }); - _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((DTWebApi.Issue issue, string logMessage) => + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, ExecutionContextLogOptions logOptions) => { - _issues.Add(new Tuple(issue, logMessage)); + var resolvedMessage = issue.Message; + if (logOptions.WriteToLog && !string.IsNullOrEmpty(logOptions.LogMessageOverride)) + { + resolvedMessage = logOptions.LogMessageOverride; + } + _issues.Add(new(issue, resolvedMessage)); }); _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) .Callback((string tag, string message) => diff --git a/src/Test/L0/Worker/SaveStateFileCommandL0.cs b/src/Test/L0/Worker/SaveStateFileCommandL0.cs index 45296c70978..2e7767e57c7 100644 --- a/src/Test/L0/Worker/SaveStateFileCommandL0.cs +++ b/src/Test/L0/Worker/SaveStateFileCommandL0.cs @@ -413,12 +413,16 @@ private TestHostContext Setup([CallerMemberName] string name = "") EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), WriteDebug = true, }); - _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((DTWebApi.Issue issue, string logMessage) => + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, ExecutionContextLogOptions logOptions) => { - _issues.Add(new Tuple(issue, logMessage)); - var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; - _trace.Info($"Issue '{issue.Type}': {message}"); + var resolvedMessage = issue.Message; + if (logOptions.WriteToLog && !string.IsNullOrEmpty(logOptions.LogMessageOverride)) + { + resolvedMessage = logOptions.LogMessageOverride; + } + _issues.Add(new(issue, resolvedMessage)); + _trace.Info($"Issue '{issue.Type}': {resolvedMessage}"); }); _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) .Callback((string tag, string message) => diff --git a/src/Test/L0/Worker/SetEnvFileCommandL0.cs b/src/Test/L0/Worker/SetEnvFileCommandL0.cs index 41f8499d582..805f9e85953 100644 --- a/src/Test/L0/Worker/SetEnvFileCommandL0.cs +++ b/src/Test/L0/Worker/SetEnvFileCommandL0.cs @@ -411,12 +411,16 @@ private TestHostContext Setup([CallerMemberName] string name = "") EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), WriteDebug = true, }); - _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((DTWebApi.Issue issue, string logMessage) => + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, ExecutionContextLogOptions logOptions) => { - _issues.Add(new Tuple(issue, logMessage)); - var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; - _trace.Info($"Issue '{issue.Type}': {message}"); + var resolvedMessage = issue.Message; + if (logOptions.WriteToLog && !string.IsNullOrEmpty(logOptions.LogMessageOverride)) + { + resolvedMessage = logOptions.LogMessageOverride; + } + _issues.Add(new(issue, resolvedMessage)); + _trace.Info($"Issue '{issue.Type}': {resolvedMessage}"); }); _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) .Callback((string tag, string message) => diff --git a/src/Test/L0/Worker/SetOutputFileCommandL0.cs b/src/Test/L0/Worker/SetOutputFileCommandL0.cs index 8af9695db46..9831f45de14 100644 --- a/src/Test/L0/Worker/SetOutputFileCommandL0.cs +++ b/src/Test/L0/Worker/SetOutputFileCommandL0.cs @@ -413,12 +413,16 @@ private TestHostContext Setup([CallerMemberName] string name = "") EnvironmentVariables = new Dictionary(VarUtil.EnvironmentVariableKeyComparer), WriteDebug = true, }); - _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) - .Callback((DTWebApi.Issue issue, string logMessage) => + _executionContext.Setup(x => x.AddIssue(It.IsAny(), It.IsAny())) + .Callback((DTWebApi.Issue issue, ExecutionContextLogOptions logOptions) => { - _issues.Add(new Tuple(issue, logMessage)); - var message = !string.IsNullOrEmpty(logMessage) ? logMessage : issue.Message; - _trace.Info($"Issue '{issue.Type}': {message}"); + var resolvedMessage = issue.Message; + if (logOptions.WriteToLog && !string.IsNullOrEmpty(logOptions.LogMessageOverride)) + { + resolvedMessage = logOptions.LogMessageOverride; + } + _issues.Add(new(issue, resolvedMessage)); + _trace.Info($"Issue '{issue.Type}': {resolvedMessage}"); }); _executionContext.Setup(x => x.Write(It.IsAny(), It.IsAny())) .Callback((string tag, string message) => @@ -430,8 +434,8 @@ private TestHostContext Setup([CallerMemberName] string name = "") _executionContext.Setup(x => x.SetOutput(It.IsAny(), It.IsAny(), out reference)) .Callback((string name, string value, out string reference) => { - reference = value; - _outputs[name] = value; + reference = value; + _outputs[name] = value; }); // SetOutputFileCommand diff --git a/src/Test/L0/Worker/StepsRunnerL0.cs b/src/Test/L0/Worker/StepsRunnerL0.cs index 3ff1e304a4a..d17ea3e2ccc 100644 --- a/src/Test/L0/Worker/StepsRunnerL0.cs +++ b/src/Test/L0/Worker/StepsRunnerL0.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.Collections.Generic; using System.Globalization; using System.Linq; @@ -337,7 +337,7 @@ public async Task SkipsAfterFailureOnlyBaseOnCondition() // Act. await _stepsRunner.RunAsync(jobContext: _ec.Object); - // Assert. + // Assert. Assert.Equal(2, variableSet.Step.Length); variableSet.Step[0].Verify(x => x.RunAsync()); variableSet.Step[1].Verify(x => x.RunAsync(), variableSet.Expected ? Times.Once() : Times.Never()); @@ -590,7 +590,7 @@ private Mock CreateStep(TestHostContext hc, TaskResult result, st step.Setup(x => x.Condition).Returns(condition); step.Setup(x => x.ContinueOnError).Returns(new BooleanToken(null, null, null, continueOnError)); step.Setup(x => x.Action) - .Returns(new DistributedTask.Pipelines.ActionStep() + .Returns(new GitHub.DistributedTask.Pipelines.ActionStep() { Name = name, Id = Guid.NewGuid(), diff --git a/src/Test/TestData/timelinerecord_json_samples.yml b/src/Test/TestData/timelinerecord_json_samples.yml new file mode 100644 index 00000000000..124c4593678 --- /dev/null +++ b/src/Test/TestData/timelinerecord_json_samples.yml @@ -0,0 +1,70 @@ +minimal: | + { "Name": "minimal" } +invalid-attempt-value: | + { + "Name": "invalid-attempt-value", + "Attempt": -99 + } +zero-attempt-value: | + { + "Name": "zero-attempt-value", + "Attempt": 0 + } +legacy-nulls: | + { + "Name": "legacy-nulls", + "ErrorCount": null, + "WarningCount": null, + "NoticeCount": null + } +missing-counts: | + { + "Name": "missing-counts" + } +non-zero-counts: | + { + "Name": "non-zero-counts", + "ErrorCount": 10, + "WarningCount": 20, + "NoticeCount": 30 + } +explicit-null-collections: | + { + "Name": "explicit-null-collections", + "Issues": null, + "PreviousAttempts": null, + "Variables": null + } +lean: | + { + "Id": "00000000-0000-0000-0000-000000000000", + "Name": "lean", + "LastModified": "\/Date(1679073003252+0000)\/", + "Issues": [ + { + "Type": 0, + "Category": null, + "Message": null, + "IsInfrastructureIssue": null + } + ], + "Variables": [ + { "Key": "x", "Value": { "Value": "1" } }, + { "Key": "y", "Value": { "Value": "2" } }, + { "Key": "z", "Value": { "Value": "3" } } + ], + "Attempt": 4, + "PreviousAttempts": [ + { "Attempt": 1 }, + { "Attempt": 2 }, + { "Attempt": 3 } + ] + } +duplicate-variable-keys: | + { + "Name": "duplicate-variable-keys", + "Variables": [ + { "Key": "aaa", "Value": { "Value": "a.1" } }, + { "Key": "AAA", "Value": { "Value": "a.2" } } + ] + } \ No newline at end of file From 12252faaabc4749904e6a7f9243d19f9fa5482cf Mon Sep 17 00:00:00 2001 From: Gabriel Date: Thu, 11 May 2023 19:37:56 +0300 Subject: [PATCH 140/145] Add --no-default-labels config option to self-hosted runners (#2443) * Add --no-default-labels option Signed-off-by: Gabriel Adrian Samfira * Add tests Signed-off-by: Gabriel Adrian Samfira * . --------- Signed-off-by: Gabriel Adrian Samfira Co-authored-by: Tingluo Huang --- src/Runner.Common/Constants.cs | 1 + src/Runner.Listener/CommandSettings.cs | 8 +- .../Configuration/ConfigurationManager.cs | 34 ++++-- src/Runner.Listener/Runner.cs | 3 +- .../Configuration/ConfigurationManagerL0.cs | 101 ++++++++++++++++++ 5 files changed, 133 insertions(+), 14 deletions(-) diff --git a/src/Runner.Common/Constants.cs b/src/Runner.Common/Constants.cs index d14e3e88d05..95b3039d00e 100644 --- a/src/Runner.Common/Constants.cs +++ b/src/Runner.Common/Constants.cs @@ -132,6 +132,7 @@ public static class Flags public static readonly string GenerateServiceConfig = "generateServiceConfig"; public static readonly string Help = "help"; public static readonly string Local = "local"; + public static readonly string NoDefaultLabels = "no-default-labels"; public static readonly string Replace = "replace"; public static readonly string DisableUpdate = "disableupdate"; public static readonly string Once = "once"; // Keep this around since customers still relies on it diff --git a/src/Runner.Listener/CommandSettings.cs b/src/Runner.Listener/CommandSettings.cs index cce711696cb..1bc54837864 100644 --- a/src/Runner.Listener/CommandSettings.cs +++ b/src/Runner.Listener/CommandSettings.cs @@ -29,8 +29,8 @@ public sealed class CommandSettings private readonly Dictionary validOptions = new() { // Valid configure flags and args - [Constants.Runner.CommandLine.Commands.Configure] = - new string[] + [Constants.Runner.CommandLine.Commands.Configure] = + new string[] { Constants.Runner.CommandLine.Flags.DisableUpdate, Constants.Runner.CommandLine.Flags.Ephemeral, @@ -38,6 +38,7 @@ public sealed class CommandSettings Constants.Runner.CommandLine.Flags.Replace, Constants.Runner.CommandLine.Flags.RunAsService, Constants.Runner.CommandLine.Flags.Unattended, + Constants.Runner.CommandLine.Flags.NoDefaultLabels, Constants.Runner.CommandLine.Args.Auth, Constants.Runner.CommandLine.Args.Labels, Constants.Runner.CommandLine.Args.MonitorSocketAddress, @@ -85,6 +86,7 @@ public sealed class CommandSettings public bool Ephemeral => TestFlag(Constants.Runner.CommandLine.Flags.Ephemeral); public bool GenerateServiceConfig => TestFlag(Constants.Runner.CommandLine.Flags.GenerateServiceConfig); public bool Help => TestFlag(Constants.Runner.CommandLine.Flags.Help); + public bool NoDefaultLabels => TestFlag(Constants.Runner.CommandLine.Flags.NoDefaultLabels); public bool Unattended => TestFlag(Constants.Runner.CommandLine.Flags.Unattended); public bool Version => TestFlag(Constants.Runner.CommandLine.Flags.Version); public bool RemoveLocalConfig => TestFlag(Constants.Runner.CommandLine.Flags.Local); @@ -182,7 +184,7 @@ public string GetCommandName() { command = Constants.Runner.CommandLine.Commands.Warmup; } - + return command; } diff --git a/src/Runner.Listener/Configuration/ConfigurationManager.cs b/src/Runner.Listener/Configuration/ConfigurationManager.cs index 0c835504a00..eb3bbdbb3c9 100644 --- a/src/Runner.Listener/Configuration/ConfigurationManager.cs +++ b/src/Runner.Listener/Configuration/ConfigurationManager.cs @@ -259,7 +259,7 @@ public async Task ConfigureAsync(CommandSettings command) if (command.GetReplace()) { // Update existing agent with new PublicKey, agent version. - agent = UpdateExistingAgent(agent, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate); + agent = UpdateExistingAgent(agent, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate, command.NoDefaultLabels); try { @@ -293,7 +293,7 @@ public async Task ConfigureAsync(CommandSettings command) else { // Create a new agent. - agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate); + agent = CreateNewAgent(runnerSettings.AgentName, publicKey, userLabels, runnerSettings.Ephemeral, command.DisableUpdate, command.NoDefaultLabels); try { @@ -554,7 +554,7 @@ private ICredentialProvider GetCredentialProvider(CommandSettings command, strin } - private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate) + private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate, bool noDefaultLabels) { ArgUtil.NotNull(agent, nameof(agent)); agent.Authorization = new TaskAgentAuthorization @@ -571,9 +571,16 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, agent.Labels.Clear(); - agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); - agent.Labels.Add(new AgentLabel(VarUtil.OS, LabelType.System)); - agent.Labels.Add(new AgentLabel(VarUtil.OSArchitecture, LabelType.System)); + if (!noDefaultLabels) + { + agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OS, LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OSArchitecture, LabelType.System)); + } + else if (userLabels.Count == 0) + { + throw new NotSupportedException("Disabling default labels via --no-default-labels without specifying --labels is not supported"); + } foreach (var userLabel in userLabels) { @@ -583,7 +590,7 @@ private TaskAgent UpdateExistingAgent(TaskAgent agent, RSAParameters publicKey, return agent; } - private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate) + private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet userLabels, bool ephemeral, bool disableUpdate, bool noDefaultLabels) { TaskAgent agent = new(agentName) { @@ -598,9 +605,16 @@ private TaskAgent CreateNewAgent(string agentName, RSAParameters publicKey, ISet DisableUpdate = disableUpdate }; - agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); - agent.Labels.Add(new AgentLabel(VarUtil.OS, LabelType.System)); - agent.Labels.Add(new AgentLabel(VarUtil.OSArchitecture, LabelType.System)); + if (!noDefaultLabels) + { + agent.Labels.Add(new AgentLabel("self-hosted", LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OS, LabelType.System)); + agent.Labels.Add(new AgentLabel(VarUtil.OSArchitecture, LabelType.System)); + } + else if (userLabels.Count == 0) + { + throw new NotSupportedException("Disabling default labels via --no-default-labels without specifying --labels is not supported"); + } foreach (var userLabel in userLabels) { diff --git a/src/Runner.Listener/Runner.cs b/src/Runner.Listener/Runner.cs index e71ba9cb5b0..c367f88524d 100644 --- a/src/Runner.Listener/Runner.cs +++ b/src/Runner.Listener/Runner.cs @@ -683,7 +683,8 @@ private void PrintUsage(CommandSettings command) --token string Registration token. Required if unattended --name string Name of the runner to configure (default {Environment.MachineName ?? "myrunner"}) --runnergroup string Name of the runner group to add this runner to (defaults to the default runner group) - --labels string Extra labels in addition to the default: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}' + --labels string Custom labels that will be added to the runner. This option is mandatory if --no-default-labels is used. + --no-default-labels Disables adding the default labels: 'self-hosted,{Constants.Runner.Platform},{Constants.Runner.PlatformArchitecture}' --local Removes the runner config files from your local machine. Used as an option to the remove command --work string Relative runner work directory (default {Constants.Path.WorkDirectory}) --replace Replace any existing runner with the same name (default false) diff --git a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs index 12918390073..628e2cabaa1 100644 --- a/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs +++ b/src/Test/L0/Listener/Configuration/ConfigurationManagerL0.cs @@ -206,6 +206,107 @@ public async Task CanEnsureConfigure() } } + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "ConfigurationManagement")] + public async Task ConfigureErrorDefaultLabelsDisabledWithNoCustomLabels() + { + using (TestHostContext tc = CreateTestContext()) + { + Tracing trace = tc.GetTrace(); + + trace.Info("Creating config manager"); + IConfigurationManager configManager = new ConfigurationManager(); + configManager.Initialize(tc); + + trace.Info("Preparing command line arguments"); + var command = new CommandSettings( + tc, + new[] + { + "configure", + "--url", _expectedServerUrl, + "--name", _expectedAgentName, + "--runnergroup", _secondRunnerGroupName, + "--work", _expectedWorkFolder, + "--auth", _expectedAuthType, + "--token", _expectedToken, + "--no-default-labels", + "--ephemeral", + "--disableupdate", + "--unattended", + }); + trace.Info("Constructed."); + _store.Setup(x => x.IsConfigured()).Returns(false); + _configMgrAgentSettings = null; + + trace.Info("Ensuring configure fails if default labels are disabled and no custom labels are set"); + var ex = await Assert.ThrowsAsync(() => configManager.ConfigureAsync(command)); + + Assert.Contains("--no-default-labels without specifying --labels is not supported", ex.Message); + } + } + + [Fact] + [Trait("Level", "L0")] + [Trait("Category", "ConfigurationManagement")] + public async Task ConfigureDefaultLabelsDisabledWithCustomLabels() + { + using (TestHostContext tc = CreateTestContext()) + { + Tracing trace = tc.GetTrace(); + + trace.Info("Creating config manager"); + IConfigurationManager configManager = new ConfigurationManager(); + configManager.Initialize(tc); + + var userLabels = "userlabel1,userlabel2"; + + trace.Info("Preparing command line arguments"); + var command = new CommandSettings( + tc, + new[] + { + "configure", + "--url", _expectedServerUrl, + "--name", _expectedAgentName, + "--runnergroup", _secondRunnerGroupName, + "--work", _expectedWorkFolder, + "--auth", _expectedAuthType, + "--token", _expectedToken, + "--labels", userLabels, + "--no-default-labels", + "--ephemeral", + "--disableupdate", + "--unattended", + }); + trace.Info("Constructed."); + _store.Setup(x => x.IsConfigured()).Returns(false); + _configMgrAgentSettings = null; + + trace.Info("Ensuring all the required parameters are available in the command line parameter"); + await configManager.ConfigureAsync(command); + + _store.Setup(x => x.IsConfigured()).Returns(true); + + trace.Info("Configured, verifying all the parameter value"); + var s = configManager.LoadSettings(); + Assert.NotNull(s); + Assert.True(s.ServerUrl.Equals(_expectedServerUrl)); + Assert.True(s.AgentName.Equals(_expectedAgentName)); + Assert.True(s.PoolId.Equals(_secondRunnerGroupId)); + Assert.True(s.WorkFolder.Equals(_expectedWorkFolder)); + Assert.True(s.Ephemeral.Equals(true)); + + // validate GetAgentPoolsAsync gets called twice with automation pool type + _runnerServer.Verify(x => x.GetAgentPoolsAsync(It.IsAny(), It.Is(p => p == TaskAgentPoolType.Automation)), Times.Exactly(2)); + + var expectedLabels = userLabels.Split(",").ToList(); + + _runnerServer.Verify(x => x.AddAgentAsync(It.IsAny(), It.Is(a => a.Labels.Select(x => x.Name).ToHashSet().SetEquals(expectedLabels))), Times.Once); + } + } + [Fact] [Trait("Level", "L0")] [Trait("Category", "ConfigurationManagement")] From 6e2f0b09e0b54b815f10d01433c6f515f242d7bb Mon Sep 17 00:00:00 2001 From: Vallie Joseph Date: Thu, 11 May 2023 14:57:24 -0400 Subject: [PATCH 141/145] Adding Consistency to 'Failed To Resolve Action Download Info' Infrastructure Error Flagging (#2488) * adding extra catch for download failure in composite actions * Adding infra error * Adding error handling centralizing * updating try catch bubbling * cleaning up commits * cleaning up commits * cleaning up commits * updating bubbler * cleaning up test files * Fixing linting errors * updating exception bubble * reverting composite * updating catch to not exclude other exceptions * removing uneeded import * Update src/Runner.Worker/ActionRunner.cs Co-authored-by: Tingluo Huang * Update src/Runner.Worker/ActionManager.cs Co-authored-by: Tingluo Huang * Update src/Runner.Worker/ActionManager.cs Co-authored-by: Tingluo Huang * Update src/Runner.Worker/ActionManager.cs Co-authored-by: Tingluo Huang * Update src/Runner.Worker/ActionManager.cs Co-authored-by: Tingluo Huang * moving download out of for loop; reverting exception wrap * Update src/Runner.Worker/ActionManager.cs Co-authored-by: Tingluo Huang * Adding blank lines back * Adding blank lines back * removing uneeded catch for download fail * adding var back for consistency * formatting clean --------- Co-authored-by: Tingluo Huang --- src/Runner.Worker/ActionManager.cs | 15 ++++++++++++++- src/Runner.Worker/JobExtension.cs | 8 -------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/src/Runner.Worker/ActionManager.cs b/src/Runner.Worker/ActionManager.cs index efacc138ab1..0de4d4d0fbd 100644 --- a/src/Runner.Worker/ActionManager.cs +++ b/src/Runner.Worker/ActionManager.cs @@ -18,6 +18,7 @@ using WebApi = GitHub.DistributedTask.WebApi; using Pipelines = GitHub.DistributedTask.Pipelines; using PipelineTemplateConstants = GitHub.DistributedTask.Pipelines.ObjectTemplating.PipelineTemplateConstants; +using GitHub.DistributedTask.WebApi; namespace GitHub.Runner.Worker { @@ -101,7 +102,19 @@ public sealed class ActionManager : RunnerService, IActionManager } IEnumerable actions = steps.OfType(); executionContext.Output("Prepare all required actions"); - var result = await PrepareActionsRecursiveAsync(executionContext, state, actions, depth, rootStepId); + PrepareActionsState result = new PrepareActionsState(); + try + { + result = await PrepareActionsRecursiveAsync(executionContext, state, actions, depth, rootStepId); + } + catch (FailedToResolveActionDownloadInfoException ex) + { + // Log the error and fail the PrepareActionsAsync Initialization. + Trace.Error($"Caught exception from PrepareActionsAsync Initialization: {ex}"); + executionContext.InfrastructureError(ex.Message); + executionContext.Result = TaskResult.Failed; + throw; + } if (!FeatureManager.IsContainerHooksEnabled(executionContext.Global.Variables)) { if (state.ImagesToPull.Count > 0) diff --git a/src/Runner.Worker/JobExtension.cs b/src/Runner.Worker/JobExtension.cs index 42e6fea816b..75789f89901 100644 --- a/src/Runner.Worker/JobExtension.cs +++ b/src/Runner.Worker/JobExtension.cs @@ -431,14 +431,6 @@ public async Task> InitializeJob(IExecutionContext jobContext, Pipel context.Result = TaskResult.Canceled; throw; } - catch (FailedToResolveActionDownloadInfoException ex) - { - // Log the error and fail the JobExtension Initialization. - Trace.Error($"Caught exception from JobExtenion Initialization: {ex}"); - context.InfrastructureError(ex.Message); - context.Result = TaskResult.Failed; - throw; - } catch (Exception ex) { // Log the error and fail the JobExtension Initialization. From 822237fb0ce1e992c8cb1e3735d623007625e13d Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Mon, 4 Apr 2022 11:50:37 +0200 Subject: [PATCH 142/145] added defaults shell and working-directory to composite --- src/Runner.Worker/Handlers/CompositeActionHandler.cs | 7 +++++++ src/Runner.Worker/action_yaml.json | 2 +- src/update.finished | 0 3 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 src/update.finished diff --git a/src/Runner.Worker/Handlers/CompositeActionHandler.cs b/src/Runner.Worker/Handlers/CompositeActionHandler.cs index aa749b2fe17..e019dad03ca 100644 --- a/src/Runner.Worker/Handlers/CompositeActionHandler.cs +++ b/src/Runner.Worker/Handlers/CompositeActionHandler.cs @@ -92,6 +92,13 @@ public async Task RunAsync(ActionRunStage stage) } ExecutionContext.StepTelemetry.Type = "composite"; + // save job defaults run to be restored after the composite run + IDictionary jobDefaultsRun = new Dictionary(); + if (ExecutionContext.Global.JobDefaults.ContainsKey("run")) + { + jobDefaultsRun = ExecutionContext.Global.JobDefaults["run"]; + } + try { // Inputs of the composite step diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index 40813f6aa00..bc02818921a 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -1,4 +1,4 @@ -{ +{ "definitions": { "action-root": { "description": "Action file", diff --git a/src/update.finished b/src/update.finished new file mode 100644 index 00000000000..e69de29bb2d From 274b80540f179ce72b85653d74c0f612865670e8 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Fri, 12 May 2023 15:23:53 +0200 Subject: [PATCH 143/145] auto update leftover removed --- src/update.finished | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 src/update.finished diff --git a/src/update.finished b/src/update.finished deleted file mode 100644 index e69de29bb2d..00000000000 From 19f867d07bbf330da63e71487a8e0b873ffa8074 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Fri, 12 May 2023 15:51:39 +0200 Subject: [PATCH 144/145] encode every changed file to utf8 --- src/Runner.Worker/Handlers/ScriptHandler.cs | 2 +- src/Runner.Worker/action_yaml.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index f214cf15985..7328859a313 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -1,4 +1,4 @@ -using System; +using System; using System.IO; using System.Linq; using System.Text; diff --git a/src/Runner.Worker/action_yaml.json b/src/Runner.Worker/action_yaml.json index bc02818921a..40813f6aa00 100644 --- a/src/Runner.Worker/action_yaml.json +++ b/src/Runner.Worker/action_yaml.json @@ -1,4 +1,4 @@ -{ +{ "definitions": { "action-root": { "description": "Action file", From e4b8e8985d9b949ea57b9d57c8a461c353c08f29 Mon Sep 17 00:00:00 2001 From: Nikola Jokic Date: Tue, 23 May 2023 13:15:40 +0200 Subject: [PATCH 145/145] Add null reference guard --- src/Runner.Worker/Handlers/ScriptHandler.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Runner.Worker/Handlers/ScriptHandler.cs b/src/Runner.Worker/Handlers/ScriptHandler.cs index 7328859a313..b2a1eb2d06d 100644 --- a/src/Runner.Worker/Handlers/ScriptHandler.cs +++ b/src/Runner.Worker/Handlers/ScriptHandler.cs @@ -71,14 +71,14 @@ protected override void PrintActionDetails(ActionRunStage stage) if (!Inputs.TryGetValue("shell", out shell) || string.IsNullOrEmpty(shell)) { - if (ExecutionContext.Global.CompositeDefaults.TryGetValue("run", out var runCompositeDefaults)) + if (ExecutionContext.Global.CompositeDefaults != null && ExecutionContext.Global.CompositeDefaults.TryGetValue("run", out var runCompositeDefaults)) { if (runCompositeDefaults.TryGetValue("shell", out shell)) { ExecutionContext.Debug("Overwrite 'shell' base on composite defaults."); } } - else if (ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) + else if (ExecutionContext.Global.JobDefaults != null && ExecutionContext.Global.JobDefaults.TryGetValue("run", out var runDefaults)) { if (runDefaults.TryGetValue("shell", out shell)) {