Redact ?token= querystring parameter from GHES <3.8 archive URLs in logs

RELEASENOTES.md

@@ -1,2 +1,3 @@
 
-- Skip GitHub status checking on startup if environment variable `GEI_SKIP_STATUS_CHECK` is set to true.
+- Skip GitHub status checking on startup if environment variable `GEI_SKIP_STATUS_CHECK` is set to `true`
+- Redact `?token=` querystring parameter from GHES <3.8 archive URLs in logs

src/Octoshift/Services/OctoLogger.cs

@@ -3,6 +3,7 @@
 using System.IO;
 using System.Linq;
 using System.Net.Http;
+using System.Text.RegularExpressions;
 
 namespace OctoshiftCLI.Services;
 
@@ -18,6 +19,7 @@ internal static class LogLevel
 public class OctoLogger
 {
     public virtual bool Verbose { get; set; }
+    private readonly HashSet<Regex> _redactionPatterns = new();
     private readonly HashSet<string> _secrets = new();
     private readonly string _logFilePath;
     private readonly string _verboseFilePath;
@@ -59,7 +61,7 @@ public OctoLogger(Action<string> writeToLog, Action<string> writeToVerboseLog, A
     private void Log(string msg, string level)
     {
         var output = FormatMessage(msg, level);
-        output = MaskSecrets(output);
+        output = Redact(output);
         if (level == LogLevel.ERROR)
         {
             _writeToConsoleError(output);
@@ -78,7 +80,7 @@ private string FormatMessage(string msg, string level)
         return $"[{timeFormat}] [{level}] {msg}\n";
     }
 
-    private string MaskSecrets(string msg)
+    private string Redact(string msg)
     {
         var result = msg;
 
@@ -88,6 +90,11 @@ private string MaskSecrets(string msg)
                 .Replace(Uri.EscapeDataString(secret), "***");
         }
 
+        foreach (var redactionPattern in _redactionPatterns)
+        {
+            result = redactionPattern.Replace(result, "***");
+        }
+
         return result;
     }
 
@@ -117,14 +124,14 @@ public virtual void LogError(Exception ex)
         var verboseMessage = ex is HttpRequestException httpEx ? $"[HTTP ERROR {(int?)httpEx.StatusCode}] {ex}" : ex.ToString();
         var logMessage = Verbose ? verboseMessage : ex is OctoshiftCliException ? ex.Message : GENERIC_ERROR_MESSAGE;
 
-        var output = MaskSecrets(FormatMessage(logMessage, LogLevel.ERROR));
+        var output = Redact(FormatMessage(logMessage, LogLevel.ERROR));
 
         Console.ForegroundColor = ConsoleColor.Red;
         _writeToConsoleError(output);
         Console.ResetColor();
 
         _writeToLog(output);
-        _writeToVerboseLog(MaskSecrets(FormatMessage(verboseMessage, LogLevel.ERROR)));
+        _writeToVerboseLog(Redact(FormatMessage(verboseMessage, LogLevel.ERROR)));
     }
 
     public virtual void LogVerbose(string msg)
@@ -137,7 +144,7 @@ public virtual void LogVerbose(string msg)
         }
         else
         {
-            _writeToVerboseLog(MaskSecrets(FormatMessage(msg, LogLevel.VERBOSE)));
+            _writeToVerboseLog(Redact(FormatMessage(msg, LogLevel.VERBOSE)));
         }
     }
 
@@ -157,4 +164,9 @@ public virtual void LogSuccess(string msg)
     }
 
     public virtual void RegisterSecret(string secret) => _secrets.Add(secret);
+
+    public virtual void AddRedactionPattern(Regex pattern)
+    {
+        _redactionPatterns.Add(pattern);
+    }
 }

src/OctoshiftCLI.Tests/Octoshift/Services/OctoLoggerTests.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Net;
 using System.Net.Http;
+using System.Text.RegularExpressions;
 using FluentAssertions;
 using OctoshiftCLI.Services;
 using Xunit;
@@ -60,6 +61,31 @@ public void Secrets_Should_Be_Masked_From_Logs_And_Console()
         _consoleError.Should().NotContain(urlEncodedSecret);
     }
 
+    [Fact]
+    public void Redaction_Patterns_Should_Be_Replaced_In_Logs_And_Console()
+    {
+        var password = "hunter2";
+
+        _octoLogger.AddRedactionPattern(new Regex("hunter\\d", RegexOptions.IgnoreCase));
+
+        _octoLogger.Verbose = false;
+        _octoLogger.LogInformation($"Don't tell anyone that {password} is my password");
+        _octoLogger.LogVerbose($"Don't tell anyone that {password} is my password");
+        _octoLogger.LogWarning($"Don't tell anyone that {password} is my password");
+        _octoLogger.LogSuccess($"Don't tell anyone that {password} is my password");
+        _octoLogger.LogError($"Don't tell anyone that {password} is my password");
+        _octoLogger.LogError(new OctoshiftCliException($"Don't tell anyone that {password} is my password"));
+        _octoLogger.LogError(new InvalidOperationException($"Don't tell anyone that {password} is my password"));
+
+        _octoLogger.Verbose = true;
+        _octoLogger.LogVerbose($"Don't tell anyone that {password} is my password");
+
+        _consoleOutput.Should().NotContain(password);
+        _logOutput.Should().NotContain(password);
+        _verboseLogOutput.Should().NotContain(password);
+        _consoleError.Should().NotContain(password);
+    }
+
     [Fact]
     public void LogError_For_OctoshiftCliException_Should_Log_Exception_Message_In_Non_Verbose_Mode()
     {

src/gei/Commands/MigrateRepo/MigrateRepoCommand.cs

@@ -1,5 +1,6 @@
 using System;
 using System.CommandLine;
+using System.Text.RegularExpressions;
 using Microsoft.Extensions.DependencyInjection;
 using OctoshiftCLI.Commands;
 using OctoshiftCLI.Contracts;
@@ -188,6 +189,8 @@ public override MigrateRepoCommandHandler BuildHandler(MigrateRepoCommandArgs ar
             var ghesVersionChecker = ghesVersionCheckerFactory.Create(ghesApi);
             var warningsCountLogger = sp.GetRequiredService<WarningsCountLogger>();
 
+            log.AddRedactionPattern(new Regex("\\?token=[A-Z0-9]{29}"));
+
             return new MigrateRepoCommandHandler(
                 log,
                 ghesApi,