Serilog console themes + quick CSV tokenizer -> nicer query results

src/SeqCli/Cli/Commands/QueryCommand.cs

@@ -18,7 +18,9 @@
 using Seq.Api;
 using Seq.Api.Model.Signals;
 using SeqCli.Cli.Features;
+using SeqCli.Config;
 using SeqCli.Connection;
+using SeqCli.Csv;
 using Serilog;
 
 namespace SeqCli.Cli.Commands
@@ -33,14 +35,17 @@ class QueryCommand : Command
         readonly SignalExpressionFeature _signal;
         string _query;
         int? _timeoutMS;
+        bool _noColor;
 
-        public QueryCommand(SeqConnectionFactory connectionFactory)
+        public QueryCommand(SeqConnectionFactory connectionFactory, SeqCliConfig config)
         {
             _connectionFactory = connectionFactory ?? throw new ArgumentNullException(nameof(connectionFactory));
             Options.Add("q=|query=", "The query to execute", v => _query = v);
             _range = Enable<DateRangeFeature>();
             _signal = Enable<SignalExpressionFeature>();
             Options.Add("timeout=", "The query execution timeout in milliseconds", v => _timeoutMS = int.Parse(v));
+            Options.Add("no-color", "Don't colorize text output", v => _noColor = true);
+            _noColor = config.Output.DisableColor;
             _connection = Enable<ConnectionFeature>();
         }
 
@@ -58,7 +63,14 @@ protected override async Task<int> Run()
             // remove the `.Value` when _Seq.Api_ is updated to reflect this.
             // ReSharper disable once PossibleInvalidOperationException
             var result = await QueryCsvAsync(connection, _query, _range.Start, _range.End, _signal.Signal, _timeoutMS);
-            Console.WriteLine(result);
+            if (_noColor)
+            {
+                Console.Write(result);
+                return 0;
+            }
+
+            var tokens = new CsvTokenizer().Tokenize(result);
+            CsvWriter.WriteCsv(tokens, OutputFormatFeature.ConsoleTheme, Console.Out, true);
 
             return 0;
         }

src/SeqCli/Cli/Features/OutputFormatFeature.cs

@@ -30,6 +30,8 @@ public OutputFormatFeature(SeqCliOutputConfig outputConfig)
             _noColor = outputConfig.DisableColor;
         }
 
+        public static ConsoleTheme ConsoleTheme => SystemConsoleTheme.Literate;
+
         public override void Enable(OptionSet options)
         {
             options.Add(
@@ -50,7 +52,7 @@ public Logger CreateOutputLogger()
             else
                 outputConfiguration.WriteTo.Console(
                     outputTemplate: "[{Timestamp:o} {Level:u3}] {Message:lj} {Properties:j}{NewLine}{Exception}",
-                    theme: _noColor ? ConsoleTheme.None : SystemConsoleTheme.Literate);
+                    theme: _noColor ? ConsoleTheme.None : ConsoleTheme);
 
             return outputConfiguration.CreateLogger();
         }

src/SeqCli/Csv/CsvToken.cs

@@ -0,0 +1,15 @@
+namespace SeqCli.Csv
+{
+    enum CsvToken
+    {
+        None,
+        Newline,
+        DoubleQuote,
+        Comma,
+        Number,
+        Boolean,
+        Null,
+        Text,
+        EscapedDoubleQuote
+    }
+}

src/SeqCli/Csv/CsvTokenizer.cs

@@ -0,0 +1,143 @@
+using System.Collections.Generic;
+using System.Globalization;
+using Superpower;
+using Superpower.Model;
+using Superpower.Parsers;
+
+namespace SeqCli.Csv
+{
+    class CsvTokenizer : Tokenizer<CsvToken>
+    {
+        static readonly TextParser<TextSpan> Content = Span.While(ch => ch != '"');
+
+        protected override IEnumerable<Result<CsvToken>> Tokenize(TextSpan span)
+        {
+            var next = SkipInsignificant(span);
+            if (!next.HasValue)
+                yield break;
+
+            do
+            {
+                // Here we're always either at the beginning of a line, or behind a comma.
+                if (next.Value == '"')
+                {
+                    yield return Result.Value(CsvToken.DoubleQuote, next.Location, next.Remainder);
+                    next = next.Remainder.ConsumeChar();
+                    if (!next.HasValue) yield break;
+
+                    var text = Content(next.Location);
+                    while (text.HasValue)
+                    {
+                        if (text.Value.Length > 0)
+                        {
+                            if (TryMatchSpecialContent(text.Value, out var specialTokenType) &&
+                                !IsEscapedDoubleQuote(text.Remainder))
+                                yield return Result.Value(specialTokenType, text.Location, text.Remainder);
+                            else
+                                yield return Result.Value(CsvToken.Text, text.Location, text.Remainder);
+                        }
+
+                        next = text.Remainder.ConsumeChar();
+                        if (!next.HasValue) yield break;
+
+                        if (next.Value != '"')
+                        {
+                            yield return Result.Empty<CsvToken>(next.Location, new[] {"double-quote"});
+                            yield break;
+                        }
+
+                        var lookahead = next.Remainder.ConsumeChar();
+                        if (lookahead.HasValue && lookahead.Value == '"')
+                        {
+                            yield return Result.Value(CsvToken.EscapedDoubleQuote, next.Location, lookahead.Remainder);
+                            next = lookahead.Remainder.ConsumeChar();
+                            if (!next.HasValue) yield break;
+                        }
+                        else
+                        {
+                            yield return Result.Value(CsvToken.DoubleQuote, next.Location, next.Remainder);
+                            next = next.Remainder.ConsumeChar();
+                            if (!next.HasValue) yield break;
+                            break; // Done with the content
+                        }
+
+                        text = Content(next.Location);
+                    }
+
+                    next = SkipInsignificant(next.Location);
+                    if (next.Value == ',')
+                    {                        
+                        yield return Result.Value(CsvToken.Comma, next.Location, next.Remainder);
+                        next = next.Remainder.ConsumeChar();
+                        if (!next.HasValue) yield break;
+                    }
+                    else if (next.Value == '\n')
+                    {
+                        yield return Result.Value(CsvToken.Newline, next.Location, next.Remainder);
+                        next = next.Remainder.ConsumeChar();
+                        if (!next.HasValue) yield break;
+                    }
+                    else
+                    {
+                        yield return Result.Empty<CsvToken>(next.Location, new[] {"comma", "newline"});
+                        yield break;
+                    }
+                }
+                else
+                {
+                    yield return Result.Empty<CsvToken>(next.Location, new[] {"double-quote"});
+                    yield break;
+                }
+
+                next = SkipInsignificant(next.Location);
+            } while (next.HasValue);
+        }
+
+        static bool IsEscapedDoubleQuote(TextSpan span)
+        {
+            return span.Length >= 2 &&
+                   span.Source[span.Position.Absolute] == '"' &&
+                   span.Source[span.Position.Absolute + 1] == '"';
+        }
+
+        static bool TryMatchSpecialContent(TextSpan text, out CsvToken specialTokenType)
+        {       
+            // Planning a switch from "True" to "true" for CSV Booleans
+            if (text.EqualsValueIgnoreCase("true") ||
+                text.EqualsValueIgnoreCase("false"))
+            {
+                specialTokenType = CsvToken.Boolean;
+                return true;
+            }
+
+            if (text.EqualsValue("null"))
+            {
+                specialTokenType = CsvToken.Null;
+                return true;
+            }
+
+            // Just a quick temp job here until Superpower `Numerics` gets `Decimal` and `HexNumber`, plus
+            // an `IsMatch(TextSpan)` on `TextParser`.
+            var s = text.ToStringValue();
+            if (text.Length > 0 
+                && text.Length < 16
+                && (decimal.TryParse(text.ToStringValue(), out _) ||
+                    s.StartsWith("0x") && ulong.TryParse(s.Substring(2), NumberStyles.HexNumber, CultureInfo.InvariantCulture, out _)))
+            {
+                specialTokenType = CsvToken.Number;
+                return true;
+            }
+
+            specialTokenType = CsvToken.None;
+            return false;
+        }
+
+        static Result<char> SkipInsignificant(TextSpan span)
+        {
+            var result = span.ConsumeChar();
+            while (result.HasValue && result.Value != '\n' && char.IsWhiteSpace(result.Value))
+                result = result.Remainder.ConsumeChar();
+            return result;
+        }
+    }
+}

src/SeqCli/Csv/CsvWriter.cs

@@ -0,0 +1,64 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using Serilog.Sinks.SystemConsole.Themes;
+using Superpower.Model;
+
+namespace SeqCli.Csv
+{
+    static class CsvWriter
+    {
+        public static void WriteCsv(IEnumerable<Token<CsvToken>> csv, ConsoleTheme theme, TextWriter output, bool hasHeaderRow)
+        {
+            var isHeaderRow = hasHeaderRow;
+
+            foreach (var token in csv)
+            {
+                switch (token.Kind)
+                {
+                    case CsvToken.Newline:
+                        output.WriteLine();
+                        isHeaderRow = false;
+                        break;
+                    case CsvToken.Comma:
+                        theme.Set(output, ConsoleThemeStyle.TertiaryText);
+                        output.Write(',');
+                        theme.Reset(output);
+                        break;
+                    case CsvToken.DoubleQuote:
+                        theme.Set(output, ConsoleThemeStyle.TertiaryText);
+                        output.Write('"');
+                        theme.Reset(output);
+                        break;
+                    case CsvToken.Boolean:
+                        theme.Set(output, ConsoleThemeStyle.Boolean);
+                        output.Write(token.ToStringValue());
+                        theme.Reset(output);
+                        break;
+                    case CsvToken.Null:
+                        theme.Set(output, ConsoleThemeStyle.Null);
+                        output.Write(token.ToStringValue());
+                        theme.Reset(output);
+                        break;
+                    case CsvToken.Number:
+                        theme.Set(output, ConsoleThemeStyle.Number);
+                        output.Write(token.ToStringValue());
+                        theme.Reset(output);
+                        break;
+                    case CsvToken.EscapedDoubleQuote:
+                        theme.Set(output, ConsoleThemeStyle.Scalar);
+                        output.Write(token.ToStringValue());
+                        theme.Reset(output);
+                        break;
+                    case CsvToken.Text:
+                        theme.Set(output, isHeaderRow ? ConsoleThemeStyle.Name : ConsoleThemeStyle.Text);
+                        output.Write(token.ToStringValue());
+                        theme.Reset(output);
+                        break;
+                    default:
+                        throw new ArgumentException($"Unrecognized token `{token}`.");
+                }
+            }
+        }
+    }
+}

src/SeqCli/SeqCli.csproj

@@ -29,6 +29,7 @@
     <PackageReference Include="Serilog.Sinks.Console" Version="3.0.1" />
     <PackageReference Include="Autofac" Version="4.0.0" />
     <PackageReference Include="ILLink.Tasks" Version="0.1.4-preview-981901" />
+    <PackageReference Include="Superpower" Version="1.1.0" />
     <PackageReference Include="System.Reactive" Version="3.1.1" />
     <PackageReference Include="System.Security.Cryptography.ProtectedData" Version="4.4.0" />
     <PackageReference Include="Seq.Api" Version="4.2.2" />

test/SeqCli.Tests/Csv/CsvTokenizerTests.cs

@@ -0,0 +1,54 @@
+using System.Collections.Generic;
+using System.Linq;
+using SeqCli.Csv;
+using Superpower.Model;
+using Xunit;
+
+namespace SeqCli.Tests.Csv
+{
+    public class CsvTokenizerTests
+    {
+        [Fact]
+        public void AnEmptyStringYieldsNoTokens()
+        {
+            Assert.Empty(Tokenize(""));
+        }
+
+        [Fact]
+        public void TokenizesATextCell()
+        {
+            var tokens = Tokenize("\"abc\"");
+            Assert.Equal(tokens.Count(), 3);
+            Assert.Equal(tokens.Select(t => t.Kind), new[]{CsvToken.DoubleQuote, CsvToken.Text, CsvToken.DoubleQuote});
+        }
+
+        [Fact]
+        public void TokenizesARow()
+        {
+            var tokens = Tokenize("\"abc\",\"def\"\r\n");
+            Assert.Equal(tokens.Count(), 8);
+        }
+
+        [Theory]
+        [InlineData(CsvToken.Text, "abc")]
+        [InlineData(CsvToken.Text, "1a")]
+        [InlineData(CsvToken.Text, "1\"\"")]
+        [InlineData(CsvToken.Number, "1")]
+        [InlineData(CsvToken.Number, "-123.45")]
+        [InlineData(CsvToken.Number, "0xa123")]
+        [InlineData(CsvToken.Boolean, "true")]
+        [InlineData(CsvToken.Boolean, "false")]
+        [InlineData(CsvToken.Null, "null")]
+        public void DetectsSpecialTokenTypes(object o, string cell)
+        {
+            var tokenKind = (CsvToken) o;
+            var content = Tokenize($"\"{cell}\"").ElementAt(1);
+            Assert.Equal(tokenKind, content.Kind);
+        }
+
+        static TokenList<CsvToken> Tokenize(string csv)
+        {
+            return new CsvTokenizer().Tokenize(csv);
+        }
+    }
+}

test/SeqCli.Tests/SeqCli.Tests.csproj

@@ -1,22 +1,17 @@
 <Project Sdk="Microsoft.NET.Sdk">
-
-  <PropertyGroup>    
-    <TargetFramework>netcoreapp2.0</TargetFramework>    
+  <PropertyGroup>
+    <TargetFramework>netcoreapp2.0</TargetFramework>
     <Platforms>x64</Platforms>
   </PropertyGroup>
-
   <ItemGroup>
     <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" />
     <PackageReference Include="xunit" Version="2.2.0" />
     <PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" />
   </ItemGroup>
-
   <ItemGroup>
     <ProjectReference Include="..\..\src\SeqCli\SeqCli.csproj" />
   </ItemGroup>
-
   <ItemGroup>
     <Service Include="{82a7f48d-3b50-4b1e-b82e-3ada8210c358}" />
   </ItemGroup>
-
-</Project>
+</Project>