Perf Avore: A Rule Based CrossPlatform Performance Minded Monitoring and Analysis Tool

// src/PerfAvore/PerfAvore.Console/RulesEngine/Domain.fs

type Condition = 
    {  Conditioner      : Conditioner
       ConditionType    : ConditionType
       ConditionalValue : ConditionalValue }
and Conditioner = 
    { ConditionerEvent    : ConditionerEvent 
      ConditionerProperty : ConditionerProperty }
and ConditionType = 
    | LessThan
    | LessThanEqualTo
    | GreaterThan
    | GreaterThanEqualTo
    | Equal
    | NotEqual
    | IsAnomaly
and ConditionalValue =
    | Value of double
    | AnomalyDetectionType of AnomalyDetectionType 
and ConditionerEvent    = string
and ConditionerProperty = string
and AnomalyDetectionType =
    | DetectIIDSpike

// src/PerfAvore/PerfAvore.Console/RulesEngine/Domain.fs

type Action = 
    { ActionOperator: ActionOperator; ActionOperand: ActionOperand }
and ActionOperator = 
    |  Print
and ActionOperand =
    | Alert
    | CallStack
    | Chart

// src/PerfAvore/PerfAvore.Console/RulesEngine/Domain.fs

type Rule = 
    { Id           : Guid
      Condition    : Condition
      Action       : Action 
      InputRule    : string }

[ 
    "GC/AllocationTick.AllocationAmount > 108000: Print Alert",
    "GC/AllocationTick.AllocationAmount isAnomaly DetectIIDSpike : Print CallStack"
]

// src/PerfAvore/PerfAvore.Console/RulesEngine/Parser.fs

let parseCondition (conditionAsString : string) : Condition = 

    let splitCondition : string[] = conditionAsString.Split(" ", StringSplitOptions.RemoveEmptyEntries)

    // Precondition check
    if splitCondition.Length <> 3
    then invalidArg (nameof conditionAsString) ("Incorrect format of the condition. Format is: Event.Property Condition ConditionalValue. For example: GCEnd.SuspensionTimeMSec >= 298")

    // Condition Event and Property
    let parseConditioner : Conditioner = 
        let splitConditioner : string[] = splitCondition.[0].Split(".", StringSplitOptions.RemoveEmptyEntries)
        let parseConditionEvent : ConditionerEvent = splitConditioner.[0]
        let parseConditionProperty : ConditionerProperty = splitConditioner.[1]

        { ConditionerEvent = parseConditionEvent; ConditionerProperty = parseConditionProperty }

    // Condition Type
    let parseConditionType : ConditionType =
        match splitCondition.[1].ToLower() with
        | ">"  | "greaterthan"                                 -> ConditionType.GreaterThan 
        | "<"  | "lessthan"                                    -> ConditionType.LessThan
        | ">=" | "greaterthanequalto" | "greaterthanorequalto" -> ConditionType.GreaterThanEqualTo
        | "<=" | "lessthanequalto"    | "lessthanorequalto"    -> ConditionType.LessThanEqualTo
        | "="  | "equal"              | "equals"               -> ConditionType.Equal
        | "!=" | "notequal"                                    -> ConditionType.NotEqual
        | "isanomaly"                                          -> ConditionType.IsAnomaly
        | _                                                    -> invalidArg (nameof splitCondition) ("${splitCondition.[1]} is an unrecognized condition type.")

    // Condition Value
    let parseConditionValue : ConditionalValue =
        let conditionalValueAsString = splitCondition.[2].ToLower()
        let checkDouble, doubleValue = Double.TryParse conditionalValueAsString 
        match checkDouble, doubleValue with
        | true, v -> ConditionalValue.Value(v)
        | false, _ -> 
            match conditionalValueAsString with
            | "detectiidspike" -> ConditionalValue.AnomalyDetectionType(AnomalyDetectionType.DetectIIDSpike)
            | _                -> invalidArg (nameof splitCondition) ($"{conditionalValueAsString} is an unrecognized anomaly detection type.")

    { Conditioner = parseConditioner; ConditionType = parseConditionType; ConditionalValue = parseConditionValue }

// src/PerfAvore/PerfAvore.Console/RulesEngine/Parser.fs

let parseAction (actionAsAString : string) : Action = 
    let splitAction : string[] = actionAsAString.Split(" ", StringSplitOptions.RemoveEmptyEntries)

    // ActionOperator
    let parseActionOperator : ActionOperator = 
        match splitAction.[0].ToLower() with
        | "print" -> ActionOperator.Print
        | _       -> invalidArg (nameof splitAction) ($"{splitAction.[0]} is an unrecognized Action Operator.")

    // ActionOperand 
    let parseActionOperand : ActionOperand = 
        match splitAction.[1].ToLower() with
        | "alert"     -> ActionOperand.Alert
        | "callstack" -> ActionOperand.CallStack
        | "chart"     -> ActionOperand.Chart
        | _           -> invalidArg (nameof splitAction) ($"{splitAction.[1]} is an unrecognized Action Operand.")


    { ActionOperator = parseActionOperator; ActionOperand = parseActionOperand }

// src/PerfAvore/PerfAvore.Console/RulesEngine/Parser.fs

let parseRule (ruleAsString : string) : Rule = 
    let splitRuleAsAString : string[] = ruleAsString.Split(":")
    let condition : Condition = parseCondition splitRuleAsAString.[0]
    let action : Action = parseAction splitRuleAsAString.[1]
    { Condition = condition; Action = action; InputRule = ruleAsString; Id = Guid.NewGuid() }

// src/PerfAvore/PerfAvore.Console/CommandLine.fs

#r "nuget:Argu" // Added specifically for this notebook.

open Argu

type Arguments = 
    | [<Mandatory>] ProcessName of string
    | TracePath of Path : string
    | RulesPath of Path : string

    interface IArgParserTemplate with
        member s.Usage =
            match s with
            | TracePath   _ -> "Specify a Path to the Trace."
            | ProcessName _ -> "Specify a Process Name."
            | RulesPath   _ -> "Specify a Path to a Json File With the Rules."

// src/PerfAvore/PerfAvore.Console/Program.fs

// This is passed in from the command line but for the sake of demonstration, we'll include this as a literal []<string>.
let argv              = [| "--tracepath"; "Path.etl"; "--processname"; "Test.exe"|]

let parser            = ArgumentParser.Create<Arguments>()
let parsedCommandline = parser.Parse(inputs = argv)

let containsTracePath : bool = parsedCommandline.Contains TracePath
containsTracePath

// src/PerfAvore/PerfAvore.Console/TraceSession.fs

#r "nuget: Microsoft.Diagnostics.Tracing.TraceEvent"

open Microsoft.Diagnostics.Tracing.Etlx

let getTraceLogFromTracePath (tracePath : string) : TraceLog = 
    TraceLog.OpenOrConvert tracePath

// src/PerfAvore/PerfAvore.Console/TraceSession.fs

// Needed for compilation
#r "nuget:Microsoft.Diagnostics.NETCore.Client"

// Ignore this impl for now. More details about this in Step 3 but for the sake of success compilation, we need this.
let applyRule (rule: Rule) (traceEvent : TraceEvent) : unit = 
    ()

open System
open System.Collections.Generic
open System.Runtime.InteropServices

open Microsoft.Diagnostics.NETCore.Client
open Microsoft.Diagnostics.Tracing.Etlx
open Microsoft.Diagnostics.Tracing.Session
open Microsoft.Diagnostics.Tracing.Parsers
open Microsoft.Diagnostics.Tracing

open System.Diagnostics

let getProcessIdForProcessName (processName : string) : int =
        let processes = Process.GetProcessesByName(processName)
        if processes.Length < 1 then invalidArg processName $"No processes with name: {processName} exists."
        // For the sake of simplicity, choose the first process available with the said name. 
        else processes.[0].Id

let getRealTimeSession (processName : string) (parsedRules : Rule list) : TraceEventDispatcher * IDisposable = 

    let callbackForAllEvents (processId : int): Action<TraceEvent> = 
        Action<TraceEvent>(fun traceEvent -> 
            parsedRules
            |> List.iter(fun rule ->
                if processId = traceEvent.ProcessID then applyRule rule traceEvent))

    let processId = getProcessIdForProcessName processName

    // Windows.
    if RuntimeInformation.IsOSPlatform OSPlatform.Windows then
        let traceEventSession : TraceEventSession = new TraceEventSession($"Session_{Guid.NewGuid()}");

        let keywords : uint64 = uint64(ClrTraceEventParser.Keywords.All) 

        traceEventSession.EnableKernelProvider(KernelTraceEventParser.Keywords.All, KernelTraceEventParser.Keywords.None) |> ignore
        traceEventSession.EnableProvider(ClrTraceEventParser.ProviderGuid, TraceEventLevel.Verbose, keywords)             |> ignore

        // Once the pertinent providers are enabled, create the trace log event source. 
        let traceLogEventSource = TraceLog.CreateFromTraceEventSession traceEventSession

        // Add all the necessary callbacks.
        traceLogEventSource.Clr.add_All(callbackForAllEvents processId)    |> ignore
        traceLogEventSource.Kernel.add_All(callbackForAllEvents processId) |> ignore

        // TODO: Enable the GLAD events - only available for real time processing.
        // ala: https://devblogs.microsoft.com/dotnet/556-2/
        traceLogEventSource, traceEventSession

    // Linux / MacOS.
    else
        let keywords : int64 = int64(ClrTraceEventParser.Keywords.All) 
        let eventPipeProvider : EventPipeProvider = 
            EventPipeProvider("Microsoft-Windows-DotNETRuntime", Tracing.EventLevel.Informational, keywords)
        let providers = List<EventPipeProvider>()
        providers.Add eventPipeProvider

        // For the sake of simplicity, choose the first process available with the said name. 
        let processId        = getProcessIdForProcessName processName
        let client           = DiagnosticsClient(processId)
        let eventPipeSession = client.StartEventPipeSession(providers, false)
        let source           = new EventPipeEventSource(eventPipeSession.EventStream)

        source.Clr.add_All(callbackForAllEvents processId)     |> ignore
        source.Kernel.add_All(callbackForAllEvents processId ) |> ignore

        source, eventPipeSession

// src/PerfAvore/PerfAvore.Console/Program.fs

 // Hard coded values for the sake of successful compilation.
 let processName = "Test.exe"
 let processID   = -1
 let parsedRules = [ parseRule "GC/AllocationTick.AllocationAmount > 110000 : Print CallStack" ]

let startProcessingEvents() : unit = 
    // If the trace log file is provided, use the Trace Log API to traverse through all events.
    if containsTracePath then 
        let tracePathArgs = parsedCommandline.GetResult TracePath
        let traceLog = getTraceLogFromTracePath tracePathArgs
        let events = traceLog.Events 
        let eventNamesToFilter = parsedRules |> List.map(fun r -> r.Condition.Conditioner.ConditionerEvent.ToString())

        let applyRulesForAllEvents (events : TraceEvent seq) (rules : Rule list) = 
            events
            // Consider events with name of the process and if they contain the events defined in the rules.
            |> Seq.filter(fun e -> e.ProcessID = processID && 
                                    eventNamesToFilter |> List.contains(e.EventName))
            |> Seq.iter(fun e -> 
                rules
                |> List.iter(fun rule -> applyRule rule e ))
        applyRulesForAllEvents events parsedRules

    // Else, start a Real Time Session.
    // Requires admin privileges
    else
        let traceLogEventSource, session = getRealTimeSession processName parsedRules
        Console.CancelKeyPress.Add(fun e -> session.Dispose() |> ignore )

        traceLogEventSource.Process() |> ignore
        ()

// src/PerfAvore/PerfAvore.Console/RulesEngine/ActionEngine.fs

open System
open System.Linq

let applyRule (rule : Rule) (traceEvent : TraceEvent) : unit =

    // Helper fn checks if the condition is met for the traceEvent.
    let checkCondition : bool =
        let condition : Condition = rule.Condition

        // Match the event name.
        let matchEventName (traceEvent : TraceEvent) : bool = 
            traceEvent.EventName = condition.Conditioner.ConditionerEvent

        // Check if the specified payload exists.
        let checkPayload (traceEvent : TraceEvent) : bool = 
            if traceEvent.PayloadNames.Contains condition.Conditioner.ConditionerProperty then true
            else false

        // Early return if the payload is unavailable since it will except later if we let it slide. 
        if ( checkPayload traceEvent ) = false then 
            false
        else
            let payload : double = Double.Parse (traceEvent.PayloadByName(condition.Conditioner.ConditionerProperty).ToString())

            // Check if the condition matches.
            let checkConditionValue (rule : Rule) (traceEvent : TraceEvent) : bool =
                let conditionalValue : ConditionalValue = rule.Condition.ConditionalValue

                match conditionalValue with
                | ConditionalValue.Value value ->
                    match condition.ConditionType with
                    | ConditionType.Equal              -> payload = value
                    | ConditionType.GreaterThan        -> payload > value
                    | ConditionType.GreaterThanEqualTo -> payload >= value
                    | ConditionType.LessThan           -> payload < value
                    | ConditionType.LessThanEqualTo    -> payload <= value
                    | ConditionType.NotEqual           -> payload <> value
                    | ConditionType.IsAnomaly          -> false // This case should technically not be reached but adding it to prevent warnings.
                | ConditionalValue.AnomalyDetectionType anomalyDetectionType ->
                    match anomalyDetectionType with
                    | AnomalyDetectionType.DetectIIDSpike ->
                        // We'll be going over this logic below. Right now simply return false.
                        false

            // Match on Event Name, if the payload exists and the condition based on the trace event is met.
            matchEventName traceEvent && checkPayload traceEvent && checkConditionValue rule traceEvent

            // ... => Apply Actions. 
    ()

// src/PerfAvore/PerfAvore.Console/RulesEngine/AnomalyDetection/Service.fs

open System
open System.Collections.Concurrent

type FixedSizeQueue<'T> (capacity : int) =
    // Concurrency might not be necessary but better to be safe than sorry.
    let queue = ConcurrentQueue<'T>()

    member this.Capacity : int = capacity
    member this.Count    : int = queue.Count
    member this.Print() : unit = 
        let stringRepr : string = String.Join(",", queue.ToArray())
        printfn "%A" stringRepr

    member this.Insert (item : 'T) : unit = 
        // If we are at capacity, evict the first item.
        if queue.Count = capacity then 
            queue.TryDequeue() |> ignore

        // Enqueue the new item to the list.
        queue.Enqueue(item)

    member this.GetAll() : seq<'T> = 
        queue

// src/PerfAvore/PerfAvore.Console/RulesEngine/Domain.fs

#r "nuget:Microsoft.ML"

open Microsoft.ML.Data

type AnomalyDetectionInput() =
    [<DefaultValue>]
    [<LoadColumn(0)>]
    val mutable public timestamp : double 

    [<DefaultValue>]
    [<LoadColumn(1)>]
    val mutable public value : float32 

type AnomalyDetectionContext = 
    { Rule  : Rule 
      Input : AnomalyDetectionInput }
type AnomalyDetectionResult = 
    { Context   : AnomalyDetectionContext
      IsAnomaly : bool
      PValue    : double }

// src/PerfAvore/PerfAvore.Console/RulesEngine/AnomalyDetection/Service.fs

open System
open System.Collections.Concurrent

type AnomalyDetectionContextService(capacity : int) = 
    // Keyed on the Rule Id and Value is a FixedSizeQueueForTraceEvents.
    // Each Rule that has Anomaly Detection associated with it must have its own Fixed Size Queue.
    let cache = ConcurrentDictionary<Guid, FixedSizeQueue<AnomalyDetectionInput>>()

    static member AnomalyPValueHistoryLength : int    = 30
    static member AnomalyConfidence          : double = 95.

    member this.Upsert (ruleId : Guid) (item : AnomalyDetectionInput) : unit =
        let queueExists, queue = cache.TryGetValue ruleId
        match queueExists, queue with
        | true, q  -> 
            q.Insert item
        | false, _ -> 
            cache.[ruleId] <- FixedSizeQueue( capacity )
            cache.[ruleId].Insert item

    member this.TryRetrieve(ruleId : Guid) : AnomalyDetectionInput seq option = 
        let queueExists, queue = cache.TryGetValue ruleId
        match queueExists, queue with
        | true, q  -> Some (q.GetAll())
        | false, _ -> None

// src/PerfAvore/PerfAvore.Console/RulesEngine/AnomalyDetection/IIDSpike.fs

#r "nuget:Microsoft.ML.TimeSeries"

open Microsoft.ML
open Microsoft.ML.Data
open Microsoft.ML.Transforms.TimeSeries
open System.Collections.Generic

open System.Linq

let ctx : MLContext = MLContext()

type Prediction() = 
    [<DefaultValue>]
    [<VectorType(3)>] // prediction i.e. 0/1 + value i.e. payload + p-value
    val mutable public Prediction : double[]

let getAnomaliesUsingIIDSpikeEstimation (input : AnomalyDetectionContext) 
                                        (service : AnomalyDetectionContextService) 
                                        : AnomalyDetectionResult =
    let retrievedInput = service.TryRetrieve input.Rule.Id 
    let buffer =          
        match retrievedInput with
        | Some b -> b 
        | None   -> failwith $"Failed to look up Anomaly Detection Buffer for rule: {input.Rule.InputRule}" 

    let dataView = 
        ctx.Data.LoadFromEnumerable<AnomalyDetectionInput>(buffer)

    // If p-value < (1 - confidence / 100.0) -> Alert i.e. anomaly.
    let anomalyPipeline : IidSpikeEstimator =
        ctx.Transforms.DetectIidSpike(
        outputColumnName    = "Prediction",
        inputColumnName     = "value",
        side                = AnomalySide.Positive,
        confidence          = AnomalyDetectionContextService.AnomalyConfidence,  //  Alert Threshold = 1 - options.Confidence / 100;
        pvalueHistoryLength = AnomalyDetectionContextService.AnomalyPValueHistoryLength )

    // For this model, fitting doesn't matter.
    let trainedAnomalyModel : IidSpikeDetector 
        = anomalyPipeline.Fit(ctx.Data.LoadFromEnumerable(List<AnomalyDetectionInput>()))
    let transformedAnomalyData : IDataView 
        = trainedAnomalyModel.Transform(dataView)
    let anomalies : Prediction seq = 
        ctx.Data.CreateEnumerable<Prediction>(transformedAnomalyData, reuseRowObject = false)

    // Last one in the buffer since it's the most recent one. 
    let inputPoint = anomalies.Last()
    { Context   = input 
      IsAnomaly = inputPoint.Prediction[0] = 1
      PValue    = inputPoint.Prediction[2] }

// src/PerfAvore/PerfAvore.Console/RulesEngine/ActionEngine.fs

let anomalyDetectionContextService : AnomalyDetectionContextService = 
    AnomalyDetectionContextService(AnomalyDetectionContextService.AnomalyPValueHistoryLength)

let applyRule (rule : Rule) (traceEvent : TraceEvent) : unit =

    // Helper fn checks if the condition is met for the traceEvent.
    let checkCondition : bool =
        let condition : Condition = rule.Condition

        // Match the event name.
        let matchEventName (traceEvent : TraceEvent) : bool = 
            traceEvent.EventName = condition.Conditioner.ConditionerEvent

        // Check if the specified payload exists.
        let checkPayload (traceEvent : TraceEvent) : bool = 
            if traceEvent.PayloadNames.Contains condition.Conditioner.ConditionerProperty then true
            else false

        // Early return if the payload is unavailable since it will except later if we let it slide. 
        if ( checkPayload traceEvent ) = false then 
            false
        else
            let payload : double = Double.Parse (traceEvent.PayloadByName(condition.Conditioner.ConditionerProperty).ToString())

            // Add the new data point to the anomaly detection dict.
            let anomalyDetectionInput : AnomalyDetectionInput = 
                AnomalyDetectionInput(timestamp = traceEvent.TimeStampRelativeMSec, value = float32(payload))
            anomalyDetectionContextService.Upsert rule.Id anomalyDetectionInput |> ignore

            // Check if the condition matches.
            let checkConditionValue (rule : Rule) (traceEvent : TraceEvent) : bool =
                let conditionalValue : ConditionalValue = rule.Condition.ConditionalValue

                match conditionalValue with
                | ConditionalValue.Value value ->
                    match condition.ConditionType with
                    | ConditionType.Equal              -> payload = value
                    | ConditionType.GreaterThan        -> payload > value
                    | ConditionType.GreaterThanEqualTo -> payload >= value
                    | ConditionType.LessThan           -> payload < value
                    | ConditionType.LessThanEqualTo    -> payload <= value
                    | ConditionType.NotEqual           -> payload <> value
                    | ConditionType.IsAnomaly          -> false // This case should technically not be reached but adding it to prevent warnings.
                | ConditionalValue.AnomalyDetectionType anomalyDetectionType ->
                    match anomalyDetectionType with
                    | AnomalyDetectionType.DetectIIDSpike ->
                        let context = { Rule = rule; Input = anomalyDetectionInput }
                        let result  = getAnomaliesUsingIIDSpikeEstimation context anomalyDetectionContextService 
                        result.IsAnomaly

            // Match on Event Name, if the payload exists and the condition based on the trace event is met.
            matchEventName traceEvent && checkPayload traceEvent && checkConditionValue rule traceEvent

    ()

// src/PerfAvore/PerfAvore.Console/RulesEngine/Actions/Alerts.fs

#r "nuget:Spectre.Console"

open Microsoft.Diagnostics.Tracing
open Spectre.Console

// Added this back to distinguish between this Rule and the open imported from
// opening up Spectre.Console.
type Rule = 
    { Id           : Guid
      Condition    : Condition
      Action       : Action 
      InputRule    : string }

let printAlert (rule : Rule) (traceEvent : TraceEvent) : unit = 

    // Create a table
    let table = Spectre.Console.Table();
    table.Title <- Spectre.Console.TableTitle "[underline red] Alert! [/]"

    table.AddColumn("Input Rule")      |> ignore
    table.AddColumn("Timestamp")       |> ignore
    table.AddColumn("Event Name")      |> ignore
    table.AddColumn("Event Property")  |> ignore
    table.AddColumn("Payload")         |> ignore

    table.AddRow( rule.InputRule, 
                  traceEvent.TimeStampRelativeMSec.ToString(), 
                  traceEvent.EventName,
                  rule.Condition.Conditioner.ConditionerProperty,
                  traceEvent.PayloadByName(rule.Condition.Conditioner.ConditionerProperty).ToString() ) |> ignore

    table.Border <- Spectre.Console.TableBorder.Square

    // Render the table to the console
    Spectre.Console.AnsiConsole.Write(table);

// src/PerfAvore/PerfAvore.Console/RulesEngine/Actions/CallStack.fs

open System.IO

open Microsoft.Diagnostics.Tracing
open Microsoft.Diagnostics.Tracing.Etlx
open Microsoft.Diagnostics.Symbols

open Spectre.Console

// Added this back to distinguish between this Rule and the open imported from
// opening up Spectre.Console.
type Rule = 
    { Id           : Guid
      Condition    : Condition
      Action       : Action 
      InputRule    : string }

let symbolReader : SymbolReader = new SymbolReader(TextWriter.Null, SymbolPath.SymbolPathFromEnvironment)

// Helper fn responsible for getting the call stack from a particular trace event.
let printCallStack (rule: Rule) (traceEvent : TraceEvent) : unit =

    let callStack = traceEvent.CallStack()
    if isNull callStack then 
        printfn $"Rule: {rule.InputRule} invoked for Event: {traceEvent} however, the call stack associated with the event is null." 
        ()

    let root = Tree(Rule(rule.InputRule.EscapeMarkup()))

    let printStackFrame (callStack : TraceCallStack) : unit =
        if not (isNull callStack.CodeAddress.ModuleFile)
        then
            callStack.CodeAddress.CodeAddresses.LookupSymbolsForModule(symbolReader, callStack.CodeAddress.ModuleFile)
            let frameValue = sprintf "%s!%s" callStack.CodeAddress.ModuleName callStack.CodeAddress.FullMethodName
            root.AddNode ( frameValue.EscapeMarkup() ) |> ignore

    let rec processFrame (callStack : TraceCallStack) : unit =
        if isNull callStack then ()
        else
            printStackFrame callStack
            processFrame callStack.Caller

    processFrame callStack
    AnsiConsole.Write root
    printfn "\n"

// src/PerfAvore/PerfAvore.Console/RulesEngine/Actions/Chart.fs

#r "nuget:Fsharp.Plotly"

open System.Linq
open FSharp.Plotly

let printChart (rule : Rule) (service : AnomalyDetectionContextService) : unit = 

    let v = service.TryRetrieve(rule.Id).Value
    let x = 
        v
        |> Seq.map(fun i -> i.timestamp)
    let y = 
        v
        |> Seq.map(fun i -> i.value)
    let input = Seq.zip x y
    let point = v.Last()
    let scatterPoint = seq { point.timestamp, point.value }

    [
        Chart.Line (input, Name = $"Trend for {point.timestamp}") 
        Chart.Scatter (scatterPoint, mode = StyleParam.Mode.Markers, Name="Anomaly Point")
    ]
    |> Chart.Combine
    |> Chart.withX_AxisStyle(title = "Relative Timestamp (ms)")
    |> Chart.withY_AxisStyle(title = $"{rule.Condition.Conditioner.ConditionerProperty}")
    |> Chart.Show

❯ dotnet --version
6.0.100

MokoSan:~:% lsb_release -a
No LSB modules are available.
Distributor ID: Ubuntu
Description:    Ubuntu 20.04 LTS
Release:        20.04
Codename:       focal