From 93bbda30b3c677ce0b33169826a71f8df534a335 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 22 Feb 2019 16:20:20 +0000 Subject: [PATCH 001/353] Ingester WIP --- equinox-ingest/Ingest/Infrastructure.fs | 19 ++ equinox-ingest/Ingest/Ingest.fsproj | 22 ++ equinox-ingest/Ingest/Program.fs | 360 ++++++++++++++++++++++++ 3 files changed, 401 insertions(+) create mode 100644 equinox-ingest/Ingest/Infrastructure.fs create mode 100644 equinox-ingest/Ingest/Ingest.fsproj create mode 100644 equinox-ingest/Ingest/Program.fs diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs new file mode 100644 index 000000000..5942e97c1 --- /dev/null +++ b/equinox-ingest/Ingest/Infrastructure.fs @@ -0,0 +1,19 @@ +[] +module private ProjectorTemplate.Ingester.Infrastructure + +open System +open System.Threading +open System.Threading.Tasks + +#nowarn "21" // re AwaitKeyboardInterrupt +#nowarn "40" // re AwaitKeyboardInterrupt + +type Async with + static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) + /// Asynchronously awaits the next keyboard interrupt event + static member AwaitKeyboardInterrupt () : Async = + Async.FromContinuations(fun (sc,_,_) -> + let isDisposed = ref 0 + let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore + and d : IDisposable = Console.CancelKeyPress.Subscribe callback + in ()) diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj new file mode 100644 index 000000000..910a3b777 --- /dev/null +++ b/equinox-ingest/Ingest/Ingest.fsproj @@ -0,0 +1,22 @@ + + + + Exe + netcoreapp2.1 + 5 + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs new file mode 100644 index 000000000..2494c1e1d --- /dev/null +++ b/equinox-ingest/Ingest/Program.fs @@ -0,0 +1,360 @@ +module ProjectorTemplate.Ingester.Program + +open Equinox.Cosmos +open Equinox.Store +open FSharp.Control +open Serilog +open System +open Equinox.EventStore + +module CmdParser = + open Argu + type LogEventLevel = Serilog.Events.LogEventLevel + + exception MissingArg of string + let envBackstop msg key = + match Environment.GetEnvironmentVariable key with + | null -> raise <| MissingArg (sprintf "Please provide a %s, either as an argment or via the %s environment variable" msg key) + | x -> x + + module Cosmos = + type [] Arguments = + | [] ConnectionMode of Equinox.Cosmos.ConnectionMode + | [] Timeout of float + | [] Retries of int + | [] RetriesWaitTime of int + | [] Connection of string + | [] Database of string + | [] Collection of string + interface IArgParserTemplate with + member a.Usage = + match a with + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." + | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" + | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION, Cosmos Emulator)." + | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." + | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE, test)." + | Collection _ -> "specify a collection name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_COLLECTION, test)." + type Info(args : ParseResults) = + member __.Connection = match args.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" + member __.Database = match args.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" + member __.Collection = match args.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" + + member __.Timeout = args.GetResult(Timeout,5.) |> TimeSpan.FromSeconds + member __.Mode = args.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) + member __.Retries = args.GetResult(Retries, 1) + member __.MaxRetryWaitTime = args.GetResult(RetriesWaitTime, 5) + + /// Connect with the provided parameters and/or environment variables + member x.Connnect + /// Connection/Client identifier for logging purposes + name : Async = + let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = Discovery.FromConnectionString x.Connection + Log.Information("CosmosDb {mode} {endpointUri} Database {database} Collection {collection}.", + x.Mode, endpointUri, x.Database, x.Collection) + Log.Information("CosmosDb timeout: {timeout}s, {retries} retries; Throttling maxRetryWaitTime {maxRetryWaitTime}", + (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) + let c = + CosmosConnector(log=Log.Logger, mode=x.Mode, requestTimeout=x.Timeout, + maxRetryAttemptsOnThrottledRequests=x.Retries, maxRetryWaitTimeInSeconds=x.MaxRetryWaitTime) + c.Connect(name, discovery) + + /// To establish a local node to run against: + /// 1. cinst eventstore-oss -y # where cinst is an invocation of the Chocolatey Package Installer on Windows + /// 2. & $env:ProgramData\chocolatey\bin\EventStore.ClusterNode.exe --gossip-on-single-node --discover-via-dns 0 --ext-http-port=30778 + module EventStore = + open Equinox.EventStore + type [] Arguments = + | [] VerboseStore + | [] Timeout of float + | [] Retries of int + | [] Host of string + | [] Username of string + | [] Password of string + | [] ConcurrentOperationsLimit of int + | [] HeartbeatTimeout of float + | [] MaxItems of int + | [] Cosmos of ParseResults + interface IArgParserTemplate with + member a.Usage = + match a with + | VerboseStore -> "Include low level Store logging." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." + | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." + | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." + | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." + | ConcurrentOperationsLimit _ -> "max concurrent operations in flight (default: 5000)." + | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." + | MaxItems _ -> "maximum item count to request. Default: 4096" + | Cosmos _ -> "specify CosmosDb parameters" + type Info(args : ParseResults ) = + let connect (log: ILogger) (heartbeatTimeout, col) (operationTimeout, operationRetries) discovery (username, password) connection = + let log = if log.IsEnabled LogEventLevel.Debug then Logger.SerilogVerbose log else Logger.SerilogNormal log + GesConnector(username, password, operationTimeout, operationRetries,heartbeatTimeout=heartbeatTimeout, + concurrentOperationsLimit=col, log=log, tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string]) + .Establish("ProjectorTemplate", discovery, connection) + member val Cosmos = Cosmos.Info(args.GetResult Cosmos) + member __.CreateGateway conn = GesGateway(conn, GesBatchingPolicy(maxBatchSize = args.GetResult(MaxItems,4096))) + member __.Host = match args.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" + member __.User = match args.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" + member __.Password = match args.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" + member val CacheStrategy = let c = Caching.Cache("ProjectorTemplate", sizeMb = 50) in CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) + member __.Connect(log: ILogger, storeLog, connection) = + let (timeout, retries) as operationThrottling = args.GetResult(Timeout,5.) |> TimeSpan.FromSeconds, args.GetResult(Retries,1) + let heartbeatTimeout = args.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds + let concurrentOperationsLimit = args.GetResult(ConcurrentOperationsLimit,5000) + log.Information("EventStore {host} heartbeat: {heartbeat}s MaxConcurrentRequests {concurrency} Timeout: {timeout}s Retries {retries}", + __.Host, heartbeatTimeout.TotalSeconds, concurrentOperationsLimit, timeout.TotalSeconds, retries) + connect storeLog (heartbeatTimeout, concurrentOperationsLimit) operationThrottling (Discovery.GossipDns __.Host) (__.User,__.Password) connection + + [] + type Arguments = + | [] ConsumerGroupName of string + | [] ForceStartFromHere + | [] BatchSize of int + | [] LagFreqS of float + | [] Verbose + | [] Stream of string + | [] Es of ParseResults + interface IArgParserTemplate with + member a.Usage = + match a with + | ConsumerGroupName _ ->"projector group name." + | ForceStartFromHere _ -> "(iff `suffix` represents a fresh LeaseId) - force skip to present Position. Default: Never skip an event on a lease." + | BatchSize _ -> "maximum item count to request from feed. Default: 1000" + | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" + | Verbose -> "request Verbose Logging. Default: off" + | Stream _ -> "specify stream(s) to seed the processing with" + | Es _ -> "specify EventStore parameters" + and Parameters(args : ParseResults) = + member val EventStore = EventStore.Info(args.GetResult Es) + member __.ConsumerGroupName = args.GetResult ConsumerGroupName + member __.Verbose = args.Contains Verbose + member __.BatchSize = args.GetResult(BatchSize,1000) + member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds + member __.StartFromHere = args.Contains ForceStartFromHere + member x.BuildFeedParams() = + Log.Information("Processing {leaseId} in batches of {batchSize}", x.ConsumerGroupName, x.BatchSize) + if x.StartFromHere then Log.Warning("(If new projector group) Skipping projection of all existing events.") + x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) + x.ConsumerGroupName, x.StartFromHere, x.BatchSize, x.LagFrequency, args.GetResults Stream + + /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args + let parse argv : Parameters = + let programName = System.Reflection.Assembly.GetEntryAssembly().GetName().Name + let parser = ArgumentParser.Create(programName = programName) + parser.ParseCommandLine argv |> Parameters + +// Illustrates how to emit direct to the Console using Serilog +// Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog +module Logging = + let initialize verbose = + Log.Logger <- + LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() + |> fun c -> if verbose then c.MinimumLevel.Debug() else c + |> fun c -> c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) + .CreateLogger() + +//let mkRangeProjector (broker, topic) = +// let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch +// let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = LZ4) +// let producer = KafkaProducer.Create(Log.Logger, cfg, topic) +// let disposeProducer = (producer :> IDisposable).Dispose +// let projectBatch (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { +// sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us +// let toKafkaEvent (e: DocumentParser.IEvent) : RenderedEvent = { s = e.Stream; i = e.Index; c = e.EventType; t = e.TimeStamp; d = e.Data; m = e.Meta } +// let pt,events = (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> Seq.map toKafkaEvent |> Array.ofSeq) |> Stopwatch.Time +// let es = [| for e in events -> e.s, JsonConvert.SerializeObject e |] +// let! et,_ = producer.ProduceBatch es |> Stopwatch.Time +// let r = ctx.FeedResponse +// Log.Information("{range} Fetch: {token} {requestCharge:n0}RU {count} docs {l:n1}s; Parse: {events} events {p:n3}s; Emit: {e:n1}s", +// ctx.PartitionKeyRangeId, r.ResponseContinuation.Trim[|'"'|], r.RequestCharge, docs.Count, float sw.ElapsedMilliseconds / 1000., +// events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) +// sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor +// } +// ChangeFeedObserver.Create(Log.Logger, projectBatch, disposeProducer) + +open System.Collections.Concurrent +open System.Diagnostics +open System.Threading +open System.Threading.Tasks + +module Ingester = + open Equinox.Cosmos.Core + open Equinox.Cosmos.Store + + type [] Batch = { stream: string; pos: int64; events: IEvent[] } + type [] Result = + | Ok of stream: string * updatedPos: int64 + | Duplicate of stream: string * updatedPos: int64 + | Conflict of overage: Batch + | Exn of exn: exn * batch: Batch + let private write (ctx : CosmosContext) (batch : Batch) = async { + let stream = ctx.CreateStream batch.stream + try let! res = ctx.Sync(stream, { index = batch.pos; etag = None }, batch.events) + match res with + | AppendResult.Ok pos -> return Ok (batch.stream, pos.index) + | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> + if pos.index >= batch.pos + batch.events.LongLength then return Duplicate (batch.stream, pos.index) + else return Conflict { stream = batch.stream; pos = pos.index; events = batch.events |> Array.skip (pos.index-(batch.pos+batch.events.LongLength) |> int) } + with e -> return Exn (e, batch) } + + /// Manages distribution of work across a specified number of concurrent writers + type Writer (ctx : CosmosContext, queueLen, ct : CancellationToken) = + let buffer = new BlockingCollection<_>(ConcurrentQueue(), queueLen) + let result = Event<_>() + let child = async { + let! ct = Async.CancellationToken // i.e. cts.Token + for item in buffer.GetConsumingEnumerable(ct) do + let! res = write ctx item + result.Trigger res } + member internal __.StartConsumers n = + for _ in 1..n do + Async.StartAsTask(child, cancellationToken=ct) |> ignore + + /// Supply an item to be processed + member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) + [] member __.Result = result.Publish + // TEMP - to be removed + member __.Add item = buffer.Add item + + type Queue(log: Serilog.ILogger, writer : Writer) = + member __.TryAdd(item, timeout) = writer.TryAdd(item, timeout) + // TEMP - this blocks, and real impl wont do that + member __.Post item = + writer.Add item + member __.HandleWriteResult res = + match res with + | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) + | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) + | Conflict overage -> log.Information("Requeing {stream} {pos} ({count} events)", overage.stream, overage.pos, overage.events.Length) + | Exn (exn, batch) -> + log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.events.Length) + // TODO remove; this is not a sustainable idea + writer.Add batch + type Ingester(writer : Writer) = + member __.Add batch = writer.Add batch + + /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does + let start(ctx : Equinox.Cosmos.Core.CosmosContext, writerQueueLen, writerCount) = async { + let! ct = Async.CancellationToken + let writer = Writer(ctx, writerQueueLen, ct) + let queue = Queue(Log.Logger,writer) + let _ = writer.Result.Subscribe queue.HandleWriteResult // codependent, wont worry about unsubcribing + writer.StartConsumers writerCount + return Ingester(writer) + } + +module Reader = + open EventStore.ClientAPI + + let loadSpecificStreamsTemp (conn: IEventStoreConnection, batchSize) streams (postBatch : (Ingester.Batch -> unit)) = + let fetchStream stream = + let rec fetchFrom pos = async { + let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect + if currentSlice.IsEndOfStream then return () else + let events = + [| for x in currentSlice.Events -> + let e = x.Event + Equinox.Cosmos.Store.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Cosmos.Store.IEvent |] + postBatch { stream = stream; pos = currentSlice.FromEventNumber; events = events } + return! fetchFrom currentSlice.NextEventNumber } + fetchFrom 0L + async { + for stream in streams do + do! fetchStream stream } + +open EventStore.ClientAPI +let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseId, forceSkip, batchSize, writerQueueLen, writerCount, lagReportFreq : TimeSpan option, streams: string list) = async { + //let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { + // Log.Information("Lags by Range {@rangeLags}", remainingWork) + // return! Async.Sleep interval } + //let maybeLogLag = lagReportFreq |> Option.map logLag + let enumEvents (slice : AllEventsSlice) = seq { + for e in slice.Events -> + match e.Event with + | e when e.IsJson -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) + | e -> Choice2Of2 e + } + let mutable total = 0L + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let followAll (postBatch : Ingester.Batch -> unit) = + let rec loop pos = async { + let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(pos, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let received = currentSlice.Events.Length + total <- total + int64 received + let streams = + enumEvents currentSlice + |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + |> Seq.groupBy (fun (s,_,_) -> s) + |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) + |> Array.ofSeq + + match streams |> Seq.map (fun (_s,xs) -> Array.length xs) |> Seq.sum with + | extracted when extracted <> received -> Log.Warning("Dropped {count} of {total}", received-extracted, received) + | _ -> () + + let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head + let cats = seq { for (s,_) in streams -> category s } |> Seq.distinct |> Seq.length + let postSw = Stopwatch.StartNew() + for s,xs in streams do + for pos, item in xs do + postBatch { stream = s; pos = pos; events = [| item |]} + Log.Information("Fetch {count} {ft:n0}ms; Parse c {categories} s {streams}; Post {pt:n0}ms", + received, sw.ElapsedMilliseconds, cats, streams.Length, postSw.ElapsedMilliseconds) + if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", total) + + sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + if not currentSlice.IsEndOfStream then return! loop currentSlice.NextPosition + } + fun pos -> loop pos + + //let fetchBatches stream pos size = + // let fetchFull pos = async { + // let! currentSlice = source.ReadConnection.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect + // if currentSlice.IsEndOfStream then return None + // else return Some (currentSlice.NextEventNumber,currentSlice.Events) + // } + // AsyncSeq.unfoldAsync fetchFull 0L + + //let fetchSpecific streams = async { + // let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + // for stream in streams do + // sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + // enumStreamEvents currentSlice + // |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + // |> Seq.groupBy (fun (s,_,_) -> s) + // |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) + // |> Array.ofSeq + // |> xs.AddRange + // sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + // if not currentSlice.IsEndOfStream then return! fetch currentSlice.NextPosition + + //} + + + let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) + let! ingester = Ingester.start(ctx, writerQueueLen, writerCount) + let! _ = Async.StartChild (Reader.loadSpecificStreamsTemp (source.ReadConnection, batchSize) streams ingester.Add) + let! _ = Async.StartChild (followAll ingester.Add Position.Start) + do! Async.AwaitKeyboardInterrupt() } + +[] +let main argv = + try let args = CmdParser.parse argv + Logging.initialize args.Verbose + let connectionMode = Equinox.EventStore.ConnectionStrategy.ClusterSingle Equinox.EventStore.NodePreference.Master + let source = args.EventStore.Connect(Log.Logger, Log.Logger, connectionMode) |> Async.RunSynchronously + let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu + let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously + let colls = CosmosCollections(cosmos.Database, cosmos.Collection) + let leaseId, startFromHere, batchSize, lagFrequency, streams = args.BuildFeedParams() + let writerQueueLen, writerCount = 10, 2 + run (destination, colls) source (leaseId, startFromHere, batchSize, writerQueueLen, writerCount, lagFrequency, streams) |> Async.RunSynchronously + 0 + with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 + | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 + | e -> eprintfn "%s" e.Message; 1 \ No newline at end of file From 8cfaf2de4596b799343d403f1538ba50356eac01 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 25 Feb 2019 12:14:13 +0000 Subject: [PATCH 002/353] Logging + tweaks --- equinox-ingest/Ingest/Program.fs | 71 +++++++++----------------------- 1 file changed, 19 insertions(+), 52 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 2494c1e1d..ae7d4ed80 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -152,34 +152,16 @@ module CmdParser = module Logging = let initialize verbose = Log.Logger <- - LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() + LoggerConfiguration() + .Destructure.FSharpTypes() + .Enrich.FromLogContext() |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) - .CreateLogger() - -//let mkRangeProjector (broker, topic) = -// let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch -// let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = LZ4) -// let producer = KafkaProducer.Create(Log.Logger, cfg, topic) -// let disposeProducer = (producer :> IDisposable).Dispose -// let projectBatch (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { -// sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us -// let toKafkaEvent (e: DocumentParser.IEvent) : RenderedEvent = { s = e.Stream; i = e.Index; c = e.EventType; t = e.TimeStamp; d = e.Data; m = e.Meta } -// let pt,events = (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> Seq.map toKafkaEvent |> Array.ofSeq) |> Stopwatch.Time -// let es = [| for e in events -> e.s, JsonConvert.SerializeObject e |] -// let! et,_ = producer.ProduceBatch es |> Stopwatch.Time -// let r = ctx.FeedResponse -// Log.Information("{range} Fetch: {token} {requestCharge:n0}RU {count} docs {l:n1}s; Parse: {events} events {p:n3}s; Emit: {e:n1}s", -// ctx.PartitionKeyRangeId, r.ResponseContinuation.Trim[|'"'|], r.RequestCharge, docs.Count, float sw.ElapsedMilliseconds / 1000., -// events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) -// sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor -// } -// ChangeFeedObserver.Create(Log.Logger, projectBatch, disposeProducer) + |> fun c -> c.CreateLogger() open System.Collections.Concurrent open System.Diagnostics open System.Threading -open System.Threading.Tasks module Ingester = open Equinox.Cosmos.Core @@ -197,8 +179,12 @@ module Ingester = match res with | AppendResult.Ok pos -> return Ok (batch.stream, pos.index) | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> - if pos.index >= batch.pos + batch.events.LongLength then return Duplicate (batch.stream, pos.index) - else return Conflict { stream = batch.stream; pos = pos.index; events = batch.events |> Array.skip (pos.index-(batch.pos+batch.events.LongLength) |> int) } + match pos.index, batch.pos + batch.events.LongLength with + | actual, expectedMax when actual >= expectedMax -> return Duplicate (batch.stream, pos.index) + | actual, _ when batch.pos >= actual -> return Conflict batch + | actual, _ -> + Log.Warning("pos {pos} batch.pos {bpos} len {blen} skip {skio}", actual, batch.pos, batch.events.LongLength, actual-batch.pos) + return Conflict { stream = batch.stream; pos = actual; events = batch.events |> Array.skip (actual-batch.pos |> int) } with e -> return Exn (e, batch) } /// Manages distribution of work across a specified number of concurrent writers @@ -229,7 +215,9 @@ module Ingester = match res with | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) - | Conflict overage -> log.Information("Requeing {stream} {pos} ({count} events)", overage.stream, overage.pos, overage.events.Length) + | Conflict overage -> + log.Warning("Requeing {stream} {pos} ({count} events)", overage.stream, overage.pos, overage.events.Length) + writer.Add overage | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.events.Length) // TODO remove; this is not a sustainable idea @@ -275,8 +263,11 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI let enumEvents (slice : AllEventsSlice) = seq { for e in slice.Events -> match e.Event with - | e when e.IsJson -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) - | e -> Choice2Of2 e + | e when not e.IsJson + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventType = "$statsCollected" -> + Choice2Of2 e + | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) } let mutable total = 0L let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch @@ -312,30 +303,6 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI } fun pos -> loop pos - //let fetchBatches stream pos size = - // let fetchFull pos = async { - // let! currentSlice = source.ReadConnection.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect - // if currentSlice.IsEndOfStream then return None - // else return Some (currentSlice.NextEventNumber,currentSlice.Events) - // } - // AsyncSeq.unfoldAsync fetchFull 0L - - //let fetchSpecific streams = async { - // let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - // for stream in streams do - // sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - // enumStreamEvents currentSlice - // |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - // |> Seq.groupBy (fun (s,_,_) -> s) - // |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) - // |> Array.ofSeq - // |> xs.AddRange - // sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - // if not currentSlice.IsEndOfStream then return! fetch currentSlice.NextPosition - - //} - - let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) let! ingester = Ingester.start(ctx, writerQueueLen, writerCount) let! _ = Async.StartChild (Reader.loadSpecificStreamsTemp (source.ReadConnection, batchSize) streams ingester.Add) @@ -352,7 +319,7 @@ let main argv = let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) let leaseId, startFromHere, batchSize, lagFrequency, streams = args.BuildFeedParams() - let writerQueueLen, writerCount = 10, 2 + let writerQueueLen, writerCount = 16,4 run (destination, colls) source (leaseId, startFromHere, batchSize, writerQueueLen, writerCount, lagFrequency, streams) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 From 29b9ca6108f8297cc3e6e0d63834c7787f99333d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 26 Feb 2019 15:12:45 +0000 Subject: [PATCH 003/353] Bad initial algorithm --- equinox-ingest/Ingest/Program.fs | 224 ++++++++++++++++++++++++------- 1 file changed, 176 insertions(+), 48 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index ae7d4ed80..9e4fbfd48 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -1,14 +1,13 @@ module ProjectorTemplate.Ingester.Program -open Equinox.Cosmos -open Equinox.Store +open Equinox.Store // Infra open FSharp.Control open Serilog open System -open Equinox.EventStore module CmdParser = open Argu + open Equinox.Cosmos type LogEventLevel = Serilog.Events.LogEventLevel exception MissingArg of string @@ -19,7 +18,7 @@ module CmdParser = module Cosmos = type [] Arguments = - | [] ConnectionMode of Equinox.Cosmos.ConnectionMode + | [] ConnectionMode of ConnectionMode | [] Timeout of float | [] Retries of int | [] RetriesWaitTime of int @@ -70,6 +69,7 @@ module CmdParser = | [] Timeout of float | [] Retries of int | [] Host of string + | [] Port of int | [] Username of string | [] Password of string | [] ConcurrentOperationsLimit of int @@ -83,6 +83,7 @@ module CmdParser = | Timeout _ -> "specify operation timeout in seconds (default: 5)." | Retries _ -> "specify operation retries (default: 1)." | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." + | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." | ConcurrentOperationsLimit _ -> "max concurrent operations in flight (default: 5000)." @@ -98,6 +99,7 @@ module CmdParser = member val Cosmos = Cosmos.Info(args.GetResult Cosmos) member __.CreateGateway conn = GesGateway(conn, GesBatchingPolicy(maxBatchSize = args.GetResult(MaxItems,4096))) member __.Host = match args.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" + member __.Port = match args.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int member __.User = match args.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" member __.Password = match args.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" member val CacheStrategy = let c = Caching.Cache("ProjectorTemplate", sizeMb = 50) in CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) @@ -107,7 +109,8 @@ module CmdParser = let concurrentOperationsLimit = args.GetResult(ConcurrentOperationsLimit,5000) log.Information("EventStore {host} heartbeat: {heartbeat}s MaxConcurrentRequests {concurrency} Timeout: {timeout}s Retries {retries}", __.Host, heartbeatTimeout.TotalSeconds, concurrentOperationsLimit, timeout.TotalSeconds, retries) - connect storeLog (heartbeatTimeout, concurrentOperationsLimit) operationThrottling (Discovery.GossipDns __.Host) (__.User,__.Password) connection + let discovery = match __.Port with None -> Discovery.GossipDns __.Host | Some p -> Discovery.GossipDnsCustomPort (__.Host, p) + connect storeLog (heartbeatTimeout, concurrentOperationsLimit) operationThrottling discovery (__.User,__.Password) connection [] type Arguments = @@ -116,6 +119,8 @@ module CmdParser = | [] BatchSize of int | [] LagFreqS of float | [] Verbose + | [] VerboseConsole + | [] LocalSeq | [] Stream of string | [] Es of ParseResults interface IArgParserTemplate with @@ -126,12 +131,16 @@ module CmdParser = | BatchSize _ -> "maximum item count to request from feed. Default: 1000" | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | Verbose -> "request Verbose Logging. Default: off" + | VerboseConsole -> "request Verbose Console Logging. Default: off" + | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Stream _ -> "specify stream(s) to seed the processing with" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) member __.ConsumerGroupName = args.GetResult ConsumerGroupName member __.Verbose = args.Contains Verbose + member __.ConsoleMinLevel = if args.Contains VerboseConsole then LogEventLevel.Information else LogEventLevel.Warning + member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None member __.BatchSize = args.GetResult(BatchSize,1000) member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.StartFromHere = args.Contains ForceStartFromHere @@ -150,13 +159,14 @@ module CmdParser = // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog module Logging = - let initialize verbose = + let initialize verbose consoleMinLevel maybeSeqEndpoint = Log.Logger <- LoggerConfiguration() .Destructure.FSharpTypes() .Enrich.FromLogContext() |> fun c -> if verbose then c.MinimumLevel.Debug() else c - |> fun c -> c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) + |> fun c -> c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) + |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() open System.Collections.Concurrent @@ -167,24 +177,56 @@ module Ingester = open Equinox.Cosmos.Core open Equinox.Cosmos.Store - type [] Batch = { stream: string; pos: int64; events: IEvent[] } + type [] Span = { pos: int64; events: IEvent[] } + module Span = + let private (|Max|) x = x.pos + x.events.LongLength + let private trim min (Max m as x) = + // Full remove + if m <= min then { pos = min; events = [||] } + // Trim until min + elif m > min && x.pos < min then { pos = min; events = x.events |> Array.skip (min - x.pos |> int) } + // Leave it + else x + let merge min (xs : Span seq) = + let buffer = ResizeArray() + let mutable curr = { pos = min; events = [||]} + for x in xs |> Seq.sortBy (fun x -> x.pos) do + match curr, trim min x with + // no data incoming, skip + | _, x when x.events.Length = 0 -> + () + // Not overlapping, no data buffered -> buffer + | c, x when c.events.Length = 0 -> + curr <- x + // Overlapping, join + | Max cMax as c, x when cMax >= x.pos -> + curr <- { c with events = Array.append c.events (trim cMax x).events } + // Not overlapping, new data + | c, x -> + buffer.Add c + curr <- x + if curr.events.Length <> 0 then buffer.Add curr + if buffer.Count = 0 then null else buffer.ToArray() + + type [] Batch = { stream: string; span: Span } type [] Result = | Ok of stream: string * updatedPos: int64 | Duplicate of stream: string * updatedPos: int64 | Conflict of overage: Batch | Exn of exn: exn * batch: Batch - let private write (ctx : CosmosContext) (batch : Batch) = async { - let stream = ctx.CreateStream batch.stream - try let! res = ctx.Sync(stream, { index = batch.pos; etag = None }, batch.events) + let private write (ctx : CosmosContext) ({ stream = s; span={ pos = p; events = e}} as batch) = async { + let stream = ctx.CreateStream s + Log.Information("Writing {s}@{i}x{n}",s,p,e.Length) + try let! res = ctx.Sync(stream, { index = p; etag = None }, e) match res with - | AppendResult.Ok pos -> return Ok (batch.stream, pos.index) + | AppendResult.Ok _pos -> return Ok (s, p) | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> - match pos.index, batch.pos + batch.events.LongLength with - | actual, expectedMax when actual >= expectedMax -> return Duplicate (batch.stream, pos.index) - | actual, _ when batch.pos >= actual -> return Conflict batch + match pos.index, p + e.LongLength with + | actual, expectedMax when actual >= expectedMax -> return Duplicate (s, pos.index) + | actual, _ when p >= actual -> return Conflict batch | actual, _ -> - Log.Warning("pos {pos} batch.pos {bpos} len {blen} skip {skio}", actual, batch.pos, batch.events.LongLength, actual-batch.pos) - return Conflict { stream = batch.stream; pos = actual; events = batch.events |> Array.skip (actual-batch.pos |> int) } + Log.Debug("pos {pos} batch.pos {bpos} len {blen} skip {skio}", actual, p, e.LongLength, actual-p) + return Conflict { stream = s; span = { pos = actual; events = e |> Array.skip (actual-p |> int) } } with e -> return Exn (e, batch) } /// Manages distribution of work across a specified number of concurrent writers @@ -203,42 +245,123 @@ module Ingester = /// Supply an item to be processed member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) [] member __.Result = result.Publish - // TEMP - to be removed - member __.Add item = buffer.Add item - type Queue(log: Serilog.ILogger, writer : Writer) = - member __.TryAdd(item, timeout) = writer.TryAdd(item, timeout) - // TEMP - this blocks, and real impl wont do that - member __.Post item = - writer.Add item - member __.HandleWriteResult res = - match res with - | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) - | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) - | Conflict overage -> - log.Warning("Requeing {stream} {pos} ({count} events)", overage.stream, overage.pos, overage.events.Length) - writer.Add overage - | Exn (exn, batch) -> - log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.events.Length) - // TODO remove; this is not a sustainable idea - writer.Add batch - type Ingester(writer : Writer) = - member __.Add batch = writer.Add batch + type [] StreamState = { read: int64 option; write: int64 option; queue: Span[] } with + member __.IsReady = __.queue <> null && match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false + let inline optionCombine f (r1: int64 option) (r2: int64 option) = + match r1, r2 with + | Some x, Some y -> f x y |> Some + | None, None -> None + | None, x | x, None -> x + + let combine (s1: StreamState) (s2: StreamState) : StreamState = + let writePos = optionCombine max s1.write s2.write + let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } + { read = optionCombine max s1.read s2.read; write = writePos; queue = Span.merge (defaultArg writePos 0L) items} + + type StreamStates() = + let states = System.Collections.Generic.Dictionary() + let dirty = System.Collections.Generic.Queue() + + let update stream (state : StreamState) = + Log.Debug("Updated {s} r{r} w{w}", stream, state.read, state.write) + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + dirty.Enqueue stream + | true, current -> + let updated = combine current state + states.[stream] <- updated + if updated.IsReady then dirty.Enqueue stream + let updateWritePos stream pos queue = + update stream { read = None; write = Some pos; queue = queue } + + member __.Add (item: Batch) = updateWritePos item.stream 0L [|item.span|] + member __.HandleWriteResult = function + | Ok (stream, pos) -> updateWritePos stream pos null + | Duplicate (stream, pos) -> updateWritePos stream pos null + | Conflict overage -> updateWritePos overage.stream overage.span.pos [|overage.span|] + | Exn (_exn, batch) -> __.Add batch + member __.TryPending() = +#if NET461 + if dirty.Count = 0 then None else + let stream = dirty.Dequeue() +#else + match dirty.TryDequeue() with + | false, _ -> None + | true, stream -> +#endif + let state = states.[stream] + if not state.IsReady then None else + match state.queue |> Array.tryHead with + | None -> None + | Some x -> Some { stream = stream; span = { pos = x.pos; events = x.events } } + + type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken) = + let states = StreamStates() + let results = ConcurrentQueue<_>() + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + member __.Add item = work.Add item + member __.HandleWriteResult = results.Enqueue + member __.Pump() = + let fiveMs = TimeSpan.FromMilliseconds 5. + let mutable wip = None + while not cancellationToken.IsCancellationRequested do + let mutable moreResults = true + while moreResults do + match results.TryDequeue() with + | true, res -> + states.HandleWriteResult res + match res with + | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) + | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) + | Conflict overage -> log.Warning( "Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.pos, overage.span.events.Length) + | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) + | false, _ -> moreResults <- false + let mutable moreWork = true + while moreWork do + let wrk = + match wip with + | Some w -> + wip <- None + Some w + | None -> + let pending = states.TryPending() + match pending with + | Some p -> Some p + | None -> + let mutable t = Unchecked.defaultof<_> + if work.TryTake(&t, 5(*ms*), cancellationToken) then + states.Add t + None + else + moreWork <- false + None + match wrk with + | None -> () + | Some w -> + if not (writer.TryAdd(w,fiveMs)) then + moreWork <- false + wip <- Some w + type Ingester(queue : Queue) = + member __.Add batch = queue.Add batch /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does let start(ctx : Equinox.Cosmos.Core.CosmosContext, writerQueueLen, writerCount) = async { let! ct = Async.CancellationToken let writer = Writer(ctx, writerQueueLen, ct) - let queue = Queue(Log.Logger,writer) + let queue = Queue(Log.Logger, writer, ct) let _ = writer.Result.Subscribe queue.HandleWriteResult // codependent, wont worry about unsubcribing writer.StartConsumers writerCount - return Ingester(writer) + do Async.Start(async { queue.Pump() }, ct) + return Ingester(queue) } +open EventStore.ClientAPI + module Reader = - open EventStore.ClientAPI - let loadSpecificStreamsTemp (conn: IEventStoreConnection, batchSize) streams (postBatch : (Ingester.Batch -> unit)) = + let loadSpecificStreamsTemp (conn:IEventStoreConnection, batchSize) streams (postBatch : (Ingester.Batch -> unit)) = let fetchStream stream = let rec fetchFrom pos = async { let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect @@ -247,14 +370,16 @@ module Reader = [| for x in currentSlice.Events -> let e = x.Event Equinox.Cosmos.Store.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Cosmos.Store.IEvent |] - postBatch { stream = stream; pos = currentSlice.FromEventNumber; events = events } + postBatch { stream = stream; span = { pos = currentSlice.FromEventNumber; events = events } } return! fetchFrom currentSlice.NextEventNumber } fetchFrom 0L async { for stream in streams do do! fetchStream stream } -open EventStore.ClientAPI +open Equinox.EventStore +open Equinox.Cosmos + let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseId, forceSkip, batchSize, writerQueueLen, writerCount, lagReportFreq : TimeSpan option, streams: string list) = async { //let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { // Log.Information("Lags by Range {@rangeLags}", remainingWork) @@ -265,7 +390,9 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI match e.Event with | e when not e.IsJson || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventType = "$statsCollected" -> + || e.EventStreamId.StartsWith("$") + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) } @@ -293,8 +420,8 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI let postSw = Stopwatch.StartNew() for s,xs in streams do for pos, item in xs do - postBatch { stream = s; pos = pos; events = [| item |]} - Log.Information("Fetch {count} {ft:n0}ms; Parse c {categories} s {streams}; Post {pt:n0}ms", + postBatch { stream = s; span = { pos = pos; events = [| item |]}} + Log.Warning("Fetch {count} {ft:n0}ms; Parse c {categories} s {streams}; Post {pt:n0}ms", received, sw.ElapsedMilliseconds, cats, streams.Length, postSw.ElapsedMilliseconds) if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", total) @@ -312,14 +439,15 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI [] let main argv = try let args = CmdParser.parse argv - Logging.initialize args.Verbose + Logging.initialize args.Verbose args.ConsoleMinLevel args.MaybeSeqEndpoint let connectionMode = Equinox.EventStore.ConnectionStrategy.ClusterSingle Equinox.EventStore.NodePreference.Master let source = args.EventStore.Connect(Log.Logger, Log.Logger, connectionMode) |> Async.RunSynchronously let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) let leaseId, startFromHere, batchSize, lagFrequency, streams = args.BuildFeedParams() - let writerQueueLen, writerCount = 16,4 + let writerQueueLen, writerCount = 200,32 + if Threading.ThreadPool.SetMaxThreads(512,512) |> not then failwith "Could not set max threads" run (destination, colls) source (leaseId, startFromHere, batchSize, writerQueueLen, writerCount, lagFrequency, streams) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 From 83b3fb5a3a2fb922d31e295032a48017d27f62ed Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 26 Feb 2019 21:32:27 +0000 Subject: [PATCH 004/353] Fix most things, improve logging --- equinox-ingest/Ingest/Program.fs | 46 +++++++++++++++++++------------- 1 file changed, 28 insertions(+), 18 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 9e4fbfd48..63544bcc0 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -262,17 +262,18 @@ module Ingester = type StreamStates() = let states = System.Collections.Generic.Dictionary() let dirty = System.Collections.Generic.Queue() + let markDirty stream = if dirty.Contains stream |> not then dirty.Enqueue stream let update stream (state : StreamState) = Log.Debug("Updated {s} r{r} w{w}", stream, state.read, state.write) match states.TryGetValue stream with | false, _ -> states.Add(stream, state) - dirty.Enqueue stream + markDirty stream |> ignore | true, current -> let updated = combine current state states.[stream] <- updated - if updated.IsReady then dirty.Enqueue stream + if updated.IsReady then markDirty stream |> ignore let updateWritePos stream pos queue = update stream { read = None; write = Some pos; queue = queue } @@ -292,6 +293,7 @@ module Ingester = | true, stream -> #endif let state = states.[stream] + if not state.IsReady then None else match state.queue |> Array.tryHead with | None -> None @@ -305,12 +307,15 @@ module Ingester = member __.HandleWriteResult = results.Enqueue member __.Pump() = let fiveMs = TimeSpan.FromMilliseconds 5. - let mutable wip = None + let mutable pendingWriterAdd = None + let resultsHandled, ingestionsHandled, workPended = ref 0, ref 0, ref 0 + let progressTimer = Stopwatch.StartNew() while not cancellationToken.IsCancellationRequested do let mutable moreResults = true while moreResults do match results.TryDequeue() with | true, res -> + incr resultsHandled states.HandleWriteResult res match res with | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) @@ -318,31 +323,35 @@ module Ingester = | Conflict overage -> log.Warning( "Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.pos, overage.span.events.Length) | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) | false, _ -> moreResults <- false + let mutable t = Unchecked.defaultof<_> + while work.TryTake(&t, 5(*ms*), cancellationToken) do + incr ingestionsHandled + states.Add t let mutable moreWork = true while moreWork do let wrk = - match wip with + match pendingWriterAdd with | Some w -> - wip <- None + pendingWriterAdd <- None Some w | None -> let pending = states.TryPending() match pending with | Some p -> Some p | None -> - let mutable t = Unchecked.defaultof<_> - if work.TryTake(&t, 5(*ms*), cancellationToken) then - states.Add t - None - else - moreWork <- false - None + moreWork <- false + None match wrk with | None -> () | Some w -> if not (writer.TryAdd(w,fiveMs)) then + incr workPended moreWork <- false - wip <- Some w + pendingWriterAdd <- Some w + if progressTimer.ElapsedMilliseconds > 10000L then + progressTimer.Restart() + Log.Warning("Ingested {ingestions} Queued {queued} Completed {completed}", !ingestionsHandled, !workPended, !resultsHandled) + ingestionsHandled := 0; workPended := 0; resultsHandled := 0 type Ingester(queue : Queue) = member __.Add batch = queue.Add batch @@ -411,9 +420,10 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) |> Array.ofSeq - match streams |> Seq.map (fun (_s,xs) -> Array.length xs) |> Seq.sum with - | extracted when extracted <> received -> Log.Warning("Dropped {count} of {total}", received-extracted, received) - | _ -> () + let dropped = + match streams |> Seq.map (fun (_s,xs) -> Array.length xs) |> Seq.sum with + | extracted when extracted <> received -> let dropped = received-extracted in dropped //Log.Warning("Dropped {count} of {total}", dropped, received) + | _ -> 0 let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head let cats = seq { for (s,_) in streams -> category s } |> Seq.distinct |> Seq.length @@ -421,8 +431,8 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI for s,xs in streams do for pos, item in xs do postBatch { stream = s; span = { pos = pos; events = [| item |]}} - Log.Warning("Fetch {count} {ft:n0}ms; Parse c {categories} s {streams}; Post {pt:n0}ms", - received, sw.ElapsedMilliseconds, cats, streams.Length, postSw.ElapsedMilliseconds) + Log.Warning("Fetch {count} Dropped {dropped} {ft:n0}ms Pos {pos}; Parse c {categories} s {streams}; Post {pt:n0}ms", + received, dropped, sw.ElapsedMilliseconds, currentSlice.NextPosition.CommitPosition, cats, streams.Length, postSw.ElapsedMilliseconds) if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", total) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor From ec1e9c78e7b3f8dd56163978a283984a3e344b0c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 27 Feb 2019 17:21:11 +0000 Subject: [PATCH 005/353] Fix error handling, polish logging --- equinox-ingest/Ingest/Program.fs | 179 ++++++++++++++++++------------- 1 file changed, 104 insertions(+), 75 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 63544bcc0..f7ccb45c2 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -114,41 +114,34 @@ module CmdParser = [] type Arguments = - | [] ConsumerGroupName of string - | [] ForceStartFromHere | [] BatchSize of int - | [] LagFreqS of float | [] Verbose | [] VerboseConsole | [] LocalSeq | [] Stream of string + | [] AllPos of int64 | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = match a with - | ConsumerGroupName _ ->"projector group name." - | ForceStartFromHere _ -> "(iff `suffix` represents a fresh LeaseId) - force skip to present Position. Default: Never skip an event on a lease." | BatchSize _ -> "maximum item count to request from feed. Default: 1000" - | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Stream _ -> "specify stream(s) to seed the processing with" + | AllPos _ -> "Specify EventStore $all Stream Position to commence from" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) - member __.ConsumerGroupName = args.GetResult ConsumerGroupName member __.Verbose = args.Contains Verbose member __.ConsoleMinLevel = if args.Contains VerboseConsole then LogEventLevel.Information else LogEventLevel.Warning member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None member __.BatchSize = args.GetResult(BatchSize,1000) - member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds - member __.StartFromHere = args.Contains ForceStartFromHere member x.BuildFeedParams() = - Log.Information("Processing {leaseId} in batches of {batchSize}", x.ConsumerGroupName, x.BatchSize) - if x.StartFromHere then Log.Warning("(If new projector group) Skipping projection of all existing events.") - x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) - x.ConsumerGroupName, x.StartFromHere, x.BatchSize, x.LagFrequency, args.GetResults Stream + Log.Information("Processing in batches of {batchSize}", x.BatchSize) + let startPos = args.TryGetResult AllPos + startPos |> Option.iter (fun p -> Log.Information("Processing will commence at $all Position {p}", p)) + x.BatchSize, args.GetResults Stream, startPos /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args let parse argv : Parameters = @@ -176,6 +169,7 @@ open System.Threading module Ingester = open Equinox.Cosmos.Core open Equinox.Cosmos.Store + open System.Threading.Tasks type [] Span = { pos: int64; events: IEvent[] } module Span = @@ -213,7 +207,13 @@ module Ingester = | Ok of stream: string * updatedPos: int64 | Duplicate of stream: string * updatedPos: int64 | Conflict of overage: Batch - | Exn of exn: exn * batch: Batch + | Exn of exn: exn * batch: Batch with + member __.WriteTo(log: ILogger) = + match __ with + | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) + | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) + | Conflict overage -> log.Information("Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.pos, overage.span.events.Length) + | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) let private write (ctx : CosmosContext) ({ stream = s; span={ pos = p; events = e}} as batch) = async { let stream = ctx.CreateStream s Log.Information("Writing {s}@{i}x{n}",s,p,e.Length) @@ -254,6 +254,10 @@ module Ingester = | None, None -> None | None, x | x, None -> x + let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length + let inline cosmosPayloadBytes (x: IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 + let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = arrayBytes x.Event.Data + arrayBytes x.Event.Metadata + x.OriginalStreamId.Length * 2 + let combine (s1: StreamState) (s2: StreamState) : StreamState = let writePos = optionCombine max s1.write s2.write let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } @@ -295,20 +299,31 @@ module Ingester = let state = states.[stream] if not state.IsReady then None else - match state.queue |> Array.tryHead with - | None -> None - | Some x -> Some { stream = stream; span = { pos = x.pos; events = x.events } } + + match state.queue |> Array.tryHead with + | None -> None + | Some x -> + + let mutable bytesBudget = 1 * 1024 * 1024 + let mutable countBudget = if x.pos = 0L then 10 else 1000 + let max1MbMax1000EventsMax10EventsFirstTranche (x : IEvent) = + bytesBudget <- bytesBudget - cosmosPayloadBytes x + countBudget <- countBudget - 1 + countBudget >= 0 && bytesBudget >= 0 + Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max1MbMax1000EventsMax10EventsFirstTranche } } - type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken) = + type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen) = let states = StreamStates() let results = ConcurrentQueue<_>() - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let work = new BlockingCollection<_>(ConcurrentQueue<_>(), readerQueueLen) + member __.Add item = work.Add item member __.HandleWriteResult = results.Enqueue member __.Pump() = let fiveMs = TimeSpan.FromMilliseconds 5. let mutable pendingWriterAdd = None - let resultsHandled, ingestionsHandled, workPended = ref 0, ref 0, ref 0 + let mutable bytesPended = 0L + let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let progressTimer = Stopwatch.StartNew() while not cancellationToken.IsCancellationRequested do let mutable moreResults = true @@ -317,15 +332,13 @@ module Ingester = | true, res -> incr resultsHandled states.HandleWriteResult res - match res with - | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) - | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) - | Conflict overage -> log.Warning( "Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.pos, overage.span.events.Length) - | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) + res.WriteTo log | false, _ -> moreResults <- false let mutable t = Unchecked.defaultof<_> - while work.TryTake(&t, 5(*ms*), cancellationToken) do + let mutable toIngest = 4096 * 5 + while work.TryTake(&t) && toIngest > 0 do incr ingestionsHandled + toIngest <- toIngest - 1 states.Add t let mutable moreWork = true while moreWork do @@ -345,24 +358,29 @@ module Ingester = | None -> () | Some w -> if not (writer.TryAdd(w,fiveMs)) then - incr workPended moreWork <- false pendingWriterAdd <- Some w + else + incr workPended + eventsPended := !eventsPended + w.span.events.Length + bytesPended <- bytesPended + int64 (Array.sumBy cosmosPayloadBytes w.span.events) + if progressTimer.ElapsedMilliseconds > 10000L then progressTimer.Restart() - Log.Warning("Ingested {ingestions} Queued {queued} Completed {completed}", !ingestionsHandled, !workPended, !resultsHandled) - ingestionsHandled := 0; workPended := 0; resultsHandled := 0 + Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", + !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) + ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 type Ingester(queue : Queue) = member __.Add batch = queue.Add batch /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does - let start(ctx : Equinox.Cosmos.Core.CosmosContext, writerQueueLen, writerCount) = async { + let start(ctx : CosmosContext, writerQueueLen, writerCount, readerQueueLen) = async { let! ct = Async.CancellationToken let writer = Writer(ctx, writerQueueLen, ct) - let queue = Queue(Log.Logger, writer, ct) + let queue = Queue(Log.Logger, writer, ct, readerQueueLen) let _ = writer.Result.Subscribe queue.HandleWriteResult // codependent, wont worry about unsubcribing writer.StartConsumers writerCount - do Async.Start(async { queue.Pump() }, ct) + let! _ = Async.StartChild(async { queue.Pump() }) return Ingester(queue) } @@ -389,11 +407,9 @@ module Reader = open Equinox.EventStore open Equinox.Cosmos -let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseId, forceSkip, batchSize, writerQueueLen, writerCount, lagReportFreq : TimeSpan option, streams: string list) = async { - //let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - // Log.Information("Lags by Range {@rangeLags}", remainingWork) - // return! Async.Sleep interval } - //let maybeLogLag = lagReportFreq |> Option.map logLag +let run (destination : CosmosConnection, colls) (source : GesConnection) + (batchSize, streams: string list, startPos: int64 option) + (writerQueueLen, writerCount, readerQueueLen) = async { let enumEvents (slice : AllEventsSlice) = seq { for e in slice.Events -> match e.Event with @@ -405,60 +421,73 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (leaseI Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) } - let mutable total = 0L - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let followAll (postBatch : Ingester.Batch -> unit) = - let rec loop pos = async { - let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(pos, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let received = currentSlice.Events.Length - total <- total + int64 received - let streams = - enumEvents currentSlice - |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (s,_,_) -> s) - |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) - |> Array.ofSeq - - let dropped = - match streams |> Seq.map (fun (_s,xs) -> Array.length xs) |> Seq.sum with - | extracted when extracted <> received -> let dropped = received-extracted in dropped //Log.Warning("Dropped {count} of {total}", dropped, received) - | _ -> 0 + let mutable totalEvents, totalBytes = 0L, 0L + let followAll (postBatch : Ingester.Batch -> unit) = async { + let mutable currentPos = match startPos with Some p -> EventStore.ClientAPI.Position(p,0L) | None -> Position.Start + let run = async { + let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect + let max = lastItemBatch.NextPosition.CommitPosition + Log.Warning("EventStore Max @{pos:n0}", max) + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let rec loop () = async { + let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(currentPos, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + let cur = currentSlice.NextPosition.CommitPosition + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let received = currentSlice.Events.Length + totalEvents <- totalEvents + int64 received + for x in currentSlice.Events do totalBytes <- totalBytes + int64 (Ingester.esPayloadBytes x) + let streams = + enumEvents currentSlice + |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + |> Seq.groupBy (fun (s,_,_) -> s) + |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) + |> Array.ofSeq - let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head - let cats = seq { for (s,_) in streams -> category s } |> Seq.distinct |> Seq.length - let postSw = Stopwatch.StartNew() - for s,xs in streams do - for pos, item in xs do - postBatch { stream = s; span = { pos = pos; events = [| item |]}} - Log.Warning("Fetch {count} Dropped {dropped} {ft:n0}ms Pos {pos}; Parse c {categories} s {streams}; Post {pt:n0}ms", - received, dropped, sw.ElapsedMilliseconds, currentSlice.NextPosition.CommitPosition, cats, streams.Length, postSw.ElapsedMilliseconds) - if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", total) - - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - if not currentSlice.IsEndOfStream then return! loop currentSlice.NextPosition + let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head + let cats = seq { for (s,_) in streams -> category s } |> Seq.distinct |> Seq.length + let postSw = Stopwatch.StartNew() + let extracted = ref 0 + for s,xs in streams do + for pos, item in xs do + incr extracted + postBatch { stream = s; span = { pos = pos; events = [| item |]}} + Log.Warning("ES {ft:n3}s; Found c {categories} s {streams} e {count}-{dropped} {pt:n0}ms; Ingres {gb:n3}GB @{pos:n0} {pct:p1}", + (let e = sw.Elapsed in e.TotalSeconds), cats, streams.Length, received, received- !extracted, postSw.ElapsedMilliseconds, + float totalBytes / 1024. / 1024. / 1024., cur, float cur/float max) + if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", totalEvents) + sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + if not currentSlice.IsEndOfStream then + currentPos <- currentSlice.NextPosition + return! loop () + } + do! loop () } - fun pos -> loop pos + let mutable finished = false + while not finished do + try do! run + finished <- true + with e -> Log.Warning(e,"Ingestion error") + } let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) - let! ingester = Ingester.start(ctx, writerQueueLen, writerCount) + let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) let! _ = Async.StartChild (Reader.loadSpecificStreamsTemp (source.ReadConnection, batchSize) streams ingester.Add) - let! _ = Async.StartChild (followAll ingester.Add Position.Start) + let! _ = Async.StartChild (followAll ingester.Add) do! Async.AwaitKeyboardInterrupt() } [] let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ConsoleMinLevel args.MaybeSeqEndpoint - let connectionMode = Equinox.EventStore.ConnectionStrategy.ClusterSingle Equinox.EventStore.NodePreference.Master + let connectionMode = ConnectionStrategy.ClusterSingle NodePreference.Master let source = args.EventStore.Connect(Log.Logger, Log.Logger, connectionMode) |> Async.RunSynchronously let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) - let leaseId, startFromHere, batchSize, lagFrequency, streams = args.BuildFeedParams() - let writerQueueLen, writerCount = 200,32 + let batchSize, streams, startPos = args.BuildFeedParams() + let writerQueueLen, writerCount, readerQueueLen = 2048,32,4096*10*10 if Threading.ThreadPool.SetMaxThreads(512,512) |> not then failwith "Could not set max threads" - run (destination, colls) source (leaseId, startFromHere, batchSize, writerQueueLen, writerCount, lagFrequency, streams) |> Async.RunSynchronously + run (destination, colls) source (batchSize, streams, startPos) (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 From baabc4d42089283fc49e795eecf82410acfea519 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 28 Feb 2019 03:48:16 +0000 Subject: [PATCH 006/353] Adjust timeouts, default batch sizes and logging --- equinox-ingest/Ingest/Ingest.fsproj | 5 ++++- equinox-ingest/Ingest/Program.fs | 30 +++++++++++++++-------------- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj index 910a3b777..ba5419255 100644 --- a/equinox-ingest/Ingest/Ingest.fsproj +++ b/equinox-ingest/Ingest/Ingest.fsproj @@ -2,8 +2,9 @@ Exe - netcoreapp2.1 + netcoreapp2.1;net461 5 + $(DefineConstants);NET461 @@ -16,7 +17,9 @@ + + \ No newline at end of file diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index f7ccb45c2..e23a48b6e 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -28,8 +28,8 @@ module CmdParser = interface IArgParserTemplate with member a.Usage = match a with - | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 1)." + | Timeout _ -> "specify operation timeout in seconds (default: 10)." + | Retries _ -> "specify operation retries (default: 0)." | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION, Cosmos Emulator)." | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." @@ -40,9 +40,9 @@ module CmdParser = member __.Database = match args.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" member __.Collection = match args.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" - member __.Timeout = args.GetResult(Timeout,5.) |> TimeSpan.FromSeconds + member __.Timeout = args.GetResult(Timeout,10.) |> TimeSpan.FromSeconds member __.Mode = args.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) - member __.Retries = args.GetResult(Retries, 1) + member __.Retries = args.GetResult(Retries, 0) member __.MaxRetryWaitTime = args.GetResult(RetriesWaitTime, 5) /// Connect with the provided parameters and/or environment variables @@ -80,8 +80,8 @@ module CmdParser = member a.Usage = match a with | VerboseStore -> "Include low level Store logging." - | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 1)." + | Timeout _ -> "specify operation timeout in seconds (default: 20)." + | Retries _ -> "specify operation retries (default: 3)." | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." @@ -104,7 +104,7 @@ module CmdParser = member __.Password = match args.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" member val CacheStrategy = let c = Caching.Cache("ProjectorTemplate", sizeMb = 50) in CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) member __.Connect(log: ILogger, storeLog, connection) = - let (timeout, retries) as operationThrottling = args.GetResult(Timeout,5.) |> TimeSpan.FromSeconds, args.GetResult(Retries,1) + let (timeout, retries) as operationThrottling = args.GetResult(Timeout,20.) |> TimeSpan.FromSeconds, args.GetResult(Retries,3) let heartbeatTimeout = args.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds let concurrentOperationsLimit = args.GetResult(ConcurrentOperationsLimit,5000) log.Information("EventStore {host} heartbeat: {heartbeat}s MaxConcurrentRequests {concurrency} Timeout: {timeout}s Retries {retries}", @@ -124,7 +124,7 @@ module CmdParser = interface IArgParserTemplate with member a.Usage = match a with - | BatchSize _ -> "maximum item count to request from feed. Default: 1000" + | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" @@ -136,7 +136,7 @@ module CmdParser = member __.Verbose = args.Contains Verbose member __.ConsoleMinLevel = if args.Contains VerboseConsole then LogEventLevel.Information else LogEventLevel.Warning member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None - member __.BatchSize = args.GetResult(BatchSize,1000) + member __.BatchSize = args.GetResult(BatchSize,4096) member x.BuildFeedParams() = Log.Information("Processing in batches of {batchSize}", x.BatchSize) let startPos = args.TryGetResult AllPos @@ -427,7 +427,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) let run = async { let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.NextPosition.CommitPosition - Log.Warning("EventStore Max @{pos:n0}", max) + Log.Warning("EventStore Write Position: @ {pos}", max) let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(currentPos, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -435,7 +435,9 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us let received = currentSlice.Events.Length totalEvents <- totalEvents + int64 received - for x in currentSlice.Events do totalBytes <- totalBytes + int64 (Ingester.esPayloadBytes x) + let mutable batchBytes = 0 + for x in currentSlice.Events do batchBytes <- batchBytes + Ingester.esPayloadBytes x + totalBytes <- totalBytes + int64 batchBytes let streams = enumEvents currentSlice |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) @@ -451,9 +453,9 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) for pos, item in xs do incr extracted postBatch { stream = s; span = { pos = pos; events = [| item |]}} - Log.Warning("ES {ft:n3}s; Found c {categories} s {streams} e {count}-{dropped} {pt:n0}ms; Ingres {gb:n3}GB @{pos:n0} {pct:p1}", - (let e = sw.Elapsed in e.TotalSeconds), cats, streams.Length, received, received- !extracted, postSw.ElapsedMilliseconds, - float totalBytes / 1024. / 1024. / 1024., cur, float cur/float max) + Log.Warning("Read {count} {ft:n3}s {mb:n1}MB c {categories,2} s {streams,4} e {events,4} Queue {pt:n0}ms Total {gb:n3}GB @ {pos} {pct:p1}", + received, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., cats, streams.Length, !extracted, + postSw.ElapsedMilliseconds, float totalBytes / 1024. / 1024. / 1024., cur, float cur/float max) if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", totalEvents) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor if not currentSlice.IsEndOfStream then From ef4e9ebaada27cd6e17608e03423bf35f8c50a4d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 1 Mar 2019 17:21:49 +0000 Subject: [PATCH 007/353] Fix waits, add size guard for events too large for cosmos --- equinox-ingest/Ingest/Program.fs | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index e23a48b6e..0cc0dc1e8 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -169,7 +169,6 @@ open System.Threading module Ingester = open Equinox.Cosmos.Core open Equinox.Cosmos.Store - open System.Threading.Tasks type [] Span = { pos: int64; events: IEvent[] } module Span = @@ -255,8 +254,10 @@ module Ingester = | None, x | x, None -> x let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length + let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 let inline cosmosPayloadBytes (x: IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 - let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = arrayBytes x.Event.Data + arrayBytes x.Event.Metadata + x.OriginalStreamId.Length * 2 + let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata + let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 let combine (s1: StreamState) (s2: StreamState) : StreamState = let writePos = optionCombine max s1.write s2.write @@ -304,13 +305,13 @@ module Ingester = | None -> None | Some x -> - let mutable bytesBudget = 1 * 1024 * 1024 - let mutable countBudget = if x.pos = 0L then 10 else 1000 - let max1MbMax1000EventsMax10EventsFirstTranche (x : IEvent) = - bytesBudget <- bytesBudget - cosmosPayloadBytes x - countBudget <- countBudget - 1 - countBudget >= 0 && bytesBudget >= 0 - Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max1MbMax1000EventsMax10EventsFirstTranche } } + let mutable bytesBudget = cosmosPayloadLimit + let mutable count = 0 + let max2MbMax1000EventsMax10EventsFirstTranche (y : IEvent) = + bytesBudget <- bytesBudget - cosmosPayloadBytes y + count <- count + 1 + count < (if x.pos = 0L then 10 else 1000) && (bytesBudget >= 0 || count = 1) + Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen) = let states = StreamStates() @@ -336,7 +337,7 @@ module Ingester = | false, _ -> moreResults <- false let mutable t = Unchecked.defaultof<_> let mutable toIngest = 4096 * 5 - while work.TryTake(&t) && toIngest > 0 do + while work.TryTake(&t,fiveMs) && toIngest > 0 do incr ingestionsHandled toIngest <- toIngest - 1 states.Add t @@ -412,13 +413,18 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) (writerQueueLen, writerCount, readerQueueLen) = async { let enumEvents (slice : AllEventsSlice) = seq { for e in slice.Events -> + let eb = Ingester.esPayloadBytes e match e.Event with | e when not e.IsJson || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.StartsWith("$") || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> Choice2Of2 e + | e when eb > Ingester.cosmosPayloadLimit -> + Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) + Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) } let mutable totalEvents, totalBytes = 0L, 0L @@ -487,7 +493,7 @@ let main argv = let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) let batchSize, streams, startPos = args.BuildFeedParams() - let writerQueueLen, writerCount, readerQueueLen = 2048,32,4096*10*10 + let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 if Threading.ThreadPool.SetMaxThreads(512,512) |> not then failwith "Could not set max threads" run (destination, colls) source (batchSize, streams, startPos) (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 From 06ce5734ac25e0765878bd8517731deeb09f19f2 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 7 Mar 2019 13:40:52 +0000 Subject: [PATCH 008/353] Use Equinox 2.0.0-preview1 --- equinox-ingest/Ingest/Ingest.fsproj | 4 ++-- equinox-ingest/Ingest/Program.fs | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj index ba5419255..ca8d76add 100644 --- a/equinox-ingest/Ingest/Ingest.fsproj +++ b/equinox-ingest/Ingest/Ingest.fsproj @@ -15,8 +15,8 @@ - - + + diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 0cc0dc1e8..ace2b6c35 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -170,7 +170,7 @@ module Ingester = open Equinox.Cosmos.Core open Equinox.Cosmos.Store - type [] Span = { pos: int64; events: IEvent[] } + type [] Span = { pos: int64; events: Equinox.Codec.IEvent[] } module Span = let private (|Max|) x = x.pos + x.events.LongLength let private trim min (Max m as x) = @@ -255,7 +255,7 @@ module Ingester = let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 - let inline cosmosPayloadBytes (x: IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 + let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 @@ -307,7 +307,7 @@ module Ingester = let mutable bytesBudget = cosmosPayloadLimit let mutable count = 0 - let max2MbMax1000EventsMax10EventsFirstTranche (y : IEvent) = + let max2MbMax1000EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 count < (if x.pos = 0L then 10 else 1000) && (bytesBudget >= 0 || count = 1) @@ -397,7 +397,7 @@ module Reader = let events = [| for x in currentSlice.Events -> let e = x.Event - Equinox.Cosmos.Store.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Cosmos.Store.IEvent |] + Equinox.Codec.Core.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Codec.IEvent |] postBatch { stream = stream; span = { pos = currentSlice.FromEventNumber; events = events } } return! fetchFrom currentSlice.NextEventNumber } fetchFrom 0L @@ -425,7 +425,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) | e when eb > Ingester.cosmosPayloadLimit -> Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) Choice2Of2 e - | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Cosmos.Store.EventData.Create(e.EventType, e.Data, e.Metadata)) + | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) } let mutable totalEvents, totalBytes = 0L, 0L let followAll (postBatch : Ingester.Batch -> unit) = async { From cb22c8fffdc7f8ec017e75c23aff2c3cc8d4547a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 10 Mar 2019 02:12:48 +0000 Subject: [PATCH 009/353] Move test from other repo --- equinox-ingest/Ingest/Infrastructure.fs | 2 +- equinox-ingest/Ingest/Program.fs | 2 +- .../Sync.Tests/Sync.Tests/Sync.Tests.fsproj | 25 +++++++ equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs | 70 +++++++++++++++++++ 4 files changed, 97 insertions(+), 2 deletions(-) create mode 100644 equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj create mode 100644 equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs index 5942e97c1..d8c3fa2b3 100644 --- a/equinox-ingest/Ingest/Infrastructure.fs +++ b/equinox-ingest/Ingest/Infrastructure.fs @@ -1,5 +1,5 @@ [] -module private ProjectorTemplate.Ingester.Infrastructure +module private SyncTemplate.Infrastructure open System open System.Threading diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index ace2b6c35..7eee61c69 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -1,4 +1,4 @@ -module ProjectorTemplate.Ingester.Program +module SyncTemplate.Program open Equinox.Store // Infra open FSharp.Control diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj b/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj new file mode 100644 index 000000000..b1a0d7f20 --- /dev/null +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj @@ -0,0 +1,25 @@ + + + + netcoreapp2.1 + + false + + + + + + + + + + + + + + + + + + + diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs new file mode 100644 index 000000000..7a66acb10 --- /dev/null +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs @@ -0,0 +1,70 @@ +module SyncTemplate.Tests.IngesterTests + +open Swensen.Unquote +open SyncTemplate.Program.Ingester +open Xunit + +//type Span = { pos: int64; events: string[] } + +//let (|Max|) x = x.pos + x.events.LongLength +//let trim min (Max m as x) = +// // Full remove +// if m <= min then { pos = min; events = [||] } +// // Trim until min +// elif m > min && x.pos < min then { pos = min; events = x.events |> Array.skip (min - x.pos |> int) } +// // Leave it +// else x +//let mergeSpans min (xs : Span seq) = +// let buffer = ResizeArray() +// let mutable curr = { pos = min; events = [||]} +// for x in xs |> Seq.sortBy (fun s -> s.pos) do +// match curr, trim min x with +// // no data incoming, skip +// | _, x when x.events.Length = 0 -> +// () +// // Not overlapping, no data buffered -> buffer +// | c, x when c.events.Length = 0 -> +// curr <- x +// // Overlapping, join +// | Max cMax as c, x when cMax >= x.pos -> +// curr <- { c with events = Array.append c.events (trim cMax x).events } +// // Not overlapping, new data +// | c, x -> +// buffer.Add c +// curr <- x +// if curr.events.Length <> 0 then buffer.Add curr +// if buffer.Count = 0 then null else buffer.ToArray() + +let mk p c : Span = { pos = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null) |] } +let mergeSpans = Span.merge +let [] ``nothing`` () = + let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] + r =! null + +let [] ``no overlap`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] + r =! [| mk 0L 1; mk 2L 2 |] + +let [] ``overlap`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] + r =! [| mk 0L 2 |] + +let [] ``remove nulls`` () = + let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] + r =! [| mk 1L 1 |] + +let [] ``adjacent`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] + r =! [| mk 0L 3 |] + +let [] ``adjacent trim`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] + r =! [| mk 1L 3 |] + +let [] ``adjacent trim append`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] + r =! [| mk 1L 3; mk 5L 1 |] + +let [] ``mixed adjacent trim append`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2; ] + r =! [| mk 1L 3; mk 5L 1 |] \ No newline at end of file From 8d5241fb89f63e5bf7ff5dd9c40cb6c8814f2044 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 10 Mar 2019 02:15:23 +0000 Subject: [PATCH 010/353] Tidy --- equinox-ingest/Ingest/Program.fs | 14 ++++---- equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs | 32 +------------------ 2 files changed, 8 insertions(+), 38 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 7eee61c69..e4fd8e6de 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -124,13 +124,13 @@ module CmdParser = interface IArgParserTemplate with member a.Usage = match a with - | BatchSize _ -> "maximum item count to request from feed. Default: 4096" - | Verbose -> "request Verbose Logging. Default: off" - | VerboseConsole -> "request Verbose Console Logging. Default: off" - | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | Stream _ -> "specify stream(s) to seed the processing with" - | AllPos _ -> "Specify EventStore $all Stream Position to commence from" - | Es _ -> "specify EventStore parameters" + | BatchSize _ -> "maximum item count to request from feed. Default: 4096" + | Verbose -> "request Verbose Logging. Default: off" + | VerboseConsole -> "request Verbose Console Logging. Default: off" + | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" + | Stream _ -> "specify stream(s) to seed the processing with" + | AllPos _ -> "Specify EventStore $all Stream Position to commence from" + | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) member __.Verbose = args.Contains Verbose diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs index 7a66acb10..ab661da87 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs @@ -4,39 +4,9 @@ open Swensen.Unquote open SyncTemplate.Program.Ingester open Xunit -//type Span = { pos: int64; events: string[] } - -//let (|Max|) x = x.pos + x.events.LongLength -//let trim min (Max m as x) = -// // Full remove -// if m <= min then { pos = min; events = [||] } -// // Trim until min -// elif m > min && x.pos < min then { pos = min; events = x.events |> Array.skip (min - x.pos |> int) } -// // Leave it -// else x -//let mergeSpans min (xs : Span seq) = -// let buffer = ResizeArray() -// let mutable curr = { pos = min; events = [||]} -// for x in xs |> Seq.sortBy (fun s -> s.pos) do -// match curr, trim min x with -// // no data incoming, skip -// | _, x when x.events.Length = 0 -> -// () -// // Not overlapping, no data buffered -> buffer -// | c, x when c.events.Length = 0 -> -// curr <- x -// // Overlapping, join -// | Max cMax as c, x when cMax >= x.pos -> -// curr <- { c with events = Array.append c.events (trim cMax x).events } -// // Not overlapping, new data -// | c, x -> -// buffer.Add c -// curr <- x -// if curr.events.Length <> 0 then buffer.Add curr -// if buffer.Count = 0 then null else buffer.ToArray() - let mk p c : Span = { pos = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null) |] } let mergeSpans = Span.merge + let [] ``nothing`` () = let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] r =! null From d9fef919055084678fbc52d394230e47880d3a9a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 13 Mar 2019 16:51:38 +0000 Subject: [PATCH 011/353] Implement percentage start --- equinox-ingest/Ingest/Program.fs | 38 ++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 7 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index e4fd8e6de..95f385671 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -120,6 +120,7 @@ module CmdParser = | [] LocalSeq | [] Stream of string | [] AllPos of int64 + | [] PercentagePos of float | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = @@ -130,6 +131,7 @@ module CmdParser = | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Stream _ -> "specify stream(s) to seed the processing with" | AllPos _ -> "Specify EventStore $all Stream Position to commence from" + | PercentagePos _ -> "Specify EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) @@ -139,8 +141,11 @@ module CmdParser = member __.BatchSize = args.GetResult(BatchSize,4096) member x.BuildFeedParams() = Log.Information("Processing in batches of {batchSize}", x.BatchSize) - let startPos = args.TryGetResult AllPos - startPos |> Option.iter (fun p -> Log.Information("Processing will commence at $all Position {p}", p)) + let startPos = + match args.TryGetResult AllPos, args.TryGetResult PercentagePos with + | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Choice1Of3 p + | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P0}", p/100.); Choice2Of3 p + | None, None -> Log.Warning "Processing will commence at start"; Choice3Of3 () x.BatchSize, args.GetResults Stream, startPos /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args @@ -409,7 +414,7 @@ open Equinox.EventStore open Equinox.Cosmos let run (destination : CosmosConnection, colls) (source : GesConnection) - (batchSize, streams: string list, startPos: int64 option) + (batchSize, streams: string list, startPos: Choice) (writerQueueLen, writerCount, readerQueueLen) = async { let enumEvents (slice : AllEventsSlice) = seq { for e in slice.Events -> @@ -429,11 +434,18 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) } let mutable totalEvents, totalBytes = 0L, 0L let followAll (postBatch : Ingester.Batch -> unit) = async { - let mutable currentPos = match startPos with Some p -> EventStore.ClientAPI.Position(p,0L) | None -> Position.Start - let run = async { + let fetchMax = async { let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.NextPosition.CommitPosition Log.Warning("EventStore Write Position: @ {pos}", max) + return max + } + let mutable currentPos = + match startPos with + | Choice1Of3 p -> EventStore.ClientAPI.Position(p,0L) + | Choice2Of3 _ -> Position.End // placeholder, will be overwritten below + | Choice3Of3 () -> Position.Start + let run max = async { let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(currentPos, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -471,14 +483,26 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) do! loop () } let mutable finished = false + let mutable max = None while not finished do - try do! run + try if max = None then + let! currMax = fetchMax + max <- Some currMax + match startPos with + | Choice2Of3 pct -> + let rawPos = float currMax * pct / 100. |> int64 + let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L ;) + // event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk + Log.Warning("Start: {pos}",chunkBase) + currentPos <- EventStore.ClientAPI.Position(chunkBase,0L) + | _ -> () + do! run (Option.get max) finished <- true with e -> Log.Warning(e,"Ingestion error") } let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) - let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) + let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) let! _ = Async.StartChild (Reader.loadSpecificStreamsTemp (source.ReadConnection, batchSize) streams ingester.Add) let! _ = Async.StartChild (followAll ingester.Add) do! Async.AwaitKeyboardInterrupt() } From 5791826ad46ace2dc70cc7d257f91e8669385b95 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 14 Mar 2019 16:29:17 +0000 Subject: [PATCH 012/353] Add Category stats --- equinox-ingest/Ingest/Program.fs | 83 +++++++++++++++++++------------- 1 file changed, 49 insertions(+), 34 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 95f385671..e539fb515 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -7,7 +7,6 @@ open System module CmdParser = open Argu - open Equinox.Cosmos type LogEventLevel = Serilog.Events.LogEventLevel exception MissingArg of string @@ -17,6 +16,7 @@ module CmdParser = | x -> x module Cosmos = + open Equinox.Cosmos type [] Arguments = | [] ConnectionMode of ConnectionMode | [] Timeout of float @@ -143,9 +143,9 @@ module CmdParser = Log.Information("Processing in batches of {batchSize}", x.BatchSize) let startPos = match args.TryGetResult AllPos, args.TryGetResult PercentagePos with - | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Choice1Of3 p - | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P0}", p/100.); Choice2Of3 p - | None, None -> Log.Warning "Processing will commence at start"; Choice3Of3 () + | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Choice1Of3 p + | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P0}", p/100.); Choice2Of3 p + | None, None -> Log.Warning "Processing will commence at $all Start"; Choice3Of3 () x.BatchSize, args.GetResults Stream, startPos /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args @@ -250,8 +250,8 @@ module Ingester = member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) [] member __.Result = result.Publish - type [] StreamState = { read: int64 option; write: int64 option; queue: Span[] } with - member __.IsReady = __.queue <> null && match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false + type [] StreamState = { read: int64 option; write: int64 option; isMalformed : bool; queue: Span[] } with + member __.IsReady = __.queue <> null && not __.isMalformed && match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false let inline optionCombine f (r1: int64 option) (r2: int64 option) = match r1, r2 with | Some x, Some y -> f x y |> Some @@ -263,11 +263,13 @@ module Ingester = let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + let isMalformedException (e: #exn) = + e.ToString().Contains "SyntaxError: JSON.parse Error: Unexpected input at position" let combine (s1: StreamState) (s2: StreamState) : StreamState = let writePos = optionCombine max s1.write s2.write let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } - { read = optionCombine max s1.read s2.read; write = writePos; queue = Span.merge (defaultArg writePos 0L) items} + { read = optionCombine max s1.read s2.read; write = writePos; isMalformed = s1.isMalformed || s2.isMalformed; queue = Span.merge (defaultArg writePos 0L) items} type StreamStates() = let states = System.Collections.Generic.Dictionary() @@ -284,15 +286,15 @@ module Ingester = let updated = combine current state states.[stream] <- updated if updated.IsReady then markDirty stream |> ignore - let updateWritePos stream pos queue = - update stream { read = None; write = Some pos; queue = queue } + let updateWritePos stream pos isMalformed span = + update stream { read = None; write = Some pos; isMalformed = isMalformed; queue = span } - member __.Add (item: Batch) = updateWritePos item.stream 0L [|item.span|] + member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream 0L (defaultArg isMalformed false) [|item.span|] member __.HandleWriteResult = function - | Ok (stream, pos) -> updateWritePos stream pos null - | Duplicate (stream, pos) -> updateWritePos stream pos null - | Conflict overage -> updateWritePos overage.stream overage.span.pos [|overage.span|] - | Exn (_exn, batch) -> __.Add batch + | Ok (stream, pos) -> updateWritePos stream pos false null + | Duplicate (stream, pos) -> updateWritePos stream pos false null + | Conflict overage -> updateWritePos overage.stream overage.span.pos false [|overage.span|] + | Exn (exn, batch) -> __.Add(batch,isMalformedException exn) member __.TryPending() = #if NET461 if dirty.Count = 0 then None else @@ -371,7 +373,7 @@ module Ingester = eventsPended := !eventsPended + w.span.events.Length bytesPended <- bytesPended + int64 (Array.sumBy cosmosPayloadBytes w.span.events) - if progressTimer.ElapsedMilliseconds > 10000L then + if progressTimer.ElapsedMilliseconds > 60L * 1000L then progressTimer.Restart() Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) @@ -432,19 +434,35 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) } - let mutable totalEvents, totalBytes = 0L, 0L + let fetchMax = async { + let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect + let max = lastItemBatch.NextPosition.CommitPosition + Log.Warning("EventStore Write Position: @ {pos}", max) + return max } + let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head let followAll (postBatch : Ingester.Batch -> unit) = async { - let fetchMax = async { - let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let max = lastItemBatch.NextPosition.CommitPosition - Log.Warning("EventStore Write Position: @ {pos}", max) - return max - } + let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() + let gatherStats (slice: AllEventsSlice) = + let mutable batchBytes = 0 + for x in slice.Events do + let cat = category x.OriginalStreamId + let eventBytes = Ingester.esPayloadBytes x + match recentCats.TryGetValue cat with + | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) + | false, _ -> recentCats.[cat] <- (1, eventBytes) + batchBytes <- batchBytes + eventBytes + batchBytes + let dumpCatStatsIfNecessary () = + if accStart.ElapsedMilliseconds > 1000L * 60L then + Log.Warning("Top MB/cat/count {@cats}", recentCats |> Seq.sortByDescending (fun x -> snd x.Value) |> Seq.truncate 10 |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c)) + recentCats.Clear() + accStart.Restart() let mutable currentPos = match startPos with | Choice1Of3 p -> EventStore.ClientAPI.Position(p,0L) | Choice2Of3 _ -> Position.End // placeholder, will be overwritten below | Choice3Of3 () -> Position.Start + let mutable totalEvents, totalBytes = 0L, 0L let run max = async { let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { @@ -453,18 +471,17 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us let received = currentSlice.Events.Length totalEvents <- totalEvents + int64 received - let mutable batchBytes = 0 - for x in currentSlice.Events do batchBytes <- batchBytes + Ingester.esPayloadBytes x + let batchBytes = gatherStats currentSlice totalBytes <- totalBytes + int64 batchBytes let streams = enumEvents currentSlice |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (s,_,_) -> s) - |> Seq.map (fun (s,xs) -> s, [| for _s, i, e in xs -> i, e |]) + |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) + |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) |> Array.ofSeq - let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head - let cats = seq { for (s,_) in streams -> category s } |> Seq.distinct |> Seq.length + let usedCats = streams |> Seq.map fst |> Seq.distinct |> Seq.length + dumpCatStatsIfNecessary () let postSw = Stopwatch.StartNew() let extracted = ref 0 for s,xs in streams do @@ -472,16 +489,14 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) incr extracted postBatch { stream = s; span = { pos = pos; events = [| item |]}} Log.Warning("Read {count} {ft:n3}s {mb:n1}MB c {categories,2} s {streams,4} e {events,4} Queue {pt:n0}ms Total {gb:n3}GB @ {pos} {pct:p1}", - received, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., cats, streams.Length, !extracted, + received, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., usedCats, streams.Length, !extracted, postSw.ElapsedMilliseconds, float totalBytes / 1024. / 1024. / 1024., cur, float cur/float max) if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", totalEvents) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor if not currentSlice.IsEndOfStream then currentPos <- currentSlice.NextPosition - return! loop () - } - do! loop () - } + return! loop () } + do! loop () } let mutable finished = false let mutable max = None while not finished do @@ -491,8 +506,8 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) match startPos with | Choice2Of3 pct -> let rawPos = float currMax * pct / 100. |> int64 + // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L ;) - // event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk Log.Warning("Start: {pos}",chunkBase) currentPos <- EventStore.ClientAPI.Position(chunkBase,0L) | _ -> () From 3af0322326b098c1657f5a7f1aacdd52b49fef4b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 14 Mar 2019 16:43:41 +0000 Subject: [PATCH 013/353] Tidy breakdown stats --- equinox-ingest/Ingest/Program.fs | 51 ++++++++++++---------- equinox-ingest/Sync.Tests/equinox-sync.sln | 35 +++++++++++++++ 2 files changed, 64 insertions(+), 22 deletions(-) create mode 100644 equinox-ingest/Sync.Tests/equinox-sync.sln diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index e539fb515..e0ddfa577 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -378,8 +378,6 @@ module Ingester = Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 - type Ingester(queue : Queue) = - member __.Add batch = queue.Add batch /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does let start(ctx : CosmosContext, writerQueueLen, writerCount, readerQueueLen) = async { @@ -389,7 +387,7 @@ module Ingester = let _ = writer.Result.Subscribe queue.HandleWriteResult // codependent, wont worry about unsubcribing writer.StartConsumers writerCount let! _ = Async.StartChild(async { queue.Pump() }) - return Ingester(queue) + return queue } open EventStore.ClientAPI @@ -415,6 +413,31 @@ module Reader = open Equinox.EventStore open Equinox.Cosmos +let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head + +type SliceStatsBuffer(?interval) = + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 + let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() + member __.Ingest(slice: AllEventsSlice) = + let mutable batchBytes = 0 + for x in slice.Events do + let cat = category x.OriginalStreamId + let eventBytes = Ingester.esPayloadBytes x + match recentCats.TryGetValue cat with + | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) + | false, _ -> recentCats.[cat] <- (1, eventBytes) + batchBytes <- batchBytes + eventBytes + batchBytes + member __.DumpIfIntervalExpired() = + if accStart.ElapsedMilliseconds > intervalMs then + Log.Warning("Breakdown MB/cat/count {@cats}", + recentCats + |> Seq.sortByDescending (fun x -> snd x.Value) + |> Seq.truncate 10 + |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c)) + recentCats.Clear() + accStart.Restart() + let run (destination : CosmosConnection, colls) (source : GesConnection) (batchSize, streams: string list, startPos: Choice) (writerQueueLen, writerCount, readerQueueLen) = async { @@ -439,24 +462,8 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) let max = lastItemBatch.NextPosition.CommitPosition Log.Warning("EventStore Write Position: @ {pos}", max) return max } - let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head + let stats = SliceStatsBuffer() let followAll (postBatch : Ingester.Batch -> unit) = async { - let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() - let gatherStats (slice: AllEventsSlice) = - let mutable batchBytes = 0 - for x in slice.Events do - let cat = category x.OriginalStreamId - let eventBytes = Ingester.esPayloadBytes x - match recentCats.TryGetValue cat with - | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) - | false, _ -> recentCats.[cat] <- (1, eventBytes) - batchBytes <- batchBytes + eventBytes - batchBytes - let dumpCatStatsIfNecessary () = - if accStart.ElapsedMilliseconds > 1000L * 60L then - Log.Warning("Top MB/cat/count {@cats}", recentCats |> Seq.sortByDescending (fun x -> snd x.Value) |> Seq.truncate 10 |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c)) - recentCats.Clear() - accStart.Restart() let mutable currentPos = match startPos with | Choice1Of3 p -> EventStore.ClientAPI.Position(p,0L) @@ -471,7 +478,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us let received = currentSlice.Events.Length totalEvents <- totalEvents + int64 received - let batchBytes = gatherStats currentSlice + let batchBytes = stats.Ingest currentSlice totalBytes <- totalBytes + int64 batchBytes let streams = enumEvents currentSlice @@ -481,7 +488,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) |> Array.ofSeq let usedCats = streams |> Seq.map fst |> Seq.distinct |> Seq.length - dumpCatStatsIfNecessary () + stats.DumpIfIntervalExpired() let postSw = Stopwatch.StartNew() let extracted = ref 0 for s,xs in streams do diff --git a/equinox-ingest/Sync.Tests/equinox-sync.sln b/equinox-ingest/Sync.Tests/equinox-sync.sln new file mode 100644 index 000000000..d14233807 --- /dev/null +++ b/equinox-ingest/Sync.Tests/equinox-sync.sln @@ -0,0 +1,35 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio 15 +VisualStudioVersion = 15.0.27428.2002 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{458FF6DD-5F7F-419D-8440-1DCD72AF5229}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "Sync\Sync.fsproj", "{EA288107-08F6-4766-9DD5-E1C961AFA918}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B4CC54BB-AA13-47B2-BF18-C66423EAF19A}" + ProjectSection(SolutionItems) = preProject + equinox-sync.sln = equinox-sync.sln + EndProjectSection +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Debug|Any CPU.Build.0 = Debug|Any CPU + {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Release|Any CPU.ActiveCfg = Release|Any CPU + {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Release|Any CPU.Build.0 = Release|Any CPU + {EA288107-08F6-4766-9DD5-E1C961AFA918}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EA288107-08F6-4766-9DD5-E1C961AFA918}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EA288107-08F6-4766-9DD5-E1C961AFA918}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EA288107-08F6-4766-9DD5-E1C961AFA918}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {A3702E33-9940-4A1D-917C-83F8F8828573} + EndGlobalSection +EndGlobal From 10dda19eeeb8792724fd16540f5d3ad64d419ac7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 14 Mar 2019 23:27:07 +0000 Subject: [PATCH 014/353] Dump States --- equinox-ingest/Ingest/Program.fs | 45 +++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 13 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index e0ddfa577..8bf43bec8 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -250,21 +250,28 @@ module Ingester = member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) [] member __.Result = result.Publish + let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length + type [] StreamState = { read: int64 option; write: int64 option; isMalformed : bool; queue: Span[] } with - member __.IsReady = __.queue <> null && not __.isMalformed && match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false + member __.IsHead = __.queue <> null && match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false + member __.IsReady = __.IsHead && not __.isMalformed + member __.Size = + if __.queue = null then 0 + else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length*2 + 16) + let inline optionCombine f (r1: int64 option) (r2: int64 option) = match r1, r2 with | Some x, Some y -> f x y |> Some | None, None -> None | None, x | x, None -> x - let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 let isMalformedException (e: #exn) = e.ToString().Contains "SyntaxError: JSON.parse Error: Unexpected input at position" + || e.ToString().Contains "SyntaxError: JSON.parse Error: Invalid character at position" let combine (s1: StreamState) (s2: StreamState) : StreamState = let writePos = optionCombine max s1.write s2.write @@ -291,10 +298,10 @@ module Ingester = member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream 0L (defaultArg isMalformed false) [|item.span|] member __.HandleWriteResult = function - | Ok (stream, pos) -> updateWritePos stream pos false null - | Duplicate (stream, pos) -> updateWritePos stream pos false null - | Conflict overage -> updateWritePos overage.stream overage.span.pos false [|overage.span|] - | Exn (exn, batch) -> __.Add(batch,isMalformedException exn) + | Ok (stream, pos) -> updateWritePos stream pos false null; true + | Duplicate (stream, pos) -> updateWritePos stream pos false null; true + | Conflict overage -> updateWritePos overage.stream overage.span.pos false [|overage.span|]; true + | Exn (exn, batch) -> let malformed = isMalformedException exn in __.Add(batch,malformed); not malformed member __.TryPending() = #if NET461 if dirty.Count = 0 then None else @@ -319,8 +326,20 @@ module Ingester = count <- count + 1 count < (if x.pos = 0L then 10 else 1000) && (bytesBudget >= 0 || count = 1) Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } - - type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen) = + member __.Dump() = + let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 + let mutable syncedB, readyB, waitingB, malformedB = 0L, 0L, 0L, 0L + for x in states do + let sz = x.Value.Size + if x.Value.isMalformed then malformed <- malformed + 1; malformedB <- malformedB + int64 sz + elif sz = 0 then synced <- synced + 1; syncedB <- syncedB + int64 sz + elif x.Value.IsReady then ready <- ready + 1; readyB <- readyB + int64 sz + else waiting <- waiting + 1; waitingB <- waitingB + int64 sz + let mb x = x / 1024L / 1024L + Log.Warning("Queued {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}/{syncedMb}MB", + dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced, mb syncedB) + type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen, ?interval) = + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let states = StreamStates() let results = ConcurrentQueue<_>() let work = new BlockingCollection<_>(ConcurrentQueue<_>(), readerQueueLen) @@ -332,15 +351,14 @@ module Ingester = let mutable pendingWriterAdd = None let mutable bytesPended = 0L let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let progressTimer = Stopwatch.StartNew() + let progressTimer, stateTimer = Stopwatch.StartNew(), Stopwatch.StartNew() while not cancellationToken.IsCancellationRequested do let mutable moreResults = true while moreResults do match results.TryDequeue() with | true, res -> incr resultsHandled - states.HandleWriteResult res - res.WriteTo log + if states.HandleWriteResult res then res.WriteTo log | false, _ -> moreResults <- false let mutable t = Unchecked.defaultof<_> let mutable toIngest = 4096 * 5 @@ -373,11 +391,12 @@ module Ingester = eventsPended := !eventsPended + w.span.events.Length bytesPended <- bytesPended + int64 (Array.sumBy cosmosPayloadBytes w.span.events) - if progressTimer.ElapsedMilliseconds > 60L * 1000L then + if progressTimer.ElapsedMilliseconds > intervalMs then progressTimer.Restart() Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 + states.Dump() /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does let start(ctx : CosmosContext, writerQueueLen, writerCount, readerQueueLen) = async { @@ -430,7 +449,7 @@ type SliceStatsBuffer(?interval) = batchBytes member __.DumpIfIntervalExpired() = if accStart.ElapsedMilliseconds > intervalMs then - Log.Warning("Breakdown MB/cat/count {@cats}", + Log.Warning("Cats MB/cat/count {@cats}", recentCats |> Seq.sortByDescending (fun x -> snd x.Value) |> Seq.truncate 10 From fa43594f6a647618e617ebb897fc6e8b93fd47a1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 15 Mar 2019 02:05:41 +0000 Subject: [PATCH 015/353] Add malformed category logging --- equinox-ingest/Ingest/Program.fs | 53 +++++++++++++++++++------------- 1 file changed, 31 insertions(+), 22 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 8bf43bec8..cb2e047c1 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -144,7 +144,7 @@ module CmdParser = let startPos = match args.TryGetResult AllPos, args.TryGetResult PercentagePos with | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Choice1Of3 p - | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P0}", p/100.); Choice2Of3 p + | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Choice2Of3 p | None, None -> Log.Warning "Processing will commence at $all Start"; Choice3Of3 () x.BatchSize, args.GetResults Stream, startPos @@ -253,8 +253,8 @@ module Ingester = let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length type [] StreamState = { read: int64 option; write: int64 option; isMalformed : bool; queue: Span[] } with - member __.IsHead = __.queue <> null && match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false - member __.IsReady = __.IsHead && not __.isMalformed + member __.IsHead = match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false + member __.IsReady = __.queue <> null && not __.isMalformed && __.IsHead member __.Size = if __.queue = null then 0 else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length*2 + 16) @@ -269,6 +269,7 @@ module Ingester = let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head let isMalformedException (e: #exn) = e.ToString().Contains "SyntaxError: JSON.parse Error: Unexpected input at position" || e.ToString().Contains "SyntaxError: JSON.parse Error: Invalid character at position" @@ -298,10 +299,13 @@ module Ingester = member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream 0L (defaultArg isMalformed false) [|item.span|] member __.HandleWriteResult = function - | Ok (stream, pos) -> updateWritePos stream pos false null; true - | Duplicate (stream, pos) -> updateWritePos stream pos false null; true - | Conflict overage -> updateWritePos overage.stream overage.span.pos false [|overage.span|]; true - | Exn (exn, batch) -> let malformed = isMalformedException exn in __.Add(batch,malformed); not malformed + | Ok (stream, pos) -> updateWritePos stream pos false null; None + | Duplicate (stream, pos) -> updateWritePos stream pos false null; None + | Conflict overage -> updateWritePos overage.stream overage.span.pos false [|overage.span|]; None + | Exn (exn, batch) -> + let malformed = isMalformedException exn + __.Add(batch,malformed) + if malformed then Some (category batch.stream) else None member __.TryPending() = #if NET461 if dirty.Count = 0 then None else @@ -328,16 +332,16 @@ module Ingester = Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } member __.Dump() = let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 - let mutable syncedB, readyB, waitingB, malformedB = 0L, 0L, 0L, 0L + let mutable readyB, waitingB, malformedB = 0L, 0L, 0L for x in states do - let sz = x.Value.Size - if x.Value.isMalformed then malformed <- malformed + 1; malformedB <- malformedB + int64 sz - elif sz = 0 then synced <- synced + 1; syncedB <- syncedB + int64 sz - elif x.Value.IsReady then ready <- ready + 1; readyB <- readyB + int64 sz - else waiting <- waiting + 1; waitingB <- waitingB + int64 sz + match int64 x.Value.Size with + | 0L -> synced <- synced + 1 + | sz when x.Value.isMalformed -> malformed <- malformed + 1; malformedB <- malformedB + sz + | sz when x.Value.IsReady -> ready <- ready + 1; readyB <- readyB + sz + | sz -> waiting <- waiting + 1; waitingB <- waitingB + sz let mb x = x / 1024L / 1024L - Log.Warning("Queued {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}/{syncedMb}MB", - dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced, mb syncedB) + Log.Warning("Queued {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}", + dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen, ?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let states = StreamStates() @@ -351,14 +355,20 @@ module Ingester = let mutable pendingWriterAdd = None let mutable bytesPended = 0L let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let progressTimer, stateTimer = Stopwatch.StartNew(), Stopwatch.StartNew() + let badCats = System.Collections.Generic.Dictionary() + let progressTimer = Stopwatch.StartNew() while not cancellationToken.IsCancellationRequested do let mutable moreResults = true while moreResults do match results.TryDequeue() with | true, res -> incr resultsHandled - if states.HandleWriteResult res then res.WriteTo log + match states.HandleWriteResult res with + | None -> res.WriteTo log + | Some cat -> + match badCats.TryGetValue cat with + | true, catCount -> badCats.[cat] <- catCount + 1 + | false, _ -> badCats.[cat] <- 1 | false, _ -> moreResults <- false let mutable t = Unchecked.defaultof<_> let mutable toIngest = 4096 * 5 @@ -395,7 +405,8 @@ module Ingester = progressTimer.Restart() Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) - ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 + Log.Error("Malformed {badCats}", badCats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd) + ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0; badCats.Clear() states.Dump() /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does @@ -432,15 +443,13 @@ module Reader = open Equinox.EventStore open Equinox.Cosmos -let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head - type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() member __.Ingest(slice: AllEventsSlice) = let mutable batchBytes = 0 for x in slice.Events do - let cat = category x.OriginalStreamId + let cat = Ingester.category x.OriginalStreamId let eventBytes = Ingester.esPayloadBytes x match recentCats.TryGetValue cat with | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) @@ -534,7 +543,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) let rawPos = float currMax * pct / 100. |> int64 // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L ;) - Log.Warning("Start: {pos}",chunkBase) + Log.Warning("Effective Start Position: {pos}", chunkBase) currentPos <- EventStore.ClientAPI.Position(chunkBase,0L) | _ -> () do! run (Option.get max) From 12a8904a39c8fd25432a9d0e4e9a1f6665bbd281 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 15 Mar 2019 10:45:17 +0000 Subject: [PATCH 016/353] Add Waiting stats by category --- equinox-ingest/Ingest/Program.fs | 61 ++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 23 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index cb2e047c1..5fbc01dc9 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -253,8 +253,9 @@ module Ingester = let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length type [] StreamState = { read: int64 option; write: int64 option; isMalformed : bool; queue: Span[] } with - member __.IsHead = match Array.tryHead __.queue with Some x -> x.pos = defaultArg __.write 0L | None -> false - member __.IsReady = __.queue <> null && not __.isMalformed && __.IsHead + /// Determines whether the head is ready to write (either write position is unknown, or matches) + member __.IsHeady = Array.tryHead __.queue |> Option.exists (fun x -> __.write |> Option.forall (fun w -> w = x.pos)) + member __.IsReady = __.queue <> null && not __.isMalformed && __.IsHeady member __.Size = if __.queue = null then 0 else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length*2 + 16) @@ -279,6 +280,17 @@ module Ingester = let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } { read = optionCombine max s1.read s2.read; write = writePos; isMalformed = s1.isMalformed || s2.isMalformed; queue = Span.merge (defaultArg writePos 0L) items} + /// Gathers stats relating to how many items of a given category have been observed + type CatStats() = + let cats = System.Collections.Generic.Dictionary() + member __.Ingest cat = + match cats.TryGetValue cat with + | true, catCount -> cats.[cat] <- catCount + 1 + | false, _ -> cats.[cat] <- 1 + member __.Any = cats.Count <> 0 + member __.Clear() = cats.Clear() + member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd + type StreamStates() = let states = System.Collections.Generic.Dictionary() let dirty = System.Collections.Generic.Queue() @@ -319,29 +331,30 @@ module Ingester = if not state.IsReady then None else - match state.queue |> Array.tryHead with - | None -> None - | Some x -> + let x = state.queue |> Array.head let mutable bytesBudget = cosmosPayloadLimit let mutable count = 0 let max2MbMax1000EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 - count < (if x.pos = 0L then 10 else 1000) && (bytesBudget >= 0 || count = 1) + // Reduce the item count when we don't yet know the write position + count < (if Option.isNone state.write then 10 else 1000) && (bytesBudget >= 0 || count = 1) Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } member __.Dump() = let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 let mutable readyB, waitingB, malformedB = 0L, 0L, 0L - for x in states do - match int64 x.Value.Size with + let waitCats = CatStats() + for KeyValue (stream,state) in states do + match int64 state.Size with | 0L -> synced <- synced + 1 - | sz when x.Value.isMalformed -> malformed <- malformed + 1; malformedB <- malformedB + sz - | sz when x.Value.IsReady -> ready <- ready + 1; readyB <- readyB + sz - | sz -> waiting <- waiting + 1; waitingB <- waitingB + sz + | sz when state.isMalformed -> malformed <- malformed + 1; malformedB <- malformedB + sz + | sz when state.IsReady -> ready <- ready + 1; readyB <- readyB + sz + | sz -> waitCats.Ingest(category stream); waiting <- waiting + 1; waitingB <- waitingB + sz let mb x = x / 1024L / 1024L Log.Warning("Queued {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}", dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) + if waitCats.Any then Log.Warning("Waiting {waitCats}", waitCats.StatsDescending) type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen, ?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let states = StreamStates() @@ -355,7 +368,7 @@ module Ingester = let mutable pendingWriterAdd = None let mutable bytesPended = 0L let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let badCats = System.Collections.Generic.Dictionary() + let badCats = CatStats() let progressTimer = Stopwatch.StartNew() while not cancellationToken.IsCancellationRequested do let mutable moreResults = true @@ -365,10 +378,7 @@ module Ingester = incr resultsHandled match states.HandleWriteResult res with | None -> res.WriteTo log - | Some cat -> - match badCats.TryGetValue cat with - | true, catCount -> badCats.[cat] <- catCount + 1 - | false, _ -> badCats.[cat] <- 1 + | Some cat -> badCats.Ingest cat | false, _ -> moreResults <- false let mutable t = Unchecked.defaultof<_> let mutable toIngest = 4096 * 5 @@ -405,8 +415,8 @@ module Ingester = progressTimer.Restart() Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) - Log.Error("Malformed {badCats}", badCats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd) - ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0; badCats.Clear() + if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() + ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 states.Dump() /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does @@ -458,11 +468,16 @@ type SliceStatsBuffer(?interval) = batchBytes member __.DumpIfIntervalExpired() = if accStart.ElapsedMilliseconds > intervalMs then - Log.Warning("Cats MB/cat/count {@cats}", - recentCats - |> Seq.sortByDescending (fun x -> snd x.Value) - |> Seq.truncate 10 - |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c)) + let log = function + | [||] -> () + | xs -> + xs + |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) + |> Seq.truncate 10 + |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) + |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) + recentCats |> Seq.where (fun x -> x.Key.StartsWith '$' |> not) |> Array.ofSeq |> log + recentCats |> Seq.where (fun x -> x.Key.StartsWith '$') |> Array.ofSeq |> log recentCats.Clear() accStart.Restart() From f75881ae929a1330e74331a96f09b501fbfb3f64 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 15 Mar 2019 22:21:07 +0000 Subject: [PATCH 017/353] Tidy logging --- equinox-ingest/Ingest/Program.fs | 17 +++++++++-------- equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs | 4 ++++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 5fbc01dc9..49e0628ae 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -352,7 +352,7 @@ module Ingester = | sz when state.IsReady -> ready <- ready + 1; readyB <- readyB + sz | sz -> waitCats.Ingest(category stream); waiting <- waiting + 1; waitingB <- waitingB + sz let mb x = x / 1024L / 1024L - Log.Warning("Queued {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}", + Log.Warning("Syncing {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}", dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) if waitCats.Any then Log.Warning("Waiting {waitCats}", waitCats.StatsDescending) type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen, ?interval) = @@ -476,8 +476,8 @@ type SliceStatsBuffer(?interval) = |> Seq.truncate 10 |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) - recentCats |> Seq.where (fun x -> x.Key.StartsWith '$' |> not) |> Array.ofSeq |> log - recentCats |> Seq.where (fun x -> x.Key.StartsWith '$') |> Array.ofSeq |> log + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log recentCats.Clear() accStart.Restart() @@ -503,7 +503,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) let fetchMax = async { let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.NextPosition.CommitPosition - Log.Warning("EventStore Write Position: @ {pos}", max) + Log.Warning("EventStore Write Position @ {pos}", max) return max } let stats = SliceStatsBuffer() let followAll (postBatch : Ingester.Batch -> unit) = async { @@ -538,9 +538,10 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) for pos, item in xs do incr extracted postBatch { stream = s; span = { pos = pos; events = [| item |]}} - Log.Warning("Read {count} {ft:n3}s {mb:n1}MB c {categories,2} s {streams,4} e {events,4} Queue {pt:n0}ms Total {gb:n3}GB @ {pos} {pct:p1}", - received, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., usedCats, streams.Length, !extracted, - postSw.ElapsedMilliseconds, float totalBytes / 1024. / 1024. / 1024., cur, float cur/float max) + Log.Warning("Read {count} {ft:n3}s {mb:n1}MB {gb:n3}GB Process c {categories,2} s {streams,4} e {events,4} {pt:n0}ms Pos @ {pos} {pct:p1}", + received, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., float totalBytes / 1024. / 1024. / 1024., + usedCats, streams.Length, !extracted, postSw.ElapsedMilliseconds, + cur, float cur/float max) if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", totalEvents) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor if not currentSlice.IsEndOfStream then @@ -558,7 +559,7 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) let rawPos = float currMax * pct / 100. |> int64 // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L ;) - Log.Warning("Effective Start Position: {pos}", chunkBase) + Log.Warning("Effective Start Position {pos}", chunkBase) currentPos <- EventStore.ClientAPI.Position(chunkBase,0L) | _ -> () do! run (Option.get max) diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs index ab661da87..5fd996a19 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs @@ -11,6 +11,10 @@ let [] ``nothing`` () = let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] r =! null +let [] ``synced`` () = + let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] + r =! null + let [] ``no overlap`` () = let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] r =! [| mk 0L 1; mk 2L 2 |] From cc84834c33a65918ee2714cd47bb9ae168bda041 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 19 Mar 2019 09:04:08 +0000 Subject: [PATCH 018/353] Extract StartSpec --- equinox-ingest/Ingest/Program.fs | 144 ++++++++++++++++--------------- 1 file changed, 74 insertions(+), 70 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 49e0628ae..000bb7f3f 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -5,6 +5,9 @@ open FSharp.Control open Serilog open System +type StartPos = Absolute of int64 | Percentage of float | Start +type ReaderSpec = { start: StartPos; batchSize: int; streams: string list } + module CmdParser = open Argu type LogEventLevel = Serilog.Events.LogEventLevel @@ -119,8 +122,8 @@ module CmdParser = | [] VerboseConsole | [] LocalSeq | [] Stream of string - | [] AllPos of int64 - | [] PercentagePos of float + | [] Offset of int64 + | [] Percent of float | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = @@ -128,10 +131,10 @@ module CmdParser = | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" - | LocalSeq -> "Configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | Stream _ -> "specify stream(s) to seed the processing with" - | AllPos _ -> "Specify EventStore $all Stream Position to commence from" - | PercentagePos _ -> "Specify EventStore $all Stream Position to commence from (as a percentage of current tail position)" + | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" + | Stream _ -> "specific stream(s) to read" + | Offset _ -> "EventStore $all Stream Position to commence from" + | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) @@ -139,14 +142,14 @@ module CmdParser = member __.ConsoleMinLevel = if args.Contains VerboseConsole then LogEventLevel.Information else LogEventLevel.Warning member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None member __.BatchSize = args.GetResult(BatchSize,4096) - member x.BuildFeedParams() = + member x.BuildFeedParams() : ReaderSpec = Log.Information("Processing in batches of {batchSize}", x.BatchSize) let startPos = - match args.TryGetResult AllPos, args.TryGetResult PercentagePos with - | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Choice1Of3 p - | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Choice2Of3 p - | None, None -> Log.Warning "Processing will commence at $all Start"; Choice3Of3 () - x.BatchSize, args.GetResults Stream, startPos + match args.TryGetResult Offset, args.TryGetResult Percent with + | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | None, None -> Log.Warning "Processing will commence at $all Start"; Start + { start = startPos; batchSize = x.BatchSize; streams = args.GetResults Stream } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args let parse argv : Parameters = @@ -355,6 +358,7 @@ module Ingester = Log.Warning("Syncing {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}", dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) if waitCats.Any then Log.Warning("Waiting {waitCats}", waitCats.StatsDescending) + type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen, ?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let states = StreamStates() @@ -434,7 +438,7 @@ open EventStore.ClientAPI module Reader = - let loadSpecificStreamsTemp (conn:IEventStoreConnection, batchSize) streams (postBatch : (Ingester.Batch -> unit)) = + let loadSpecificStreams (conn:IEventStoreConnection, batchSize) streams (postBatch : (Ingester.Batch -> unit)) = let fetchStream stream = let rec fetchFrom pos = async { let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect @@ -453,6 +457,23 @@ module Reader = open Equinox.EventStore open Equinox.Cosmos +let enumEvents (slice : AllEventsSlice) = seq { + for e in slice.Events -> + let eb = Ingester.esPayloadBytes e + match e.Event with + | e when not e.IsJson + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.StartsWith("$") + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> + Choice2Of2 e + | e when eb > Ingester.cosmosPayloadLimit -> + Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) + Choice2Of2 e + | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) +} + type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() @@ -465,7 +486,7 @@ type SliceStatsBuffer(?interval) = | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) | false, _ -> recentCats.[cat] <- (1, eventBytes) batchBytes <- batchBytes + eventBytes - batchBytes + slice.Events.Length, batchBytes member __.DumpIfIntervalExpired() = if accStart.ElapsedMilliseconds > intervalMs then let log = function @@ -481,69 +502,55 @@ type SliceStatsBuffer(?interval) = recentCats.Clear() accStart.Restart() -let run (destination : CosmosConnection, colls) (source : GesConnection) - (batchSize, streams: string list, startPos: Choice) - (writerQueueLen, writerCount, readerQueueLen) = async { - let enumEvents (slice : AllEventsSlice) = seq { - for e in slice.Events -> - let eb = Ingester.esPayloadBytes e - match e.Event with - | e when not e.IsJson - || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.StartsWith("$") - || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") - || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> - Choice2Of2 e - | e when eb > Ingester.cosmosPayloadLimit -> - Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) - Choice2Of2 e - | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) - } +type OverallStats() = + let mutable totalEvents, totalBytes = 0L, 0L + member __.Ingest(batchEvents, batchBytes) = + totalEvents <- totalEvents + int64 batchEvents + totalBytes <- totalBytes + int64 batchBytes + member __.Bytes = totalBytes + member __.Events = totalEvents + +let run (destination : CosmosConnection, colls) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { let fetchMax = async { let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.NextPosition.CommitPosition Log.Warning("EventStore Write Position @ {pos}", max) return max } - let stats = SliceStatsBuffer() let followAll (postBatch : Ingester.Batch -> unit) = async { let mutable currentPos = - match startPos with - | Choice1Of3 p -> EventStore.ClientAPI.Position(p,0L) - | Choice2Of3 _ -> Position.End // placeholder, will be overwritten below - | Choice3Of3 () -> Position.Start - let mutable totalEvents, totalBytes = 0L, 0L + match spec.start with + | Absolute p -> EventStore.ClientAPI.Position(p,0L) + | Percentage _ -> Position.End // placeholder, will be overwritten below + | Start -> Position.Start + let overall, slicesStats = OverallStats(), SliceStatsBuffer() let run max = async { let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { - let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(currentPos, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let cur = currentSlice.NextPosition.CommitPosition + let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(currentPos, spec.batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let received = currentSlice.Events.Length - totalEvents <- totalEvents + int64 received - let batchBytes = stats.Ingest currentSlice - totalBytes <- totalBytes + int64 batchBytes + let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overall.Ingest(batchEvents, batchBytes) + slicesStats.DumpIfIntervalExpired() let streams = enumEvents currentSlice |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) |> Array.ofSeq - let usedCats = streams |> Seq.map fst |> Seq.distinct |> Seq.length - stats.DumpIfIntervalExpired() + let postSw = Stopwatch.StartNew() - let extracted = ref 0 - for s,xs in streams do - for pos, item in xs do - incr extracted - postBatch { stream = s; span = { pos = pos; events = [| item |]}} + let usedEvents = ref 0 + for stream,streamEvents in streams do + for pos, item in streamEvents do + incr usedEvents + postBatch { stream = stream; span = { pos = pos; events = [| item |]}} + let currentOffset = currentSlice.NextPosition.CommitPosition Log.Warning("Read {count} {ft:n3}s {mb:n1}MB {gb:n3}GB Process c {categories,2} s {streams,4} e {events,4} {pt:n0}ms Pos @ {pos} {pct:p1}", - received, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., float totalBytes / 1024. / 1024. / 1024., - usedCats, streams.Length, !extracted, postSw.ElapsedMilliseconds, - cur, float cur/float max) - if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", totalEvents) - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + batchEvents, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., float overall.Bytes / 1024. / 1024. / 1024., + usedCats, streams.Length, !usedEvents, postSw.ElapsedMilliseconds, + currentOffset, float currentOffset/float max) + if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", overall.Events) + sw.Restart() // restart the clock as we hand off back to the Reader if not currentSlice.IsEndOfStream then currentPos <- currentSlice.NextPosition return! loop () } @@ -554,22 +561,20 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) try if max = None then let! currMax = fetchMax max <- Some currMax - match startPos with - | Choice2Of3 pct -> + match spec.start with + | Percentage pct -> let rawPos = float currMax * pct / 100. |> int64 // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L ;) - Log.Warning("Effective Start Position {pos}", chunkBase) + Log.Warning("Effective Start Position {pos} (chunk {chunk})", chunkBase, chunkBase >>> 29) currentPos <- EventStore.ClientAPI.Position(chunkBase,0L) | _ -> () do! run (Option.get max) finished <- true - with e -> Log.Warning(e,"Ingestion error") - } - + with e -> Log.Warning(e,"Ingestion error") } let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) - let! _ = Async.StartChild (Reader.loadSpecificStreamsTemp (source.ReadConnection, batchSize) streams ingester.Add) + let! _ = Async.StartChild (Reader.loadSpecificStreams (source.ReadConnection, spec.batchSize) spec.streams ingester.Add) let! _ = Async.StartChild (followAll ingester.Add) do! Async.AwaitKeyboardInterrupt() } @@ -577,15 +582,14 @@ let run (destination : CosmosConnection, colls) (source : GesConnection) let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ConsoleMinLevel args.MaybeSeqEndpoint - let connectionMode = ConnectionStrategy.ClusterSingle NodePreference.Master - let source = args.EventStore.Connect(Log.Logger, Log.Logger, connectionMode) |> Async.RunSynchronously + let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) |> Async.RunSynchronously + let readerSpec = args.BuildFeedParams() + let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 + if Threading.ThreadPool.SetMaxThreads(512,512) |> not then failwith "Could not set max threads" let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) - let batchSize, streams, startPos = args.BuildFeedParams() - let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 - if Threading.ThreadPool.SetMaxThreads(512,512) |> not then failwith "Could not set max threads" - run (destination, colls) source (batchSize, streams, startPos) (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously + run (destination, colls) source readerSpec (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 From 20c5447581f9916ea3f93773823a95856358c44c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 20 Mar 2019 22:58:11 +0000 Subject: [PATCH 019/353] Add Striped reading --- equinox-ingest/Ingest/Infrastructure.fs | 21 ++ equinox-ingest/Ingest/Program.fs | 350 +++++++++++++++--------- 2 files changed, 247 insertions(+), 124 deletions(-) diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs index d8c3fa2b3..588ae2ab8 100644 --- a/equinox-ingest/Ingest/Infrastructure.fs +++ b/equinox-ingest/Ingest/Infrastructure.fs @@ -1,6 +1,7 @@ [] module private SyncTemplate.Infrastructure +open Equinox.Store // AwaitTaskCorrect open System open System.Threading open System.Threading.Tasks @@ -17,3 +18,23 @@ type Async with let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore and d : IDisposable = Console.CancelKeyPress.Subscribe callback in ()) + +type SemaphoreSlim with + /// F# friendly semaphore await function + member semaphore.Await(?timeout : TimeSpan) = async { + let! ct = Async.CancellationToken + let timeout = defaultArg timeout Timeout.InfiniteTimeSpan + let task = semaphore.WaitAsync(timeout, ct) + return! Async.AwaitTaskCorrect task + } + +module Queue = + let tryDequeue (x : System.Collections.Generic.Queue<'T>) = +#if NET461 + if x.Count = 0 then None + else x.Dequeue() |> Some +#else + match x.TryDequeue() with + | false, _ -> None + | true, res -> Some res +#endif diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 000bb7f3f..c3a31d7ea 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -6,7 +6,8 @@ open Serilog open System type StartPos = Absolute of int64 | Percentage of float | Start -type ReaderSpec = { start: StartPos; batchSize: int; streams: string list } +type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; streams: string list } +let mb x = float x / 1024. / 1024. module CmdParser = open Argu @@ -100,12 +101,10 @@ module CmdParser = concurrentOperationsLimit=col, log=log, tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string]) .Establish("ProjectorTemplate", discovery, connection) member val Cosmos = Cosmos.Info(args.GetResult Cosmos) - member __.CreateGateway conn = GesGateway(conn, GesBatchingPolicy(maxBatchSize = args.GetResult(MaxItems,4096))) member __.Host = match args.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" member __.Port = match args.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int member __.User = match args.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" member __.Password = match args.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" - member val CacheStrategy = let c = Caching.Cache("ProjectorTemplate", sizeMb = 50) in CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) member __.Connect(log: ILogger, storeLog, connection) = let (timeout, retries) as operationThrottling = args.GetResult(Timeout,20.) |> TimeSpan.FromSeconds, args.GetResult(Retries,3) let heartbeatTimeout = args.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds @@ -124,6 +123,8 @@ module CmdParser = | [] Stream of string | [] Offset of int64 | [] Percent of float + | [] Stripes of int + | [] Tail of intervalMs: int | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = @@ -135,6 +136,8 @@ module CmdParser = | Stream _ -> "specific stream(s) to read" | Offset _ -> "EventStore $all Stream Position to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" + | Stripes _ -> "number of concurrent readers" + | Tail _ -> "attempt to read from tail at specified interval in milliseconds" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) @@ -142,14 +145,20 @@ module CmdParser = member __.ConsoleMinLevel = if args.Contains VerboseConsole then LogEventLevel.Information else LogEventLevel.Warning member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None member __.BatchSize = args.GetResult(BatchSize,4096) + member __.Stripes = args.GetResult(Stripes,1) + member __.TailInterval = match args.TryGetResult Tail with Some s -> s |> float |> TimeSpan.FromMilliseconds |> Some | None -> None member x.BuildFeedParams() : ReaderSpec = - Log.Information("Processing in batches of {batchSize}", x.BatchSize) + Log.Warning("Processing in batches of {batchSize}", x.BatchSize) + Log.Warning("Reading with {stripes} stripes", x.Stripes) + match x.TailInterval with + | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) + | None -> Log.Warning "Not following tail" let startPos = match args.TryGetResult Offset, args.TryGetResult Percent with | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p | None, None -> Log.Warning "Processing will commence at $all Start"; Start - { start = startPos; batchSize = x.BatchSize; streams = args.GetResults Stream } + { start = startPos; stripes = x.Stripes; batchSize = x.BatchSize; streams = args.GetResults Stream } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args let parse argv : Parameters = @@ -166,7 +175,8 @@ module Logging = .Destructure.FSharpTypes() .Enrich.FromLogContext() |> fun c -> if verbose then c.MinimumLevel.Debug() else c - |> fun c -> c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code) + |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {Properties} {NewLine}{Exception}" + c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() @@ -322,14 +332,9 @@ module Ingester = __.Add(batch,malformed) if malformed then Some (category batch.stream) else None member __.TryPending() = -#if NET461 - if dirty.Count = 0 then None else - let stream = dirty.Dequeue() -#else - match dirty.TryDequeue() with - | false, _ -> None - | true, stream -> -#endif + match dirty |> Queue.tryDequeue with + | None -> None + | Some stream -> let state = states.[stream] if not state.IsReady then None else @@ -354,8 +359,7 @@ module Ingester = | sz when state.isMalformed -> malformed <- malformed + 1; malformedB <- malformedB + sz | sz when state.IsReady -> ready <- ready + 1; readyB <- readyB + sz | sz -> waitCats.Ingest(category stream); waiting <- waiting + 1; waitingB <- waitingB + sz - let mb x = x / 1024L / 1024L - Log.Warning("Syncing {dirty} Ready {ready}/{readyMb}MB Waiting {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb}MB Synced {synced}", + Log.Warning("Syncing {dirty} Ready {ready}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) if waitCats.Any then Log.Warning("Waiting {waitCats}", waitCats.StatsDescending) @@ -418,7 +422,7 @@ module Ingester = if progressTimer.ElapsedMilliseconds > intervalMs then progressTimer.Restart() Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", - !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, float bytesPended / 1024. / 1024. / 1024.) + !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 states.Dump() @@ -434,50 +438,10 @@ module Ingester = return queue } -open EventStore.ClientAPI - -module Reader = - - let loadSpecificStreams (conn:IEventStoreConnection, batchSize) streams (postBatch : (Ingester.Batch -> unit)) = - let fetchStream stream = - let rec fetchFrom pos = async { - let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect - if currentSlice.IsEndOfStream then return () else - let events = - [| for x in currentSlice.Events -> - let e = x.Event - Equinox.Codec.Core.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Codec.IEvent |] - postBatch { stream = stream; span = { pos = currentSlice.FromEventNumber; events = events } } - return! fetchFrom currentSlice.NextEventNumber } - fetchFrom 0L - async { - for stream in streams do - do! fetchStream stream } - -open Equinox.EventStore -open Equinox.Cosmos - -let enumEvents (slice : AllEventsSlice) = seq { - for e in slice.Events -> - let eb = Ingester.esPayloadBytes e - match e.Event with - | e when not e.IsJson - || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.StartsWith("$") - || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") - || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> - Choice2Of2 e - | e when eb > Ingester.cosmosPayloadLimit -> - Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) - Choice2Of2 e - | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) -} - type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() - member __.Ingest(slice: AllEventsSlice) = + member __.Ingest(slice: EventStore.ClientAPI.AllEventsSlice) = let mutable batchBytes = 0 for x in slice.Events do let cat = Ingester.category x.OriginalStreamId @@ -486,7 +450,7 @@ type SliceStatsBuffer(?interval) = | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) | false, _ -> recentCats.[cat] <- (1, eventBytes) batchBytes <- batchBytes + eventBytes - slice.Events.Length, batchBytes + slice.Events.Length, int64 batchBytes member __.DumpIfIntervalExpired() = if accStart.ElapsedMilliseconds > intervalMs then let log = function @@ -505,77 +469,215 @@ type SliceStatsBuffer(?interval) = type OverallStats() = let mutable totalEvents, totalBytes = 0L, 0L member __.Ingest(batchEvents, batchBytes) = - totalEvents <- totalEvents + int64 batchEvents - totalBytes <- totalBytes + int64 batchBytes + totalEvents <- totalEvents + batchEvents + totalBytes <- totalBytes + batchBytes member __.Bytes = totalBytes member __.Events = totalEvents -let run (destination : CosmosConnection, colls) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { - let fetchMax = async { - let! lastItemBatch = source.ReadConnection.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let max = lastItemBatch.NextPosition.CommitPosition - Log.Warning("EventStore Write Position @ {pos}", max) +type Range(start, sliceEnd : EventStore.ClientAPI.Position option, max : EventStore.ClientAPI.Position) = + member val Current = start with get, set + member __.TryNext(pos: EventStore.ClientAPI.Position) = + __.Current <- pos + __.IsCompleted + member __.IsCompleted = + match sliceEnd with + | Some send when __.Current.CommitPosition >= send.CommitPosition -> false + | _ -> true + member __.PositionAsRangePercentage = float __.Current.CommitPosition/float max.CommitPosition + +module EventStoreReader = + open EventStore.ClientAPI + + let posFromPercentage (pct,max : Position) = + let rawPos = float max.CommitPosition * pct / 100. |> int64 + // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk + let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L + Position(chunkBase,0L) + let posFromChunkAfter (pos: Position) = + let chunkBase = pos.CommitPosition &&& 0xFFFFFFFFF0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L + let nextBase = chunkBase + 256L * 1024L * 1024L + Position(nextBase,0L) + let chunk (pos: Position) = + uint64 pos.CommitPosition >>> 28 + + let fetchMax (conn : IEventStoreConnection) = async { + let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect + let max = lastItemBatch.NextPosition + Log.Warning("EventStore {chunks} chunks Write Position @ {pos} ", chunk max, max.CommitPosition) return max } - let followAll (postBatch : Ingester.Batch -> unit) = async { - let mutable currentPos = - match spec.start with - | Absolute p -> EventStore.ClientAPI.Position(p,0L) - | Percentage _ -> Position.End // placeholder, will be overwritten below - | Start -> Position.Start - let overall, slicesStats = OverallStats(), SliceStatsBuffer() - let run max = async { - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let rec loop () = async { - let! currentSlice = source.ReadConnection.ReadAllEventsForwardAsync(currentPos, spec.batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overall.Ingest(batchEvents, batchBytes) - slicesStats.DumpIfIntervalExpired() - let streams = - enumEvents currentSlice - |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) - |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) - |> Array.ofSeq - let usedCats = streams |> Seq.map fst |> Seq.distinct |> Seq.length - - let postSw = Stopwatch.StartNew() - let usedEvents = ref 0 - for stream,streamEvents in streams do - for pos, item in streamEvents do - incr usedEvents - postBatch { stream = stream; span = { pos = pos; events = [| item |]}} - let currentOffset = currentSlice.NextPosition.CommitPosition - Log.Warning("Read {count} {ft:n3}s {mb:n1}MB {gb:n3}GB Process c {categories,2} s {streams,4} e {events,4} {pt:n0}ms Pos @ {pos} {pct:p1}", - batchEvents, (let e = sw.Elapsed in e.TotalSeconds), float batchBytes / 1024. / 1024., float overall.Bytes / 1024. / 1024. / 1024., - usedCats, streams.Length, !usedEvents, postSw.ElapsedMilliseconds, - currentOffset, float currentOffset/float max) - if currentSlice.IsEndOfStream then Log.Warning("Completed {total:n0}", overall.Events) - sw.Restart() // restart the clock as we hand off back to the Reader - if not currentSlice.IsEndOfStream then - currentPos <- currentSlice.NextPosition - return! loop () } - do! loop () } - let mutable finished = false + let establishMax (conn : IEventStoreConnection) = async { let mutable max = None - while not finished do - try if max = None then - let! currMax = fetchMax - max <- Some currMax - match spec.start with - | Percentage pct -> - let rawPos = float currMax * pct / 100. |> int64 - // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk - let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L ;) - Log.Warning("Effective Start Position {pos} (chunk {chunk})", chunkBase, chunkBase >>> 29) - currentPos <- EventStore.ClientAPI.Position(chunkBase,0L) - | _ -> () - do! run (Option.get max) - finished <- true - with e -> Log.Warning(e,"Ingestion error") } + while Option.isNone max do + try let! max_ = fetchMax conn + max <- Some max_ + with e -> + Log.Warning(e,"Could not establish max position") + do! Async.Sleep 5000 + return Option.get max } + let pullStream (conn : IEventStoreConnection, batchSize) stream (postBatch : Ingester.Batch -> unit) = + let rec fetchFrom pos = async { + let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect + if currentSlice.IsEndOfStream then return () else + let events = + [| for x in currentSlice.Events -> + let e = x.Event + Equinox.Codec.Core.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Codec.IEvent |] + postBatch { stream = stream; span = { pos = currentSlice.FromEventNumber; events = events } } + return! fetchFrom currentSlice.NextEventNumber } + fetchFrom 0L + + type [] PullResult = Exn of exn: exn | Eof | EndOfTranche + let pullSourceRange (conn : IEventStoreConnection, batchSize) (range : Range) enumEvents (postBatch : Ingester.Batch -> unit) = + let stats, slicesStats = OverallStats(), SliceStatsBuffer() + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let rec loop () = async { + let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let postSw = Stopwatch.StartNew() + let batchEvents, batchBytes = slicesStats.Ingest currentSlice in stats.Ingest(int64 batchEvents, batchBytes) + slicesStats.DumpIfIntervalExpired() + let streams = + enumEvents currentSlice.Events + |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) + |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) + |> Array.ofSeq + let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length + let mutable usedEvents = 0 + for stream,streamEvents in streams do + for pos, item in streamEvents do + usedEvents <- usedEvents + 1 + postBatch { stream = stream; span = { pos = pos; events = [| item |]}} + let shouldLoop = range.TryNext currentSlice.NextPosition + Log.Warning("Read {count} {ft:n3}s {mb:n1}MB Process c {categories,2} s {streams,4} e {events,4} {pt:n0}ms Pos @ {pos} {pct:p1}", + batchEvents, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, + usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds, + currentSlice.NextPosition.CommitPosition, range.PositionAsRangePercentage) + if shouldLoop && not currentSlice.IsEndOfStream then + sw.Restart() // restart the clock as we hand off back to the Reader + return! loop () + else + return currentSlice.IsEndOfStream } + async { + try let! eof = loop () + return (if eof then Eof else EndOfTranche), range, stats + with e -> return Exn e, range, stats } + + type [] Work = + | Stream of name: string * batchSize: int + | Tranche of range: Range * batchSize : int + type FeedQueue(batchSize, max) = + let work = ConcurrentQueue() + let add item = + work.Enqueue item + Log.Warning("Added {item}; count: {count}", item, work.Count) + member val OverallStats = OverallStats() with get + member __.AddTranche(range, ?batchSizeOverride) = + add <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) + member __.AddTranche(pos, nextPos, ?batchSizeOverride) = + __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) + member __.AddStream(name, ?batchSizeOverride) = + add <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + member __.TryDequeue () = + work.TryDequeue() + member __.Process(conn, enumEvents, postBatch, work) = async { + let adjust batchSize = if batchSize > 128 then batchSize / 2 else batchSize + match work with + | Stream (name,batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Stream",name) + Log.Warning("Reading stream; batch size {bs}", batchSize) + try do! pullStream (conn, batchSize) name postBatch + with e -> + let bs = adjust batchSize + Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + __.AddStream(name, bs) + | Tranche (range, batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) + Log.Warning("Reading chunk; batch size {bs}", batchSize) + let! eofOption, range, stats = pullSourceRange (conn, batchSize) range enumEvents postBatch + lock __.OverallStats <| fun () -> __.OverallStats.Ingest(stats.Events, stats.Bytes) + match eofOption with + | PullResult.EndOfTranche -> Log.Warning("Completed tranche") + | PullResult.Eof -> Log.Warning("REACHED THE END!") + | PullResult.Exn e -> + let bs = adjust batchSize + Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) + __.AddTranche(range, bs) + } + + type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken) = + let work = FeedQueue(spec.batchSize, max) + do for s in spec.streams do work.AddStream(s) + let mutable remainder = + let startPos = + match spec.start with + | StartPos.Start -> Position.Start + | Absolute p -> Position(p, 0L) + | Percentage pct -> + let startPos = posFromPercentage (pct, max) + Log.Warning("Effective Start Position {tranche} (chunk {chunk})", startPos.CommitPosition, chunk startPos) + startPos + let nextPos = posFromChunkAfter startPos + work.AddTranche(startPos, nextPos) + Some nextPos + + member __.Pump () = async { + (*if spec.tail then enqueue tail work*) + let maxDop = spec.stripes + let dop = new SemaphoreSlim(maxDop) + let mutable finished = false + while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + do! work.Process(conn, enumEvents, postBatch, task) + dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + match remainder with + | None -> + finished <- true + Log.Warning("Processing completed") + | Some pos -> + let nextPos = posFromChunkAfter pos + remainder <- Some nextPos + do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) } + + let start (conn, spec, enumEvents, postBatch) = async { + let! ct = Async.CancellationToken + let! max = establishMax conn + let reader = Reader(conn, spec, enumEvents, postBatch, max, ct) + let! _ = Async.StartChild <| reader.Pump() + return () + } + +open Equinox.Cosmos +open Equinox.EventStore + +let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { + for e in xs -> + let eb = Ingester.esPayloadBytes e + match e.Event with + | e when not e.IsJson + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.StartsWith("$") + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> + Choice2Of2 e + | e when eb > Ingester.cosmosPayloadLimit -> + Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) + Choice2Of2 e + | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) +} + +let run (destination : CosmosConnection, colls) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) - let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) - let! _ = Async.StartChild (Reader.loadSpecificStreams (source.ReadConnection, spec.batchSize) spec.streams ingester.Add) - let! _ = Async.StartChild (followAll ingester.Add) + let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) + let! _feeder = EventStoreReader.start(source.ReadConnection, spec, enumEvents, ingester.Add) do! Async.AwaitKeyboardInterrupt() } [] From 452d58d835691f55f5e5982c91ec20f50f1d6425 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 21 Mar 2019 01:26:01 +0000 Subject: [PATCH 020/353] Add overall ingestion stats --- equinox-ingest/Ingest/Program.fs | 71 ++++++++++++++++++-------------- 1 file changed, 40 insertions(+), 31 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index c3a31d7ea..34b9c288a 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -5,7 +5,7 @@ open FSharp.Control open Serilog open System -type StartPos = Absolute of int64 | Percentage of float | Start +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; streams: string list } let mb x = float x / 1024. / 1024. @@ -122,6 +122,7 @@ module CmdParser = | [] LocalSeq | [] Stream of string | [] Offset of int64 + | [] Chunk of int | [] Percent of float | [] Stripes of int | [] Tail of intervalMs: int @@ -135,6 +136,7 @@ module CmdParser = | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Stream _ -> "specific stream(s) to read" | Offset _ -> "EventStore $all Stream Position to commence from" + | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Stripes _ -> "number of concurrent readers" | Tail _ -> "attempt to read from tail at specified interval in milliseconds" @@ -154,10 +156,11 @@ module CmdParser = | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) | None -> Log.Warning "Not following tail" let startPos = - match args.TryGetResult Offset, args.TryGetResult Percent with - | Some p, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p - | None, None -> Log.Warning "Processing will commence at $all Start"; Start + match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent with + | Some p, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some c, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c + | _, _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | None, None, None -> Log.Warning "Processing will commence at $all Start"; Start { start = startPos; stripes = x.Stripes; batchSize = x.BatchSize; streams = args.GetResults Stream } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args @@ -439,7 +442,7 @@ module Ingester = } type SliceStatsBuffer(?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() member __.Ingest(slice: EventStore.ClientAPI.AllEventsSlice) = let mutable batchBytes = 0 @@ -451,8 +454,8 @@ type SliceStatsBuffer(?interval) = | false, _ -> recentCats.[cat] <- (1, eventBytes) batchBytes <- batchBytes + eventBytes slice.Events.Length, int64 batchBytes - member __.DumpIfIntervalExpired() = - if accStart.ElapsedMilliseconds > intervalMs then + member __.DumpIfIntervalExpired(?force) = + if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then let log = function | [||] -> () | xs -> @@ -466,13 +469,20 @@ type SliceStatsBuffer(?interval) = recentCats.Clear() accStart.Restart() -type OverallStats() = +type OverallStats(?statsInterval) = + let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() let mutable totalEvents, totalBytes = 0L, 0L member __.Ingest(batchEvents, batchBytes) = totalEvents <- totalEvents + batchEvents totalBytes <- totalBytes + batchBytes member __.Bytes = totalBytes member __.Events = totalEvents + member __.DumpIfIntervalExpired() = + if progressStart.ElapsedMilliseconds > intervalMs then + let totalMb = mb totalBytes + Log.Warning("Traversed {events} events {gb:n1}GB {mbs}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + progressStart.Restart() type Range(start, sliceEnd : EventStore.ClientAPI.Position option, max : EventStore.ClientAPI.Position) = member val Current = start with get, set @@ -488,17 +498,17 @@ type Range(start, sliceEnd : EventStore.ClientAPI.Position option, max : EventSt module EventStoreReader = open EventStore.ClientAPI - let posFromPercentage (pct,max : Position) = - let rawPos = float max.CommitPosition * pct / 100. |> int64 - // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk - let chunkBase = rawPos &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L + // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk + let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 + let posFromChunk (chunk: int) = + let chunkBase = int64 chunk * 1024L * 1024L * 256L Position(chunkBase,0L) + let posFromPercentage (pct,max : Position) = + let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) + let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L let posFromChunkAfter (pos: Position) = - let chunkBase = pos.CommitPosition &&& 0xFFFFFFFFF0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L - let nextBase = chunkBase + 256L * 1024L * 1024L - Position(nextBase,0L) - let chunk (pos: Position) = - uint64 pos.CommitPosition >>> 28 + let nextChunk = 1 + int (chunk pos) + posFromChunk nextChunk let fetchMax (conn : IEventStoreConnection) = async { let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -557,6 +567,7 @@ module EventStoreReader = sw.Restart() // restart the clock as we hand off back to the Reader return! loop () else + slicesStats.DumpIfIntervalExpired(force=true) return currentSlice.IsEndOfStream } async { try let! eof = loop () @@ -566,18 +577,15 @@ module EventStoreReader = type [] Work = | Stream of name: string * batchSize: int | Tranche of range: Range * batchSize : int - type FeedQueue(batchSize, max) = + type FeedQueue(batchSize, max, ?statsInterval) = let work = ConcurrentQueue() - let add item = - work.Enqueue item - Log.Warning("Added {item}; count: {count}", item, work.Count) - member val OverallStats = OverallStats() with get + member val OverallStats = OverallStats(?statsInterval=statsInterval) with get member __.AddTranche(range, ?batchSizeOverride) = - add <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) + work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) member __.AddTranche(pos, nextPos, ?batchSizeOverride) = __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) member __.AddStream(name, ?batchSizeOverride) = - add <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() member __.Process(conn, enumEvents, postBatch, work) = async { @@ -605,18 +613,18 @@ module EventStoreReader = __.AddTranche(range, bs) } - type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken) = - let work = FeedQueue(spec.batchSize, max) + type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = + let work = FeedQueue(spec.batchSize, max, ?statsInterval=statsInterval) do for s in spec.streams do work.AddStream(s) let mutable remainder = let startPos = match spec.start with | StartPos.Start -> Position.Start | Absolute p -> Position(p, 0L) - | Percentage pct -> - let startPos = posFromPercentage (pct, max) - Log.Warning("Effective Start Position {tranche} (chunk {chunk})", startPos.CommitPosition, chunk startPos) - startPos + | Chunk c -> posFromChunk c + | Percentage pct -> posFromPercentage (pct, max) + Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", + startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) let nextPos = posFromChunkAfter startPos work.AddTranche(startPos, nextPos) Some nextPos @@ -628,6 +636,7 @@ module EventStoreReader = let mutable finished = false while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do let! _ = dop.Await() + work.OverallStats.DumpIfIntervalExpired() let forkRunRelease task = async { let! _ = Async.StartChild <| async { do! work.Process(conn, enumEvents, postBatch, task) From c2b25b838b5558d987c4e6e9b3147b7e6d379e23 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 21 Mar 2019 01:38:23 +0000 Subject: [PATCH 021/353] Plumb completion --- equinox-ingest/Ingest/Program.fs | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 34b9c288a..e590e0b27 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -599,19 +599,24 @@ module EventStoreReader = let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) __.AddStream(name, bs) + return false | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Reading chunk; batch size {bs}", batchSize) let! eofOption, range, stats = pullSourceRange (conn, batchSize) range enumEvents postBatch lock __.OverallStats <| fun () -> __.OverallStats.Ingest(stats.Events, stats.Bytes) match eofOption with - | PullResult.EndOfTranche -> Log.Warning("Completed tranche") - | PullResult.Eof -> Log.Warning("REACHED THE END!") + | PullResult.EndOfTranche -> + Log.Warning("Completed tranche") + return false + | PullResult.Eof -> + Log.Warning("REACHED THE END!") + return true | PullResult.Exn e -> let bs = adjust batchSize Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) __.AddTranche(range, bs) - } + return false } type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = let work = FeedQueue(spec.batchSize, max, ?statsInterval=statsInterval) @@ -639,7 +644,8 @@ module EventStoreReader = work.OverallStats.DumpIfIntervalExpired() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - do! work.Process(conn, enumEvents, postBatch, task) + let! eof = work.Process(conn, enumEvents, postBatch, task) + if eof then remainder <- None dop.Release() |> ignore } return () } match work.TryDequeue() with @@ -647,13 +653,13 @@ module EventStoreReader = do! forkRunRelease task | false, _ -> match remainder with - | None -> - finished <- true - Log.Warning("Processing completed") | Some pos -> let nextPos = posFromChunkAfter pos remainder <- Some nextPos - do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) } + do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) + | None -> + finished <- true + Log.Warning("No further work to commence") } let start (conn, spec, enumEvents, postBatch) = async { let! ct = Async.CancellationToken From f19f8697bbc6878a0a6689dfd7ec964b12450919 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 21 Mar 2019 15:32:28 +0000 Subject: [PATCH 022/353] Add Tailing; tidy logging --- equinox-ingest/Ingest/Program.fs | 290 ++++++++++++++++++------------- 1 file changed, 170 insertions(+), 120 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index e590e0b27..8ff0b1765 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -5,8 +5,8 @@ open FSharp.Control open Serilog open System -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start -type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; streams: string list } +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start | Ignore +type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; streams: string list; tailInterval: TimeSpan option } let mb x = float x / 1024. / 1024. module CmdParser = @@ -121,11 +121,12 @@ module CmdParser = | [] VerboseConsole | [] LocalSeq | [] Stream of string + | [] All | [] Offset of int64 | [] Chunk of int | [] Percent of float | [] Stripes of int - | [] Tail of intervalMs: int + | [] Tail of intervalS: float | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = @@ -135,11 +136,12 @@ module CmdParser = | VerboseConsole -> "request Verbose Console Logging. Default: off" | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Stream _ -> "specific stream(s) to read" + | All -> "traverse EventStore $all from Start" | Offset _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Stripes _ -> "number of concurrent readers" - | Tail _ -> "attempt to read from tail at specified interval in milliseconds" + | Tail _ -> "attempt to read from tail at specified interval in Seconds" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = member val EventStore = EventStore.Info(args.GetResult Es) @@ -148,20 +150,21 @@ module CmdParser = member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None member __.BatchSize = args.GetResult(BatchSize,4096) member __.Stripes = args.GetResult(Stripes,1) - member __.TailInterval = match args.TryGetResult Tail with Some s -> s |> float |> TimeSpan.FromMilliseconds |> Some | None -> None + member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None member x.BuildFeedParams() : ReaderSpec = Log.Warning("Processing in batches of {batchSize}", x.BatchSize) Log.Warning("Reading with {stripes} stripes", x.Stripes) + let startPos = + match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent, args.Contains All with + | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some c, _, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c + | _, _, Some p, _ -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | None, None, None, true -> Log.Warning "Processing will commence at $all Start"; Start + | None, None, None, false -> Log.Warning "No $all processing requested"; Ignore match x.TailInterval with | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) | None -> Log.Warning "Not following tail" - let startPos = - match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent with - | Some p, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some c, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c - | _, _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p - | None, None, None -> Log.Warning "Processing will commence at $all Start"; Start - { start = startPos; stripes = x.Stripes; batchSize = x.BatchSize; streams = args.GetResults Stream } + { start = startPos; stripes = x.Stripes; batchSize = x.BatchSize; streams = args.GetResults Stream; tailInterval = x.TailInterval } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args let parse argv : Parameters = @@ -178,7 +181,7 @@ module Logging = .Destructure.FSharpTypes() .Enrich.FromLogContext() |> fun c -> if verbose then c.MinimumLevel.Debug() else c - |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {Properties} {NewLine}{Exception}" + |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {NewLine}{Exception}" c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() @@ -441,63 +444,68 @@ module Ingester = return queue } -type SliceStatsBuffer(?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() - member __.Ingest(slice: EventStore.ClientAPI.AllEventsSlice) = - let mutable batchBytes = 0 - for x in slice.Events do - let cat = Ingester.category x.OriginalStreamId - let eventBytes = Ingester.esPayloadBytes x - match recentCats.TryGetValue cat with - | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) - | false, _ -> recentCats.[cat] <- (1, eventBytes) - batchBytes <- batchBytes + eventBytes - slice.Events.Length, int64 batchBytes - member __.DumpIfIntervalExpired(?force) = - if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then - let log = function - | [||] -> () - | xs -> - xs - |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) - |> Seq.truncate 10 - |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) - |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log - recentCats.Clear() - accStart.Restart() - -type OverallStats(?statsInterval) = - let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() - let mutable totalEvents, totalBytes = 0L, 0L - member __.Ingest(batchEvents, batchBytes) = - totalEvents <- totalEvents + batchEvents - totalBytes <- totalBytes + batchBytes - member __.Bytes = totalBytes - member __.Events = totalEvents - member __.DumpIfIntervalExpired() = - if progressStart.ElapsedMilliseconds > intervalMs then - let totalMb = mb totalBytes - Log.Warning("Traversed {events} events {gb:n1}GB {mbs}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) - progressStart.Restart() - -type Range(start, sliceEnd : EventStore.ClientAPI.Position option, max : EventStore.ClientAPI.Position) = - member val Current = start with get, set - member __.TryNext(pos: EventStore.ClientAPI.Position) = - __.Current <- pos - __.IsCompleted - member __.IsCompleted = - match sliceEnd with - | Some send when __.Current.CommitPosition >= send.CommitPosition -> false - | _ -> true - member __.PositionAsRangePercentage = float __.Current.CommitPosition/float max.CommitPosition - module EventStoreReader = open EventStore.ClientAPI + type SliceStatsBuffer(?interval) = + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() + member __.Ingest(slice: AllEventsSlice) = + lock recentCats <| fun () -> + let mutable batchBytes = 0 + for x in slice.Events do + let cat = Ingester.category x.OriginalStreamId + let eventBytes = Ingester.esPayloadBytes x + match recentCats.TryGetValue cat with + | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) + | false, _ -> recentCats.[cat] <- (1, eventBytes) + batchBytes <- batchBytes + eventBytes + __.DumpIfIntervalExpired() + slice.Events.Length, int64 batchBytes + member __.DumpIfIntervalExpired(?force) = + if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then + lock recentCats <| fun () -> + let log = function + | [||] -> () + | xs -> + xs + |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) + |> Seq.truncate 10 + |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) + |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log + recentCats.Clear() + accStart.Restart() + + type OverallStats(?statsInterval) = + let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() + let mutable totalEvents, totalBytes = 0L, 0L + member __.Ingest(batchEvents, batchBytes) = + Interlocked.Add(&totalEvents,batchEvents) |> ignore + Interlocked.Add(&totalBytes,batchBytes) |> ignore + member __.Bytes = totalBytes + member __.Events = totalEvents + member __.DumpIfIntervalExpired(?force) = + if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then + let totalMb = mb totalBytes + Log.Warning("Traversed {events} events {gb:n1}GB {mbs:n2}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + progressStart.Restart() + + type Range(start, sliceEnd : Position option, max : Position) = + member val Current = start with get, set + member __.TryNext(pos: Position) = + __.Current <- pos + __.IsCompleted + member __.IsCompleted = + match sliceEnd with + | Some send when __.Current.CommitPosition >= send.CommitPosition -> false + | _ -> true + member __.PositionAsRangePercentage = + if max.CommitPosition=0L then Double.NaN + else float __.Current.CommitPosition/float max.CommitPosition + // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 let posFromChunk (chunk: int) = @@ -513,7 +521,7 @@ module EventStoreReader = let fetchMax (conn : IEventStoreConnection) = async { let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.NextPosition - Log.Warning("EventStore {chunks} chunks Write Position @ {pos} ", chunk max, max.CommitPosition) + Log.Warning("EventStore {chunks} chunks, ~{gb:n1}GB Write Position @ {pos} ", chunk max, mb max.CommitPosition/1024., max.CommitPosition) return max } let establishMax (conn : IEventStoreConnection) = async { let mutable max = None @@ -537,59 +545,61 @@ module EventStoreReader = fetchFrom 0L type [] PullResult = Exn of exn: exn | Eof | EndOfTranche - let pullSourceRange (conn : IEventStoreConnection, batchSize) (range : Range) enumEvents (postBatch : Ingester.Batch -> unit) = - let stats, slicesStats = OverallStats(), SliceStatsBuffer() - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let rec loop () = async { - let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let postSw = Stopwatch.StartNew() - let batchEvents, batchBytes = slicesStats.Ingest currentSlice in stats.Ingest(int64 batchEvents, batchBytes) - slicesStats.DumpIfIntervalExpired() - let streams = - enumEvents currentSlice.Events - |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) - |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) - |> Array.ofSeq - let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - let mutable usedEvents = 0 - for stream,streamEvents in streams do - for pos, item in streamEvents do - usedEvents <- usedEvents + 1 - postBatch { stream = stream; span = { pos = pos; events = [| item |]}} - let shouldLoop = range.TryNext currentSlice.NextPosition - Log.Warning("Read {count} {ft:n3}s {mb:n1}MB Process c {categories,2} s {streams,4} e {events,4} {pt:n0}ms Pos @ {pos} {pct:p1}", - batchEvents, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds, - currentSlice.NextPosition.CommitPosition, range.PositionAsRangePercentage) - if shouldLoop && not currentSlice.IsEndOfStream then - sw.Restart() // restart the clock as we hand off back to the Reader - return! loop () - else - slicesStats.DumpIfIntervalExpired(force=true) - return currentSlice.IsEndOfStream } - async { - try let! eof = loop () - return (if eof then Eof else EndOfTranche), range, stats - with e -> return Exn e, range, stats } + type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : Ingester.Batch -> unit) = + member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?ignoreEmptyEof) = + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let rec loop () = async { + let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let postSw = Stopwatch.StartNew() + let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) + let streams = + enumEvents currentSlice.Events + |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) + |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) + |> Array.ofSeq + let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length + let mutable usedEvents = 0 + for stream,streamEvents in streams do + for pos, item in streamEvents do + usedEvents <- usedEvents + 1 + postBatch { stream = stream; span = { pos = pos; events = [| item |]}} + if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( + Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {count,4} {mb:n1}MB {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), batchEvents, mb batchBytes, + usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds) + let shouldLoop = range.TryNext currentSlice.NextPosition + if shouldLoop && not currentSlice.IsEndOfStream then + sw.Restart() // restart the clock as we hand off back to the Reader + return! loop () + else + return currentSlice.IsEndOfStream } + async { + try let! eof = loop () + return if eof then Eof else EndOfTranche + with e -> return Exn e } type [] Work = | Stream of name: string * batchSize: int | Tranche of range: Range * batchSize : int + | Tail of pos: Position * interval: TimeSpan * batchSize : int type FeedQueue(batchSize, max, ?statsInterval) = let work = ConcurrentQueue() - member val OverallStats = OverallStats(?statsInterval=statsInterval) with get + member val OverallStats = OverallStats(?statsInterval=statsInterval) + member val SlicesStats = SliceStatsBuffer() member __.AddTranche(range, ?batchSizeOverride) = work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) member __.AddTranche(pos, nextPos, ?batchSizeOverride) = __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) member __.AddStream(name, ?batchSizeOverride) = work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + member __.AddTail(pos, interval, ?batchSizeOverride) = + work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() member __.Process(conn, enumEvents, postBatch, work) = async { - let adjust batchSize = if batchSize > 128 then batchSize / 2 else batchSize + let adjust batchSize = if batchSize > 128 then batchSize - 128 else batchSize match work with | Stream (name,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Stream",name) @@ -603,24 +613,60 @@ module EventStoreReader = | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Reading chunk; batch size {bs}", batchSize) - let! eofOption, range, stats = pullSourceRange (conn, batchSize) range enumEvents postBatch - lock __.OverallStats <| fun () -> __.OverallStats.Ingest(stats.Events, stats.Bytes) - match eofOption with + let reader = ReaderGroup(conn, enumEvents, postBatch) + let! res = reader.Pump(range, batchSize, __.SlicesStats, __.OverallStats) + match res with | PullResult.EndOfTranche -> Log.Warning("Completed tranche") + __.OverallStats.DumpIfIntervalExpired() return false | PullResult.Eof -> Log.Warning("REACHED THE END!") + __.OverallStats.DumpIfIntervalExpired(true) return true | PullResult.Exn e -> let bs = adjust batchSize Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) + __.OverallStats.DumpIfIntervalExpired() __.AddTranche(range, bs) - return false } + return false + | Tail (pos, interval, batchSize) -> + let mutable first, count, batchSize, range = true, 0, batchSize, Range(pos,None, Position.Start) + let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) + let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds + let progressSw, tailSw = Stopwatch.StartNew(), Stopwatch.StartNew() + let reader = ReaderGroup(conn, enumEvents, postBatch) + let slicesStats, stats = SliceStatsBuffer(), OverallStats() + while true do + let currentPos = range.Current + use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") + if first then + first <- false + Log.Warning("Tailing at {interval}s interval", interval.TotalSeconds) + elif progressSw.ElapsedMilliseconds > progressIntervalMs then + Log.Warning("Performed {count} tails to date @ {pos} chunk {chunk}", count, currentPos.CommitPosition, chunk currentPos) + progressSw.Restart() + count <- count + 1 + let! res = reader.Pump(range,batchSize,slicesStats,stats,ignoreEmptyEof=true) + stats.DumpIfIntervalExpired() + match tailIntervalMs - tailSw.ElapsedMilliseconds with + | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) + | _ -> () + tailSw.Restart() + match res with + | PullResult.EndOfTranche | PullResult.Eof -> () + | PullResult.Exn e -> + batchSize <- adjust batchSize + Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) + return true } type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = let work = FeedQueue(spec.batchSize, max, ?statsInterval=statsInterval) - do for s in spec.streams do work.AddStream(s) + do match spec.tailInterval with + | Some interval -> work.AddTail(max, interval) + | None -> () + for s in spec.streams do + work.AddStream s let mutable remainder = let startPos = match spec.start with @@ -628,15 +674,18 @@ module EventStoreReader = | Absolute p -> Position(p, 0L) | Chunk c -> posFromChunk c | Percentage pct -> posFromPercentage (pct, max) + | Ignore -> max Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) - let nextPos = posFromChunkAfter startPos - work.AddTranche(startPos, nextPos) - Some nextPos + if spec.start = Ignore then None + else + let nextPos = posFromChunkAfter startPos + work.AddTranche(startPos, nextPos) + Some nextPos member __.Pump () = async { (*if spec.tail then enqueue tail work*) - let maxDop = spec.stripes + let maxDop = spec.stripes + Option.count spec.tailInterval let dop = new SemaphoreSlim(maxDop) let mutable finished = false while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do @@ -644,9 +693,9 @@ module EventStoreReader = work.OverallStats.DumpIfIntervalExpired() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - let! eof = work.Process(conn, enumEvents, postBatch, task) - if eof then remainder <- None - dop.Release() |> ignore } + try let! eof = work.Process(conn, enumEvents, postBatch, task) + if eof then remainder <- None + finally dop.Release() |> ignore } return () } match work.TryDequeue() with | true, task -> @@ -658,8 +707,9 @@ module EventStoreReader = remainder <- Some nextPos do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) | None -> - finished <- true - Log.Warning("No further work to commence") } + if finished then do! Async.Sleep 1000 + else Log.Warning("No further ingestion work to commence") + finished <- true } let start (conn, spec, enumEvents, postBatch) = async { let! ct = Async.CancellationToken From 86decb4afbdeb938f91b9450fbea1cb8af527417 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 22 Mar 2019 12:44:43 +0000 Subject: [PATCH 023/353] Tidying --- equinox-ingest/Ingest/Infrastructure.fs | 26 ++++++++++++------------- equinox-ingest/Ingest/Program.fs | 14 ++++++------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs index 588ae2ab8..f289dd8f1 100644 --- a/equinox-ingest/Ingest/Infrastructure.fs +++ b/equinox-ingest/Ingest/Infrastructure.fs @@ -6,19 +6,6 @@ open System open System.Threading open System.Threading.Tasks -#nowarn "21" // re AwaitKeyboardInterrupt -#nowarn "40" // re AwaitKeyboardInterrupt - -type Async with - static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) - /// Asynchronously awaits the next keyboard interrupt event - static member AwaitKeyboardInterrupt () : Async = - Async.FromContinuations(fun (sc,_,_) -> - let isDisposed = ref 0 - let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore - and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) - type SemaphoreSlim with /// F# friendly semaphore await function member semaphore.Await(?timeout : TimeSpan) = async { @@ -38,3 +25,16 @@ module Queue = | false, _ -> None | true, res -> Some res #endif + +#nowarn "21" // re AwaitKeyboardInterrupt +#nowarn "40" // re AwaitKeyboardInterrupt + +type Async with + static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) + /// Asynchronously awaits the next keyboard interrupt event + static member AwaitKeyboardInterrupt () : Async = + Async.FromContinuations(fun (sc,_,_) -> + let isDisposed = ref 0 + let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore + and d : IDisposable = Console.CancelKeyPress.Subscribe callback + in ()) \ No newline at end of file diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 8ff0b1765..73fbdbd2a 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -50,7 +50,7 @@ module CmdParser = member __.MaxRetryWaitTime = args.GetResult(RetriesWaitTime, 5) /// Connect with the provided parameters and/or environment variables - member x.Connnect + member x.Connect /// Connection/Client identifier for logging purposes name : Async = let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = Discovery.FromConnectionString x.Connection @@ -739,8 +739,7 @@ let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) } -let run (destination : CosmosConnection, colls) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { - let ctx = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) +let run (ctx : Equinox.Cosmos.Core.CosmosContext) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) let! _feeder = EventStoreReader.start(source.ReadConnection, spec, enumEvents, ingester.Add) do! Async.AwaitKeyboardInterrupt() } @@ -752,11 +751,12 @@ let main argv = let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) |> Async.RunSynchronously let readerSpec = args.BuildFeedParams() let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 - if Threading.ThreadPool.SetMaxThreads(512,512) |> not then failwith "Could not set max threads" let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu - let destination = cosmos.Connnect "ProjectorTemplate" |> Async.RunSynchronously - let colls = CosmosCollections(cosmos.Database, cosmos.Collection) - run (destination, colls) source readerSpec (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously + let ctx = + let destination = cosmos.Connect "ProjectorTemplate" |> Async.RunSynchronously + let colls = CosmosCollections(cosmos.Database, cosmos.Collection) + Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) + run ctx source readerSpec (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 From 42558410f2b4817edb0b6d7d3f03d1c7fa490161 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 23 Mar 2019 02:19:22 +0000 Subject: [PATCH 024/353] Sync tool bugfix --- equinox-ingest/Ingest/Program.fs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 73fbdbd2a..3acd6a3aa 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -156,11 +156,11 @@ module CmdParser = Log.Warning("Reading with {stripes} stripes", x.Stripes) let startPos = match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent, args.Contains All with - | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some c, _, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c - | _, _, Some p, _ -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some c, _, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c + | _, _, Some p, _ -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p | None, None, None, true -> Log.Warning "Processing will commence at $all Start"; Start - | None, None, None, false -> Log.Warning "No $all processing requested"; Ignore + | None, None, None, false ->Log.Warning "No $all processing requested"; Ignore match x.TailInterval with | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) | None -> Log.Warning "Not following tail" @@ -242,13 +242,13 @@ module Ingester = Log.Information("Writing {s}@{i}x{n}",s,p,e.Length) try let! res = ctx.Sync(stream, { index = p; etag = None }, e) match res with - | AppendResult.Ok _pos -> return Ok (s, p) + | AppendResult.Ok pos -> return Ok (s, pos.index) | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> match pos.index, p + e.LongLength with | actual, expectedMax when actual >= expectedMax -> return Duplicate (s, pos.index) | actual, _ when p >= actual -> return Conflict batch | actual, _ -> - Log.Debug("pos {pos} batch.pos {bpos} len {blen} skip {skio}", actual, p, e.LongLength, actual-p) + Log.Debug("pos {pos} batch.pos {bpos} len {blen} skip {skip}", actual, p, e.LongLength, actual-p) return Conflict { stream = s; span = { pos = actual; events = e |> Array.skip (actual-p |> int) } } with e -> return Exn (e, batch) } From b53dc2442f2fb0effeab5a8c5dfad33c8f0789f1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 23 Mar 2019 22:34:51 +0000 Subject: [PATCH 025/353] Update to preview2 --- equinox-ingest/Ingest/Ingest.fsproj | 4 ++-- equinox-ingest/Ingest/Program.fs | 9 ++++++--- equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj | 2 +- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj index ca8d76add..aef328deb 100644 --- a/equinox-ingest/Ingest/Ingest.fsproj +++ b/equinox-ingest/Ingest/Ingest.fsproj @@ -15,8 +15,8 @@ - - + + diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 3acd6a3aa..dfd1dc9ce 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -56,7 +56,7 @@ module CmdParser = let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = Discovery.FromConnectionString x.Connection Log.Information("CosmosDb {mode} {endpointUri} Database {database} Collection {collection}.", x.Mode, endpointUri, x.Database, x.Collection) - Log.Information("CosmosDb timeout: {timeout}s, {retries} retries; Throttling maxRetryWaitTime {maxRetryWaitTime}", + Log.Information("CosmosDb timeout: {timeout}s; Throttling {retries} retries; Throttling maxRetryWaitTime {maxRetryWaitTime}", (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) let c = CosmosConnector(log=Log.Logger, mode=x.Mode, requestTimeout=x.Timeout, @@ -444,6 +444,9 @@ module Ingester = return queue } +type EventStore.ClientAPI.RecordedEvent with + member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) + module EventStoreReader = open EventStore.ClientAPI @@ -539,7 +542,7 @@ module EventStoreReader = let events = [| for x in currentSlice.Events -> let e = x.Event - Equinox.Codec.Core.EventData.Create (e.EventType, e.Data, e.Metadata) :> Equinox.Codec.IEvent |] + Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] postBatch { stream = stream; span = { pos = currentSlice.FromEventNumber; events = events } } return! fetchFrom currentSlice.NextEventNumber } fetchFrom 0L @@ -736,7 +739,7 @@ let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { | e when eb > Ingester.cosmosPayloadLimit -> Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) Choice2Of2 e - | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata)) + | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) } let run (ctx : Equinox.Cosmos.Core.CosmosContext) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj b/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj index b1a0d7f20..227af1044 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj @@ -12,7 +12,7 @@ - + From cb9619931403e57616c6beb7203346cd28bb66a2 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 24 Mar 2019 23:39:56 +0000 Subject: [PATCH 026/353] Sync with other templates --- equinox-ingest/Ingest/Program.fs | 299 +++++++++--------- equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs | 5 +- equinox-ingest/Sync.Tests/equinox-sync.sln | 7 +- 3 files changed, 153 insertions(+), 158 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index dfd1dc9ce..a9847f12a 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -4,6 +4,9 @@ open Equinox.Store // Infra open FSharp.Control open Serilog open System +open System.Collections.Concurrent +open System.Diagnostics +open System.Threading type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start | Ignore type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; streams: string list; tailInterval: TimeSpan option } @@ -11,7 +14,6 @@ let mb x = float x / 1024. / 1024. module CmdParser = open Argu - type LogEventLevel = Serilog.Events.LogEventLevel exception MissingArg of string let envBackstop msg key = @@ -21,7 +23,7 @@ module CmdParser = module Cosmos = open Equinox.Cosmos - type [] Arguments = + type [] Parameters = | [] ConnectionMode of ConnectionMode | [] Timeout of float | [] Retries of int @@ -39,15 +41,15 @@ module CmdParser = | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE, test)." | Collection _ -> "specify a collection name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_COLLECTION, test)." - type Info(args : ParseResults) = - member __.Connection = match args.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" - member __.Database = match args.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" - member __.Collection = match args.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" - - member __.Timeout = args.GetResult(Timeout,10.) |> TimeSpan.FromSeconds - member __.Mode = args.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) - member __.Retries = args.GetResult(Retries, 0) - member __.MaxRetryWaitTime = args.GetResult(RetriesWaitTime, 5) + type Arguments(a : ParseResults) = + member __.Mode = a.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) + member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" + member __.Database = match a.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" + member __.Collection = match a.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" + + member __.Timeout = a.GetResult(Timeout,10.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(Retries, 0) + member __.MaxRetryWaitTime = a.GetResult(RetriesWaitTime, 5) /// Connect with the provided parameters and/or environment variables member x.Connect @@ -56,19 +58,17 @@ module CmdParser = let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = Discovery.FromConnectionString x.Connection Log.Information("CosmosDb {mode} {endpointUri} Database {database} Collection {collection}.", x.Mode, endpointUri, x.Database, x.Collection) - Log.Information("CosmosDb timeout: {timeout}s; Throttling {retries} retries; Throttling maxRetryWaitTime {maxRetryWaitTime}", + Log.Information("CosmosDb timeout {timeout}s; Throttling retries {retries}, max wait {maxRetryWaitTime}s", (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) - let c = - CosmosConnector(log=Log.Logger, mode=x.Mode, requestTimeout=x.Timeout, - maxRetryAttemptsOnThrottledRequests=x.Retries, maxRetryWaitTimeInSeconds=x.MaxRetryWaitTime) - c.Connect(name, discovery) + let connector = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) + connector.Connect(name, discovery) /// To establish a local node to run against: /// 1. cinst eventstore-oss -y # where cinst is an invocation of the Chocolatey Package Installer on Windows /// 2. & $env:ProgramData\chocolatey\bin\EventStore.ClusterNode.exe --gossip-on-single-node --discover-via-dns 0 --ext-http-port=30778 module EventStore = open Equinox.EventStore - type [] Arguments = + type [] Parameters = | [] VerboseStore | [] Timeout of float | [] Retries of int @@ -79,7 +79,7 @@ module CmdParser = | [] ConcurrentOperationsLimit of int | [] HeartbeatTimeout of float | [] MaxItems of int - | [] Cosmos of ParseResults + | [] Cosmos of ParseResults interface IArgParserTemplate with member a.Usage = match a with @@ -94,21 +94,21 @@ module CmdParser = | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." | MaxItems _ -> "maximum item count to request. Default: 4096" | Cosmos _ -> "specify CosmosDb parameters" - type Info(args : ParseResults ) = + type Arguments(a : ParseResults ) = let connect (log: ILogger) (heartbeatTimeout, col) (operationTimeout, operationRetries) discovery (username, password) connection = - let log = if log.IsEnabled LogEventLevel.Debug then Logger.SerilogVerbose log else Logger.SerilogNormal log + let log = if log.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose log else Logger.SerilogNormal log GesConnector(username, password, operationTimeout, operationRetries,heartbeatTimeout=heartbeatTimeout, concurrentOperationsLimit=col, log=log, tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string]) .Establish("ProjectorTemplate", discovery, connection) - member val Cosmos = Cosmos.Info(args.GetResult Cosmos) - member __.Host = match args.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" - member __.Port = match args.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int - member __.User = match args.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" - member __.Password = match args.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" + member val Cosmos = Cosmos.Arguments(a.GetResult Cosmos) + member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" + member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int + member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" + member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" member __.Connect(log: ILogger, storeLog, connection) = - let (timeout, retries) as operationThrottling = args.GetResult(Timeout,20.) |> TimeSpan.FromSeconds, args.GetResult(Retries,3) - let heartbeatTimeout = args.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds - let concurrentOperationsLimit = args.GetResult(ConcurrentOperationsLimit,5000) + let (timeout, retries) as operationThrottling = a.GetResult(Timeout,20.) |> TimeSpan.FromSeconds, a.GetResult(Retries,3) + let heartbeatTimeout = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds + let concurrentOperationsLimit = a.GetResult(ConcurrentOperationsLimit,5000) log.Information("EventStore {host} heartbeat: {heartbeat}s MaxConcurrentRequests {concurrency} Timeout: {timeout}s Retries {retries}", __.Host, heartbeatTimeout.TotalSeconds, concurrentOperationsLimit, timeout.TotalSeconds, retries) let discovery = match __.Port with None -> Discovery.GossipDns __.Host | Some p -> Discovery.GossipDnsCustomPort (__.Host, p) @@ -127,7 +127,7 @@ module CmdParser = | [] Percent of float | [] Stripes of int | [] Tail of intervalS: float - | [] Es of ParseResults + | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = match a with @@ -144,16 +144,15 @@ module CmdParser = | Tail _ -> "attempt to read from tail at specified interval in Seconds" | Es _ -> "specify EventStore parameters" and Parameters(args : ParseResults) = - member val EventStore = EventStore.Info(args.GetResult Es) - member __.Verbose = args.Contains Verbose - member __.ConsoleMinLevel = if args.Contains VerboseConsole then LogEventLevel.Information else LogEventLevel.Warning - member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None - member __.BatchSize = args.GetResult(BatchSize,4096) - member __.Stripes = args.GetResult(Stripes,1) - member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None + member val EventStore = EventStore.Arguments(args.GetResult Es) + member __.Verbose = args.Contains Verbose + member __.ConsoleMinLevel = if args.Contains VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning + member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None + member __.BatchSize = args.GetResult(BatchSize,4096) + member __.Stripes = args.GetResult(Stripes,1) + member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None member x.BuildFeedParams() : ReaderSpec = - Log.Warning("Processing in batches of {batchSize}", x.BatchSize) - Log.Warning("Reading with {stripes} stripes", x.Stripes) + Log.Warning("Processing in batches of {batchSize} with {stripes} stripes", x.BatchSize, x.Stripes) let startPos = match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent, args.Contains All with | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p @@ -186,118 +185,106 @@ module Logging = |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() -open System.Collections.Concurrent -open System.Diagnostics -open System.Threading - module Ingester = open Equinox.Cosmos.Core open Equinox.Cosmos.Store - type [] Span = { pos: int64; events: Equinox.Codec.IEvent[] } - module Span = - let private (|Max|) x = x.pos + x.events.LongLength - let private trim min (Max m as x) = - // Full remove - if m <= min then { pos = min; events = [||] } - // Trim until min - elif m > min && x.pos < min then { pos = min; events = x.events |> Array.skip (min - x.pos |> int) } - // Leave it - else x - let merge min (xs : Span seq) = - let buffer = ResizeArray() - let mutable curr = { pos = min; events = [||]} - for x in xs |> Seq.sortBy (fun x -> x.pos) do - match curr, trim min x with - // no data incoming, skip - | _, x when x.events.Length = 0 -> - () - // Not overlapping, no data buffered -> buffer - | c, x when c.events.Length = 0 -> - curr <- x - // Overlapping, join - | Max cMax as c, x when cMax >= x.pos -> - curr <- { c with events = Array.append c.events (trim cMax x).events } - // Not overlapping, new data - | c, x -> - buffer.Add c - curr <- x - if curr.events.Length <> 0 then buffer.Add curr - if buffer.Count = 0 then null else buffer.ToArray() - + type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } - type [] Result = - | Ok of stream: string * updatedPos: int64 - | Duplicate of stream: string * updatedPos: int64 - | Conflict of overage: Batch - | Exn of exn: exn * batch: Batch with - member __.WriteTo(log: ILogger) = - match __ with - | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) - | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) - | Conflict overage -> log.Information("Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.pos, overage.span.events.Length) - | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) - let private write (ctx : CosmosContext) ({ stream = s; span={ pos = p; events = e}} as batch) = async { - let stream = ctx.CreateStream s - Log.Information("Writing {s}@{i}x{n}",s,p,e.Length) - try let! res = ctx.Sync(stream, { index = p; etag = None }, e) - match res with - | AppendResult.Ok pos -> return Ok (s, pos.index) - | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> - match pos.index, p + e.LongLength with - | actual, expectedMax when actual >= expectedMax -> return Duplicate (s, pos.index) - | actual, _ when p >= actual -> return Conflict batch - | actual, _ -> - Log.Debug("pos {pos} batch.pos {bpos} len {blen} skip {skip}", actual, p, e.LongLength, actual-p) - return Conflict { stream = s; span = { pos = actual; events = e |> Array.skip (actual-p |> int) } } - with e -> return Exn (e, batch) } - - /// Manages distribution of work across a specified number of concurrent writers - type Writer (ctx : CosmosContext, queueLen, ct : CancellationToken) = - let buffer = new BlockingCollection<_>(ConcurrentQueue(), queueLen) - let result = Event<_>() - let child = async { - let! ct = Async.CancellationToken // i.e. cts.Token - for item in buffer.GetConsumingEnumerable(ct) do - let! res = write ctx item - result.Trigger res } - member internal __.StartConsumers n = - for _ in 1..n do - Async.StartAsTask(child, cancellationToken=ct) |> ignore - - /// Supply an item to be processed - member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) - [] member __.Result = result.Publish + + module Writer = + type [] Result = + | Ok of stream: string * updatedPos: int64 + | Duplicate of stream: string * updatedPos: int64 + | Conflict of overage: Batch + | Exn of exn: exn * batch: Batch with + member __.WriteTo(log: ILogger) = + match __ with + | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) + | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) + | Conflict overage -> log.Information("Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.index, overage.span.events.Length) + | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) + let private write (ctx : CosmosContext) ({ stream = s; span={ index = p; events = e}} as batch) = async { + let stream = ctx.CreateStream s + Log.Information("Writing {s}@{i}x{n}",s,p,e.Length) + try let! res = ctx.Sync(stream, { index = p; etag = None }, e) + match res with + | AppendResult.Ok pos -> return Ok (s, pos.index) + | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> + match pos.index, p + e.LongLength with + | actual, expectedMax when actual >= expectedMax -> return Duplicate (s, pos.index) + | actual, _ when p >= actual -> return Conflict batch + | actual, _ -> + Log.Debug("pos {pos} batch.pos {bpos} len {blen} skip {skip}", actual, p, e.LongLength, actual-p) + return Conflict { stream = s; span = { index = actual; events = e |> Array.skip (actual-p |> int) } } + with e -> return Exn (e, batch) } + + /// Manages distribution of work across a specified number of concurrent writers + type WriteQueue(ctx : CosmosContext, queueLen, ct : CancellationToken) = + let buffer = new BlockingCollection<_>(ConcurrentQueue(), queueLen) + let result = Event<_>() + let child = async { + let! ct = Async.CancellationToken // i.e. cts.Token + for item in buffer.GetConsumingEnumerable(ct) do + let! res = write ctx item + result.Trigger res } + member internal __.StartConsumers n = + for _ in 1..n do + Async.StartAsTask(child, cancellationToken=ct) |> ignore + + /// Supply an item to be processed + member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) + [] member __.Result = result.Publish let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length type [] StreamState = { read: int64 option; write: int64 option; isMalformed : bool; queue: Span[] } with /// Determines whether the head is ready to write (either write position is unknown, or matches) - member __.IsHeady = Array.tryHead __.queue |> Option.exists (fun x -> __.write |> Option.forall (fun w -> w = x.pos)) + member __.IsHeady = Array.tryHead __.queue |> Option.exists (fun x -> __.write |> Option.forall (fun w -> w = x.index)) member __.IsReady = __.queue <> null && not __.isMalformed && __.IsHeady member __.Size = if __.queue = null then 0 else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length*2 + 16) - - let inline optionCombine f (r1: int64 option) (r2: int64 option) = - match r1, r2 with - | Some x, Some y -> f x y |> Some - | None, None -> None - | None, x | x, None -> x - - let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 - let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 - let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata - let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 - let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head - let isMalformedException (e: #exn) = - e.ToString().Contains "SyntaxError: JSON.parse Error: Unexpected input at position" - || e.ToString().Contains "SyntaxError: JSON.parse Error: Invalid character at position" - - let combine (s1: StreamState) (s2: StreamState) : StreamState = - let writePos = optionCombine max s1.write s2.write - let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } - { read = optionCombine max s1.read s2.read; write = writePos; isMalformed = s1.isMalformed || s2.isMalformed; queue = Span.merge (defaultArg writePos 0L) items} + module StreamState = + module Span = + let private (|Max|) x = x.index + x.events.LongLength + let private trim min (Max m as x) = + // Full remove + if m <= min then { index = min; events = [||] } + // Trim until min + elif m > min && x.index < min then { index = min; events = x.events |> Array.skip (min - x.index |> int) } + // Leave it + else x + let merge min (xs : Span seq) = + let buffer = ResizeArray() + let mutable curr = { index = min; events = [||]} + for x in xs |> Seq.sortBy (fun x -> x.index) do + match curr, trim min x with + // no data incoming, skip + | _, x when x.events.Length = 0 -> + () + // Not overlapping, no data buffered -> buffer + | c, x when c.events.Length = 0 -> + curr <- x + // Overlapping, join + | Max cMax as c, x when cMax >= x.index -> + curr <- { c with events = Array.append c.events (trim cMax x).events } + // Not overlapping, new data + | c, x -> + buffer.Add c + curr <- x + if curr.events.Length <> 0 then buffer.Add curr + if buffer.Count = 0 then null else buffer.ToArray() + + let inline optionCombine f (r1: int64 option) (r2: int64 option) = + match r1, r2 with + | Some x, Some y -> f x y |> Some + | None, None -> None + | None, x | x, None -> x + let combine (s1: StreamState) (s2: StreamState) : StreamState = + let writePos = optionCombine max s1.write s2.write + let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } + { read = optionCombine max s1.read s2.read; write = writePos; isMalformed = s1.isMalformed || s2.isMalformed; queue = Span.merge (defaultArg writePos 0L) items} /// Gathers stats relating to how many items of a given category have been observed type CatStats() = @@ -310,6 +297,13 @@ module Ingester = member __.Clear() = cats.Clear() member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd + let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head + let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 + let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 + let isMalformedException (e: #exn) = + e.ToString().Contains "SyntaxError: JSON.parse Error: Unexpected input at position" + || e.ToString().Contains "SyntaxError: JSON.parse Error: Invalid character at position" + type StreamStates() = let states = System.Collections.Generic.Dictionary() let dirty = System.Collections.Generic.Queue() @@ -322,18 +316,18 @@ module Ingester = states.Add(stream, state) markDirty stream |> ignore | true, current -> - let updated = combine current state + let updated = StreamState.combine current state states.[stream] <- updated if updated.IsReady then markDirty stream |> ignore let updateWritePos stream pos isMalformed span = - update stream { read = None; write = Some pos; isMalformed = isMalformed; queue = span } + update stream { read = None; write = pos; isMalformed = isMalformed; queue = span } - member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream 0L (defaultArg isMalformed false) [|item.span|] + member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream None (defaultArg isMalformed false) [|item.span|] member __.HandleWriteResult = function - | Ok (stream, pos) -> updateWritePos stream pos false null; None - | Duplicate (stream, pos) -> updateWritePos stream pos false null; None - | Conflict overage -> updateWritePos overage.stream overage.span.pos false [|overage.span|]; None - | Exn (exn, batch) -> + | Writer.Result.Ok (stream, pos) -> updateWritePos stream (Some pos) false null; None + | Writer.Result.Duplicate (stream, pos) -> updateWritePos stream (Some pos) false null; None + | Writer.Result.Conflict overage -> updateWritePos overage.stream (Some overage.span.index) false [|overage.span|]; None + | Writer.Result.Exn (exn, batch) -> let malformed = isMalformedException exn __.Add(batch,malformed) if malformed then Some (category batch.stream) else None @@ -354,7 +348,7 @@ module Ingester = count <- count + 1 // Reduce the item count when we don't yet know the write position count < (if Option.isNone state.write then 10 else 1000) && (bytesBudget >= 0 || count = 1) - Some { stream = stream; span = { pos = x.pos; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } + Some { stream = stream; span = { index = x.index; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } member __.Dump() = let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 let mutable readyB, waitingB, malformedB = 0L, 0L, 0L @@ -369,7 +363,7 @@ module Ingester = dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) if waitCats.Any then Log.Warning("Waiting {waitCats}", waitCats.StatsDescending) - type Queue(log : Serilog.ILogger, writer : Writer, cancellationToken: CancellationToken, readerQueueLen, ?interval) = + type SyncQueue(log : Serilog.ILogger, writer : Writer.WriteQueue, cancellationToken: CancellationToken, readerQueueLen, ?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let states = StreamStates() let results = ConcurrentQueue<_>() @@ -436,8 +430,8 @@ module Ingester = /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does let start(ctx : CosmosContext, writerQueueLen, writerCount, readerQueueLen) = async { let! ct = Async.CancellationToken - let writer = Writer(ctx, writerQueueLen, ct) - let queue = Queue(Log.Logger, writer, ct, readerQueueLen) + let writer = Writer.WriteQueue(ctx, writerQueueLen, ct) + let queue = SyncQueue(Log.Logger, writer, ct, readerQueueLen) let _ = writer.Result.Subscribe queue.HandleWriteResult // codependent, wont worry about unsubcribing writer.StartConsumers writerCount let! _ = Async.StartChild(async { queue.Pump() }) @@ -450,6 +444,9 @@ type EventStore.ClientAPI.RecordedEvent with module EventStoreReader = open EventStore.ClientAPI + let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = Ingester.arrayBytes x.Data + Ingester.arrayBytes x.Metadata + let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() @@ -458,7 +455,7 @@ module EventStoreReader = let mutable batchBytes = 0 for x in slice.Events do let cat = Ingester.category x.OriginalStreamId - let eventBytes = Ingester.esPayloadBytes x + let eventBytes = esPayloadBytes x match recentCats.TryGetValue cat with | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) | false, _ -> recentCats.[cat] <- (1, eventBytes) @@ -543,7 +540,7 @@ module EventStoreReader = [| for x in currentSlice.Events -> let e = x.Event Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] - postBatch { stream = stream; span = { pos = currentSlice.FromEventNumber; events = events } } + postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } return! fetchFrom currentSlice.NextEventNumber } fetchFrom 0L @@ -567,7 +564,7 @@ module EventStoreReader = for stream,streamEvents in streams do for pos, item in streamEvents do usedEvents <- usedEvents + 1 - postBatch { stream = stream; span = { pos = pos; events = [| item |]}} + postBatch { stream = stream; span = { index = pos; events = [| item |]}} if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {count,4} {mb:n1}MB {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), batchEvents, mb batchBytes, @@ -727,7 +724,7 @@ open Equinox.EventStore let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { for e in xs -> - let eb = Ingester.esPayloadBytes e + let eb = EventStoreReader.esPayloadBytes e match e.Event with | e when not e.IsJson || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs index 5fd996a19..69ba54cd1 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs @@ -4,8 +4,9 @@ open Swensen.Unquote open SyncTemplate.Program.Ingester open Xunit -let mk p c : Span = { pos = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null) |] } -let mergeSpans = Span.merge +let canonicalTime = System.DateTimeOffset.UtcNow +let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } +let mergeSpans = StreamState.Span.merge let [] ``nothing`` () = let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] diff --git a/equinox-ingest/Sync.Tests/equinox-sync.sln b/equinox-ingest/Sync.Tests/equinox-sync.sln index d14233807..ec7b74220 100644 --- a/equinox-ingest/Sync.Tests/equinox-sync.sln +++ b/equinox-ingest/Sync.Tests/equinox-sync.sln @@ -1,15 +1,12 @@ Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.27428.2002 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28714.193 MinimumVisualStudioVersion = 10.0.40219.1 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{458FF6DD-5F7F-419D-8440-1DCD72AF5229}" EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "Sync\Sync.fsproj", "{EA288107-08F6-4766-9DD5-E1C961AFA918}" EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B4CC54BB-AA13-47B2-BF18-C66423EAF19A}" - ProjectSection(SolutionItems) = preProject - equinox-sync.sln = equinox-sync.sln - EndProjectSection EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From 38afbadff803d57b48a9fa21598e9f119a9d12da Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 25 Mar 2019 12:40:54 +0000 Subject: [PATCH 027/353] Add MinBatchSize cmdline parameter --- equinox-ingest/Ingest/Program.fs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index a9847f12a..c1b50a43b 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -9,7 +9,7 @@ open System.Diagnostics open System.Threading type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start | Ignore -type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; streams: string list; tailInterval: TimeSpan option } +type ReaderSpec = { start: StartPos; streams: string list; tailInterval: TimeSpan option; stripes: int; batchSize: int; minBatchSize: int } let mb x = float x / 1024. / 1024. module CmdParser = @@ -117,6 +117,7 @@ module CmdParser = [] type Arguments = | [] BatchSize of int + | [] MinBatchSize of int | [] Verbose | [] VerboseConsole | [] LocalSeq @@ -132,6 +133,7 @@ module CmdParser = member a.Usage = match a with | BatchSize _ -> "maximum item count to request from feed. Default: 4096" + | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" @@ -148,11 +150,12 @@ module CmdParser = member __.Verbose = args.Contains Verbose member __.ConsoleMinLevel = if args.Contains VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None - member __.BatchSize = args.GetResult(BatchSize,4096) + member __.StartingBatchSize = args.GetResult(BatchSize,4096) + member __.MinBatchSize = args.GetResult(MinBatchSize,512) member __.Stripes = args.GetResult(Stripes,1) member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None member x.BuildFeedParams() : ReaderSpec = - Log.Warning("Processing in batches of {batchSize} with {stripes} stripes", x.BatchSize, x.Stripes) + Log.Warning("Processing in batches of {batchSize} (min {minBatchSize}) with {stripes} stripes", x.StartingBatchSize, x.MinBatchSize, x.Stripes) let startPos = match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent, args.Contains All with | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p @@ -163,7 +166,8 @@ module CmdParser = match x.TailInterval with | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) | None -> Log.Warning "Not following tail" - { start = startPos; stripes = x.Stripes; batchSize = x.BatchSize; streams = args.GetResults Stream; tailInterval = x.TailInterval } + { start = startPos; streams = args.GetResults Stream; tailInterval = x.TailInterval + batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args let parse argv : Parameters = @@ -566,9 +570,9 @@ module EventStoreReader = usedEvents <- usedEvents + 1 postBatch { stream = stream; span = { index = pos; events = [| item |]}} if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( - Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {count,4} {mb:n1}MB {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", - range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), batchEvents, mb batchBytes, - usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds) + Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, + batchEvents, usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds) let shouldLoop = range.TryNext currentSlice.NextPosition if shouldLoop && not currentSlice.IsEndOfStream then sw.Restart() // restart the clock as we hand off back to the Reader @@ -584,7 +588,7 @@ module EventStoreReader = | Stream of name: string * batchSize: int | Tranche of range: Range * batchSize : int | Tail of pos: Position * interval: TimeSpan * batchSize : int - type FeedQueue(batchSize, max, ?statsInterval) = + type FeedQueue(batchSize, minBatchSize, max, ?statsInterval) = let work = ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) member val SlicesStats = SliceStatsBuffer() @@ -599,7 +603,7 @@ module EventStoreReader = member __.TryDequeue () = work.TryDequeue() member __.Process(conn, enumEvents, postBatch, work) = async { - let adjust batchSize = if batchSize > 128 then batchSize - 128 else batchSize + let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize match work with | Stream (name,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Stream",name) @@ -661,7 +665,7 @@ module EventStoreReader = return true } type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = - let work = FeedQueue(spec.batchSize, max, ?statsInterval=statsInterval) + let work = FeedQueue(spec.batchSize, spec.minBatchSize, max, ?statsInterval=statsInterval) do match spec.tailInterval with | Some interval -> work.AddTail(max, interval) | None -> () From e185518cf9edb7ce3c41ffa1afa1c3dc6fad2ded Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 25 Mar 2019 15:48:55 +0000 Subject: [PATCH 028/353] delay first touch --- equinox-ingest/Ingest/Program.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index c1b50a43b..89962e219 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -616,7 +616,7 @@ module EventStoreReader = return false | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) - Log.Warning("Reading chunk; batch size {bs}", batchSize) + Log.Warning("Commencing tranche, batch size {bs}", batchSize) let reader = ReaderGroup(conn, enumEvents, postBatch) let! res = reader.Pump(range, batchSize, __.SlicesStats, __.OverallStats) match res with @@ -760,6 +760,7 @@ let main argv = let destination = cosmos.Connect "ProjectorTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) + Thread.Sleep(100) // https://github.com/EventStore/EventStore/issues/1899 run ctx source readerSpec (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 From 9d6cfca95f6b6e1727e58dce6132e636d3b19c52 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 27 Mar 2019 13:16:03 +0000 Subject: [PATCH 029/353] Update to Equinox 2.0.0-preview3 --- equinox-ingest/Ingest/Ingest.fsproj | 4 ++-- equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj index aef328deb..a6e638dd7 100644 --- a/equinox-ingest/Ingest/Ingest.fsproj +++ b/equinox-ingest/Ingest/Ingest.fsproj @@ -15,8 +15,8 @@ - - + + diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj b/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj index 227af1044..f6b9d1e05 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj +++ b/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj @@ -12,7 +12,6 @@ - From eb9213144f0702acea3184fdffcf3ac545637e65 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 2 Apr 2019 15:25:51 +0100 Subject: [PATCH 030/353] Improve error logging --- equinox-ingest/Ingest/Program.fs | 44 +++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 15 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 89962e219..cc92da2b5 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -155,7 +155,7 @@ module CmdParser = member __.Stripes = args.GetResult(Stripes,1) member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None member x.BuildFeedParams() : ReaderSpec = - Log.Warning("Processing in batches of {batchSize} (min {minBatchSize}) with {stripes} stripes", x.StartingBatchSize, x.MinBatchSize, x.Stripes) + Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent, args.Contains All with | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p @@ -304,10 +304,15 @@ module Ingester = let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 - let isMalformedException (e: #exn) = - e.ToString().Contains "SyntaxError: JSON.parse Error: Unexpected input at position" - || e.ToString().Contains "SyntaxError: JSON.parse Error: Invalid character at position" - + let (|TimedOutMessage|RateLimitedMessage|MalformedMessage|Other|) (e: exn) = + match string e with + | m when m.Contains "Microsoft.Azure.Documents.RequestTimeoutException" -> TimedOutMessage + | m when m.Contains "Microsoft.Azure.Documents.RequestRateTooLargeException" -> RateLimitedMessage + | m when m.Contains "SyntaxError: JSON.parse Error: Unexpected input at position" + || m.Contains "SyntaxError: JSON.parse Error: Invalid character at position" -> MalformedMessage + | _ -> Other + + type Result = TimedOut | RateLimited | Malformed of category: string | Ok type StreamStates() = let states = System.Collections.Generic.Dictionary() let dirty = System.Collections.Generic.Queue() @@ -328,13 +333,18 @@ module Ingester = member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream None (defaultArg isMalformed false) [|item.span|] member __.HandleWriteResult = function - | Writer.Result.Ok (stream, pos) -> updateWritePos stream (Some pos) false null; None - | Writer.Result.Duplicate (stream, pos) -> updateWritePos stream (Some pos) false null; None - | Writer.Result.Conflict overage -> updateWritePos overage.stream (Some overage.span.index) false [|overage.span|]; None + | Writer.Result.Ok (stream, pos) -> updateWritePos stream (Some pos) false null; Ok + | Writer.Result.Duplicate (stream, pos) -> updateWritePos stream (Some pos) false null; Ok + | Writer.Result.Conflict overage -> updateWritePos overage.stream (Some overage.span.index) false [|overage.span|]; Ok | Writer.Result.Exn (exn, batch) -> - let malformed = isMalformedException exn + let r, malformed = + match exn with + | RateLimitedMessage -> RateLimited, false + | TimedOutMessage -> TimedOut, false + | MalformedMessage -> Malformed (category batch.stream), true + | Other -> Ok, false __.Add(batch,malformed) - if malformed then Some (category batch.stream) else None + r member __.TryPending() = match dirty |> Queue.tryDequeue with | None -> None @@ -351,7 +361,7 @@ module Ingester = bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 // Reduce the item count when we don't yet know the write position - count < (if Option.isNone state.write then 10 else 1000) && (bytesBudget >= 0 || count = 1) + count <= (if Option.isNone state.write then 10 else 100) && (bytesBudget >= 0 || count = 1) Some { stream = stream; span = { index = x.index; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } member __.Dump() = let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 @@ -383,15 +393,18 @@ module Ingester = let badCats = CatStats() let progressTimer = Stopwatch.StartNew() while not cancellationToken.IsCancellationRequested do - let mutable moreResults = true + let mutable moreResults, rateLimited, timedOut = true, 0, 0 while moreResults do match results.TryDequeue() with | true, res -> incr resultsHandled match states.HandleWriteResult res with - | None -> res.WriteTo log - | Some cat -> badCats.Ingest cat + | Malformed cat -> badCats.Ingest cat + | RateLimited -> rateLimited <- rateLimited + 1 + | TimedOut -> timedOut <- timedOut + 1 + | Ok -> res.WriteTo log | false, _ -> moreResults <- false + if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) let mutable t = Unchecked.defaultof<_> let mutable toIngest = 4096 * 5 while work.TryTake(&t,fiveMs) && toIngest > 0 do @@ -606,9 +619,10 @@ module EventStoreReader = let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize match work with | Stream (name,batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Stream",name) + use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream; batch size {bs}", batchSize) try do! pullStream (conn, batchSize) name postBatch + Log.Warning("completed stream") with e -> let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) From 675375f2b82a01f4c3889c97ad07843bc56a47de Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 2 Apr 2019 15:34:34 +0100 Subject: [PATCH 031/353] Log large event stream name --- equinox-ingest/Ingest/Program.fs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index cc92da2b5..b498176e9 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -748,11 +748,10 @@ let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.StartsWith("$") || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") - || e.EventStreamId = "thor_useast2_to_backup_qa2_main" -> + || e.EventStreamId.EndsWith("_checkpoint") -> Choice2Of2 e | e when eb > Ingester.cosmosPayloadLimit -> - Log.Error("ES Event Id {eventId} size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, eb, Ingester.cosmosPayloadLimit) + Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, Ingester.cosmosPayloadLimit) Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) } From 5ee3db2439c941be7258d2dab7f60dbe9d927f7f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 3 Apr 2019 19:43:03 +0100 Subject: [PATCH 032/353] Fix names; Sync -> Ingest --- equinox-ingest/Ingest/Infrastructure.fs | 2 +- equinox-ingest/Ingest/Program.fs | 2 +- .../{Sync.Tests => }/Sync.Tests.fsproj | 2 +- .../Sync.Tests/{Sync.Tests => }/Tests.fs | 4 +-- equinox-ingest/Sync.Tests/equinox-sync.sln | 32 ------------------- equinox-ingest/equinox-ingest.sln | 30 +++++++++++++++++ 6 files changed, 35 insertions(+), 37 deletions(-) rename equinox-ingest/Sync.Tests/{Sync.Tests => }/Sync.Tests.fsproj (90%) rename equinox-ingest/Sync.Tests/{Sync.Tests => }/Tests.fs (94%) delete mode 100644 equinox-ingest/Sync.Tests/equinox-sync.sln create mode 100644 equinox-ingest/equinox-ingest.sln diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs index f289dd8f1..6c4cd7e28 100644 --- a/equinox-ingest/Ingest/Infrastructure.fs +++ b/equinox-ingest/Ingest/Infrastructure.fs @@ -1,5 +1,5 @@ [] -module private SyncTemplate.Infrastructure +module private IngestTemplate.Infrastructure open Equinox.Store // AwaitTaskCorrect open System diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index b498176e9..8680c5a07 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -1,4 +1,4 @@ -module SyncTemplate.Program +module IngestTemplate.Program open Equinox.Store // Infra open FSharp.Control diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj b/equinox-ingest/Sync.Tests/Sync.Tests.fsproj similarity index 90% rename from equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj rename to equinox-ingest/Sync.Tests/Sync.Tests.fsproj index f6b9d1e05..d5eff6ddc 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Sync.Tests.fsproj +++ b/equinox-ingest/Sync.Tests/Sync.Tests.fsproj @@ -18,7 +18,7 @@ - + diff --git a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs b/equinox-ingest/Sync.Tests/Tests.fs similarity index 94% rename from equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs rename to equinox-ingest/Sync.Tests/Tests.fs index 69ba54cd1..56689644b 100644 --- a/equinox-ingest/Sync.Tests/Sync.Tests/Tests.fs +++ b/equinox-ingest/Sync.Tests/Tests.fs @@ -1,7 +1,7 @@ -module SyncTemplate.Tests.IngesterTests +module IngestTemplate.Tests.IngesterTests open Swensen.Unquote -open SyncTemplate.Program.Ingester +open IngestTemplate.Program.Ingester open Xunit let canonicalTime = System.DateTimeOffset.UtcNow diff --git a/equinox-ingest/Sync.Tests/equinox-sync.sln b/equinox-ingest/Sync.Tests/equinox-sync.sln deleted file mode 100644 index ec7b74220..000000000 --- a/equinox-ingest/Sync.Tests/equinox-sync.sln +++ /dev/null @@ -1,32 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.28714.193 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{458FF6DD-5F7F-419D-8440-1DCD72AF5229}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "Sync\Sync.fsproj", "{EA288107-08F6-4766-9DD5-E1C961AFA918}" -EndProject -Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{B4CC54BB-AA13-47B2-BF18-C66423EAF19A}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Debug|Any CPU.Build.0 = Debug|Any CPU - {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Release|Any CPU.ActiveCfg = Release|Any CPU - {458FF6DD-5F7F-419D-8440-1DCD72AF5229}.Release|Any CPU.Build.0 = Release|Any CPU - {EA288107-08F6-4766-9DD5-E1C961AFA918}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EA288107-08F6-4766-9DD5-E1C961AFA918}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EA288107-08F6-4766-9DD5-E1C961AFA918}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EA288107-08F6-4766-9DD5-E1C961AFA918}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {A3702E33-9940-4A1D-917C-83F8F8828573} - EndGlobalSection -EndGlobal diff --git a/equinox-ingest/equinox-ingest.sln b/equinox-ingest/equinox-ingest.sln new file mode 100644 index 000000000..b1b51ccef --- /dev/null +++ b/equinox-ingest/equinox-ingest.sln @@ -0,0 +1,30 @@ +Microsoft Visual Studio Solution File, Format Version 12.00 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28714.193 +MinimumVisualStudioVersion = 10.0.40219.1 +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "Ingest\Ingest.fsproj", "{AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}" +EndProject +Global + GlobalSection(SolutionConfigurationPlatforms) = preSolution + Debug|Any CPU = Debug|Any CPU + Release|Any CPU = Release|Any CPU + EndGlobalSection + GlobalSection(ProjectConfigurationPlatforms) = postSolution + {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Release|Any CPU.Build.0 = Release|Any CPU + {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Release|Any CPU.Build.0 = Release|Any CPU + EndGlobalSection + GlobalSection(SolutionProperties) = preSolution + HideSolutionNode = FALSE + EndGlobalSection + GlobalSection(ExtensibilityGlobals) = postSolution + SolutionGuid = {8D7BF395-42EA-45B4-B353-040636AD73CE} + EndGlobalSection +EndGlobal From 83f1ab43b44c3b3240a95c0f00ff741f835dfd14 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 3 Apr 2019 22:15:31 +0100 Subject: [PATCH 033/353] Remove Sync tests as now in Sync template --- equinox-ingest/Sync.Tests/Sync.Tests.fsproj | 24 ----------- equinox-ingest/Sync.Tests/Tests.fs | 45 --------------------- equinox-ingest/equinox-ingest.sln | 6 --- 3 files changed, 75 deletions(-) delete mode 100644 equinox-ingest/Sync.Tests/Sync.Tests.fsproj delete mode 100644 equinox-ingest/Sync.Tests/Tests.fs diff --git a/equinox-ingest/Sync.Tests/Sync.Tests.fsproj b/equinox-ingest/Sync.Tests/Sync.Tests.fsproj deleted file mode 100644 index d5eff6ddc..000000000 --- a/equinox-ingest/Sync.Tests/Sync.Tests.fsproj +++ /dev/null @@ -1,24 +0,0 @@ - - - - netcoreapp2.1 - - false - - - - - - - - - - - - - - - - - - diff --git a/equinox-ingest/Sync.Tests/Tests.fs b/equinox-ingest/Sync.Tests/Tests.fs deleted file mode 100644 index 56689644b..000000000 --- a/equinox-ingest/Sync.Tests/Tests.fs +++ /dev/null @@ -1,45 +0,0 @@ -module IngestTemplate.Tests.IngesterTests - -open Swensen.Unquote -open IngestTemplate.Program.Ingester -open Xunit - -let canonicalTime = System.DateTimeOffset.UtcNow -let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } -let mergeSpans = StreamState.Span.merge - -let [] ``nothing`` () = - let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] - r =! null - -let [] ``synced`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] - r =! null - -let [] ``no overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] - r =! [| mk 0L 1; mk 2L 2 |] - -let [] ``overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 0L 2 |] - -let [] ``remove nulls`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 1L 1 |] - -let [] ``adjacent`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 0L 3 |] - -let [] ``adjacent trim`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] - r =! [| mk 1L 3 |] - -let [] ``adjacent trim append`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``mixed adjacent trim append`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2; ] - r =! [| mk 1L 3; mk 5L 1 |] \ No newline at end of file diff --git a/equinox-ingest/equinox-ingest.sln b/equinox-ingest/equinox-ingest.sln index b1b51ccef..61e2c6bef 100644 --- a/equinox-ingest/equinox-ingest.sln +++ b/equinox-ingest/equinox-ingest.sln @@ -2,8 +2,6 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.28714.193 MinimumVisualStudioVersion = 10.0.40219.1 -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}" -EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "Ingest\Ingest.fsproj", "{AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}" EndProject Global @@ -12,10 +10,6 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Debug|Any CPU.Build.0 = Debug|Any CPU - {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Release|Any CPU.ActiveCfg = Release|Any CPU - {4EB96E19-B6BA-4B62-97B9-4CB35B9270F4}.Release|Any CPU.Build.0 = Release|Any CPU {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Debug|Any CPU.Build.0 = Debug|Any CPU {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Release|Any CPU.ActiveCfg = Release|Any CPU From d22396a5165152520ff75b396f45e12b6ad2f55b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 3 Apr 2019 22:16:49 +0100 Subject: [PATCH 034/353] Fix template name --- equinox-ingest/Ingest/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 8680c5a07..31d334950 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -99,7 +99,7 @@ module CmdParser = let log = if log.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose log else Logger.SerilogNormal log GesConnector(username, password, operationTimeout, operationRetries,heartbeatTimeout=heartbeatTimeout, concurrentOperationsLimit=col, log=log, tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string]) - .Establish("ProjectorTemplate", discovery, connection) + .Establish("IngestTemplate", discovery, connection) member val Cosmos = Cosmos.Arguments(a.GetResult Cosmos) member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int @@ -770,7 +770,7 @@ let main argv = let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let ctx = - let destination = cosmos.Connect "ProjectorTemplate" |> Async.RunSynchronously + let destination = cosmos.Connect "IngestTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) Thread.Sleep(100) // https://github.com/EventStore/EventStore/issues/1899 From 0056db10f282c20e3393be0f1bc42a8910dec983 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 4 Apr 2019 10:53:47 +0100 Subject: [PATCH 035/353] Align Ingester with cleanmup in Sync --- equinox-ingest/Ingest/Program.fs | 40 +++++++++++++++++--------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 31d334950..c15a755cc 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -23,7 +23,8 @@ module CmdParser = module Cosmos = open Equinox.Cosmos - type [] Parameters = + [] + type Parameters = | [] ConnectionMode of ConnectionMode | [] Timeout of float | [] Retries of int @@ -34,34 +35,35 @@ module CmdParser = interface IArgParserTemplate with member a.Usage = match a with - | Timeout _ -> "specify operation timeout in seconds (default: 10)." - | Retries _ -> "specify operation retries (default: 0)." + | Connection _ -> "specify a connection string for a Cosmos account (default: envvar:EQUINOX_COSMOS_CONNECTION)." + | Database _ -> "specify a database name for Cosmos account (default: envvar:EQUINOX_COSMOS_DATABASE)." + | Collection _ -> "specify a collection name for Cosmos account (default: envvar:EQUINOX_COSMOS_COLLECTION)." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" - | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION, Cosmos Emulator)." | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." - | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE, test)." - | Collection _ -> "specify a collection name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_COLLECTION, test)." type Arguments(a : ParseResults) = member __.Mode = a.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) + member __.Discovery = Discovery.FromConnectionString __.Connection member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" member __.Database = match a.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" member __.Collection = match a.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" - member __.Timeout = a.GetResult(Timeout,10.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(Retries, 0) + member __.Timeout = a.GetResult(Timeout, 5.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(Retries, 1) member __.MaxRetryWaitTime = a.GetResult(RetriesWaitTime, 5) - + /// Connect with the provided parameters and/or environment variables member x.Connect /// Connection/Client identifier for logging purposes name : Async = - let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = Discovery.FromConnectionString x.Connection + let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = x.Discovery Log.Information("CosmosDb {mode} {endpointUri} Database {database} Collection {collection}.", x.Mode, endpointUri, x.Database, x.Collection) Log.Information("CosmosDb timeout {timeout}s; Throttling retries {retries}, max wait {maxRetryWaitTime}s", (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) - let connector = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) - connector.Connect(name, discovery) + let c = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) + c.Connect(name, discovery) /// To establish a local node to run against: /// 1. cinst eventstore-oss -y # where cinst is an invocation of the Chocolatey Package Installer on Windows @@ -115,7 +117,7 @@ module CmdParser = connect storeLog (heartbeatTimeout, concurrentOperationsLimit) operationThrottling discovery (__.User,__.Password) connection [] - type Arguments = + type Parameters = | [] BatchSize of int | [] MinBatchSize of int | [] Verbose @@ -145,7 +147,7 @@ module CmdParser = | Stripes _ -> "number of concurrent readers" | Tail _ -> "attempt to read from tail at specified interval in Seconds" | Es _ -> "specify EventStore parameters" - and Parameters(args : ParseResults) = + and Arguments(args : ParseResults) = member val EventStore = EventStore.Arguments(args.GetResult Es) member __.Verbose = args.Contains Verbose member __.ConsoleMinLevel = if args.Contains VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning @@ -170,10 +172,10 @@ module CmdParser = batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args - let parse argv : Parameters = + let parse argv : Arguments = let programName = System.Reflection.Assembly.GetEntryAssembly().GetName().Name - let parser = ArgumentParser.Create(programName = programName) - parser.ParseCommandLine argv |> Parameters + let parser = ArgumentParser.Create(programName = programName) + parser.ParseCommandLine argv |> Arguments // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog @@ -201,7 +203,7 @@ module Ingester = | Ok of stream: string * updatedPos: int64 | Duplicate of stream: string * updatedPos: int64 | Conflict of overage: Batch - | Exn of exn: exn * batch: Batch with + | Exn of exn: exn * batch: Batch member __.WriteTo(log: ILogger) = match __ with | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) @@ -773,7 +775,7 @@ let main argv = let destination = cosmos.Connect "IngestTemplate" |> Async.RunSynchronously let colls = CosmosCollections(cosmos.Database, cosmos.Collection) Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) - Thread.Sleep(100) // https://github.com/EventStore/EventStore/issues/1899 + Thread.Sleep(1000) // https://github.com/EventStore/EventStore/issues/1899 run ctx source readerSpec (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 From 59a632af2099f80837b98eac43e5e1c6addbe6b8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 5 Apr 2019 13:27:21 +0100 Subject: [PATCH 036/353] Add PlaceHolder EventStore module --- equinox-sync/.template.config/template.json | 8 ++++++++ equinox-sync/Sync/Program.fs | 3 +++ 2 files changed, 11 insertions(+) diff --git a/equinox-sync/.template.config/template.json b/equinox-sync/.template.config/template.json index 6dbe64212..d2f28f1fb 100644 --- a/equinox-sync/.template.config/template.json +++ b/equinox-sync/.template.config/template.json @@ -4,6 +4,7 @@ "classifications": [ "Equinox", "Event Sourcing", + "EventStore", "CosmosDb", "ChangeFeed" ], @@ -17,6 +18,13 @@ "preferNameDirectory": true, "symbols": { + "eventstore": { + "type": "parameter", + "datatype": "bool", + "isRequired": false, + "defaultValue": "false", + "description": "Use EventStore as a source; default: use a CosmosDb ChangeFeedProcessor as the source." + }, "marveleqx": { "type": "parameter", "datatype": "bool", diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 82f6fd2ad..0258a371b 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -172,6 +172,9 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments +//#if eventstore +module EventStoreSource = () +//#endif module CosmosSource = open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing From 239c53133943b82823690b8c96c2c0735ece7122 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 6 Apr 2019 00:52:28 +0100 Subject: [PATCH 037/353] Clean IngestTemplate based on Sync --- equinox-ingest/Ingest/Program.fs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index c15a755cc..3478ad640 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -78,7 +78,6 @@ module CmdParser = | [] Port of int | [] Username of string | [] Password of string - | [] ConcurrentOperationsLimit of int | [] HeartbeatTimeout of float | [] MaxItems of int | [] Cosmos of ParseResults @@ -92,29 +91,26 @@ module CmdParser = | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." - | ConcurrentOperationsLimit _ -> "max concurrent operations in flight (default: 5000)." | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." | MaxItems _ -> "maximum item count to request. Default: 4096" | Cosmos _ -> "specify CosmosDb parameters" type Arguments(a : ParseResults ) = - let connect (log: ILogger) (heartbeatTimeout, col) (operationTimeout, operationRetries) discovery (username, password) connection = - let log = if log.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose log else Logger.SerilogNormal log - GesConnector(username, password, operationTimeout, operationRetries,heartbeatTimeout=heartbeatTimeout, - concurrentOperationsLimit=col, log=log, tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string]) - .Establish("IngestTemplate", discovery, connection) member val Cosmos = Cosmos.Arguments(a.GetResult Cosmos) member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int + member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" - member __.Connect(log: ILogger, storeLog, connection) = - let (timeout, retries) as operationThrottling = a.GetResult(Timeout,20.) |> TimeSpan.FromSeconds, a.GetResult(Retries,3) - let heartbeatTimeout = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds - let concurrentOperationsLimit = a.GetResult(ConcurrentOperationsLimit,5000) - log.Information("EventStore {host} heartbeat: {heartbeat}s MaxConcurrentRequests {concurrency} Timeout: {timeout}s Retries {retries}", - __.Host, heartbeatTimeout.TotalSeconds, concurrentOperationsLimit, timeout.TotalSeconds, retries) - let discovery = match __.Port with None -> Discovery.GossipDns __.Host | Some p -> Discovery.GossipDnsCustomPort (__.Host, p) - connect storeLog (heartbeatTimeout, concurrentOperationsLimit) operationThrottling discovery (__.User,__.Password) connection + member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds + member __.Timeout = a.GetResult(SourceTimeout,20.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(SourceRetries,3) + member __.Connect(log: ILogger, storeLog, connectionStrategy) = + let s (x : TimeSpan) = s.TotalSeconds + log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s heartbeatTimeout, s timeout, retries) + let log = if storeLog.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose storeLog else Logger.SerilogNormal storeLog + let tags = ["M", Environment.MachineName; "I", Guid.NewGuid() |> string] + GesConnector(__.User,__.Password, __.Timeout, __.Retries, log, heartbeatTimeout=__.Heartbeat, tags=tags) + .Establish("IngestTemplate", __.Discovery, connectionStrategy) [] type Parameters = @@ -152,7 +148,7 @@ module CmdParser = member __.Verbose = args.Contains Verbose member __.ConsoleMinLevel = if args.Contains VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None - member __.StartingBatchSize = args.GetResult(BatchSize,4096) + member __.StartingBatchSize = args.GetResult(BatchSize,4096) member __.MinBatchSize = args.GetResult(MinBatchSize,512) member __.Stripes = args.GetResult(Stripes,1) member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None From da2cec4db6d47989641ee3df150286b8827a7173 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 6 Apr 2019 01:02:19 +0100 Subject: [PATCH 038/353] First Cut of add ES Sync --- equinox-ingest/Ingest/Program.fs | 10 +- equinox-sync/Sync/Program.fs | 426 ++++++++++++++++++++++++++++++- equinox-sync/Sync/Sync.fsproj | 1 + 3 files changed, 426 insertions(+), 11 deletions(-) diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index 3478ad640..bd529e2b8 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -102,11 +102,11 @@ module CmdParser = member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds - member __.Timeout = a.GetResult(SourceTimeout,20.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(SourceRetries,3) - member __.Connect(log: ILogger, storeLog, connectionStrategy) = - let s (x : TimeSpan) = s.TotalSeconds - log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s heartbeatTimeout, s timeout, retries) + member __.Timeout = a.GetResult(Timeout,20.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(Retries,3) + member __.Connect(log: ILogger, storeLog : ILogger, connectionStrategy) = + let s (x : TimeSpan) = x.TotalSeconds + log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s __.Heartbeat, s __.Timeout, __.Retries) let log = if storeLog.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose storeLog else Logger.SerilogNormal storeLog let tags = ["M", Environment.MachineName; "I", Guid.NewGuid() |> string] GesConnector(__.User,__.Password, __.Timeout, __.Retries, log, heartbeatTimeout=__.Heartbeat, tags=tags) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 0258a371b..168bf6684 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -3,6 +3,9 @@ open Equinox.Cosmos open Equinox.Cosmos.Core open Equinox.Cosmos.Projection +//#if eventStore +open Equinox.EventStore +//#endif open Equinox.Store open Serilog open System @@ -19,6 +22,11 @@ let every ms f = f () timer.Restart() +//#if eventStore +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start | Ignore +type ReaderSpec = { start: StartPos; streams: string list; tailInterval: TimeSpan option; stripes: int; batchSize: int; minBatchSize: int } +//#endif + module CmdParser = open Argu @@ -32,11 +40,23 @@ module CmdParser = type Parameters = | [] ConsumerGroupName of string | [] LocalSeq +#if cosmos | [] LeaseCollectionSource of string | [] LeaseCollectionDestination of string | [] LagFreqS of float | [] ChangeFeedVerbose - | [] ForceStartFromHere +#else + | [] MinBatchSize of int + | [] Stream of string + | [] All + | [] Offset of int64 + | [] Chunk of int + | [] Percent of float + | [] Stripes of int + | [] Tail of intervalS: float + | [] VerboseConsole +#endif + | [] ForceStartFromHere | [] BatchSize of int | [] Verbose | [] Source of ParseResults @@ -46,25 +66,48 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | ForceStartFromHere _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." +#if cosmos | BatchSize _ -> "maximum item count to request from feed. Default: 1000" | LeaseCollectionSource _ ->"specify Collection Name for Leases collection, within `source` connection/database (default: `source`'s `collection` + `-aux`)." | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" - | Source _ -> "CosmosDb input parameters." +#else + | BatchSize _ -> "maximum item count to request from feed. Default: 4096" + | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" + | Stream _ -> "specific stream(s) to read" + | All -> "traverse EventStore $all from Start" + | Offset _ -> "EventStore $all Stream Position to commence from" + | Chunk _ -> "EventStore $all Chunk to commence from" + | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" + | Stripes _ -> "number of concurrent readers" + | Tail _ -> "attempt to read from tail at specified interval in Seconds" + | VerboseConsole -> "request Verbose Console Logging. Default: off" +#endif | Verbose -> "request Verbose Logging. Default: off" + | Source _ -> "CosmosDb input parameters." and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.LeaseId = a.GetResult ConsumerGroupName member __.BatchSize = a.GetResult(BatchSize,1000) - member __.StartFromHere = a.Contains ForceStartFromHere + member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None +#if cosmos member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose +#else + member __.VerboseConsole = a.Contains VerboseConsole + member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning + member __.StartingBatchSize = a.GetResult(BatchSize,4096) + member __.MinBatchSize = a.GetResult(MinBatchSize,512) + member __.Stripes = a.GetResult(Stripes,1) + member __.TailInterval = match a.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None +#endif member __.Verbose = a.Contains Verbose member val Source : SourceArguments = SourceArguments(a.GetResult Source) member __.Destination : DestinationArguments = __.Source.Destination +#if cosmos member x.BuildChangeFeedParams() = let disco, db = match a.TryGetResult LeaseCollectionSource, a.TryGetResult LeaseCollectionDestination with @@ -76,7 +119,24 @@ module CmdParser = if x.StartFromHere then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) disco, db, x.LeaseId, x.StartFromHere, x.BatchSize, x.LagFrequency +#else + member x.BuildFeedParams() : ReaderSpec = + Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) + let startPos = + match a.TryGetResult Offset, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains All with + | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some c, _, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c + | _, _, Some p, _ -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | None, None, None, true -> Log.Warning "Processing will commence at $all Start"; Start + | None, None, None, false ->Log.Warning "No $all processing requested"; Ignore + match x.TailInterval with + | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) + | None -> Log.Warning "Not following tail" + { start = startPos; streams = a.GetResults Stream; tailInterval = x.TailInterval + batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } +#endif and [] SourceParameters = +#if cosmos | [] SourceConnectionMode of Equinox.Cosmos.ConnectionMode | [] SourceTimeout of float | [] SourceRetries of int @@ -84,12 +144,23 @@ module CmdParser = | [] SourceConnection of string | [] SourceDatabase of string | [] SourceCollection of string +#else + | [] VerboseStore + | [] SourceTimeout of float + | [] SourceRetries of int + | [] Host of string + | [] Port of int + | [] Username of string + | [] Password of string + | [] HeartbeatTimeout of float +#endif | [] CategoryBlacklist of string | [] CategoryWhitelist of string | [] Cosmos of ParseResults interface IArgParserTemplate with member a.Usage = match a with +#if cosmos | SourceConnection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION)." | SourceDatabase _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE)." | SourceCollection _ -> "specify a collection name within `SourceDatabase`." @@ -97,6 +168,16 @@ module CmdParser = | SourceRetries _ -> "specify operation retries (default: 1)." | SourceRetriesWaitTime _ ->"specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" | SourceConnectionMode _ -> "override the connection mode (default: DirectTcp)." +#else + | VerboseStore -> "Include low level Store logging." + | SourceTimeout _ -> "specify operation timeout in seconds (default: 20)." + | SourceRetries _ -> "specify operation retries (default: 3)." + | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." + | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." + | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." + | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." + | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." +#endif | CategoryBlacklist _ -> "Category whitelist" | CategoryWhitelist _ -> "Category blacklist" | Cosmos _ -> "CosmosDb destination parameters." @@ -108,6 +189,7 @@ module CmdParser = | bad, [] -> let black = Set.ofList bad in Log.Information("Excluding categories: {cats}", black); fun x -> not (black.Contains x) | [], good -> let white = Set.ofList good in Log.Information("Only copying categories: {cats}", white); fun x -> white.Contains x | _, _ -> raise (InvalidArguments "BlackList and Whitelist are mutually exclusive; inclusions and exclusions cannot be mixed") +#if cosmos member __.Mode = a.GetResult(SourceConnectionMode, Equinox.Cosmos.ConnectionMode.DirectTcp) member __.Discovery = Discovery.FromConnectionString __.Connection member __.Connection = match a.TryGetResult SourceConnection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" @@ -125,6 +207,24 @@ module CmdParser = (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) let c = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) discovery, { database = x.Database; collection = x.Collection }, c.ConnectionPolicy, x.CategoryFilterFunction +#else + member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" + member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int + member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host + member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" + member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" + member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds + member __.Timeout = a.GetResult(SourceTimeout,20.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(SourceRetries,3) + member __.Connect(log: ILogger, storeLog: ILogger, connectionStrategy) = + let s (x : TimeSpan) = x.TotalSeconds + log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s __.Heartbeat, s __.Timeout, __.Retries) + let log=if storeLog.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose storeLog else Logger.SerilogNormal storeLog + let tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string] + let catFilter = __.CategoryFilterFunction + GesConnector(__.User, __.Password, __.Timeout, __.Retries, log=log, heartbeatTimeout=__.Heartbeat, tags=tags) + .Establish("SyncTemplate", __.Discovery, connectionStrategy) |> Async.RunSynchronously, catFilter +#endif and [] DestinationParameters = | [] Connection of string | [] Database of string @@ -172,9 +272,315 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -//#if eventstore -module EventStoreSource = () -//#endif +#if !cosmos +type EventStore.ClientAPI.RecordedEvent with + member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) + +module EventStoreSource = + open EventStore.ClientAPI + + let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length + let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata + let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 + let mb x = float x / 1024. / 1024. + + type SliceStatsBuffer(?interval) = + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() + member __.Ingest(slice: AllEventsSlice) = + lock recentCats <| fun () -> + let mutable batchBytes = 0 + for x in slice.Events do + let cat = category x.OriginalStreamId + let eventBytes = esPayloadBytes x + match recentCats.TryGetValue cat with + | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) + | false, _ -> recentCats.[cat] <- (1, eventBytes) + batchBytes <- batchBytes + eventBytes + __.DumpIfIntervalExpired() + slice.Events.Length, int64 batchBytes + member __.DumpIfIntervalExpired(?force) = + if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then + lock recentCats <| fun () -> + let log = function + | [||] -> () + | xs -> + xs + |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) + |> Seq.truncate 10 + |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) + |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log + recentCats.Clear() + accStart.Restart() + + type OverallStats(?statsInterval) = + let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() + let mutable totalEvents, totalBytes = 0L, 0L + member __.Ingest(batchEvents, batchBytes) = + Interlocked.Add(&totalEvents,batchEvents) |> ignore + Interlocked.Add(&totalBytes,batchBytes) |> ignore + member __.Bytes = totalBytes + member __.Events = totalEvents + member __.DumpIfIntervalExpired(?force) = + if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then + let totalMb = mb totalBytes + Log.Warning("Traversed {events} events {gb:n1}GB {mbs:n2}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + progressStart.Restart() + + type Range(start, sliceEnd : Position option, max : Position) = + member val Current = start with get, set + member __.TryNext(pos: Position) = + __.Current <- pos + __.IsCompleted + member __.IsCompleted = + match sliceEnd with + | Some send when __.Current.CommitPosition >= send.CommitPosition -> false + | _ -> true + member __.PositionAsRangePercentage = + if max.CommitPosition=0L then Double.NaN + else float __.Current.CommitPosition/float max.CommitPosition + + // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk + let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 + let posFromChunk (chunk: int) = + let chunkBase = int64 chunk * 1024L * 1024L * 256L + Position(chunkBase,0L) + let posFromPercentage (pct,max : Position) = + let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) + let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L + let posFromChunkAfter (pos: Position) = + let nextChunk = 1 + int (chunk pos) + posFromChunk nextChunk + + let fetchMax (conn : IEventStoreConnection) = async { + let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect + let max = lastItemBatch.NextPosition + Log.Warning("EventStore {chunks} chunks, ~{gb:n1}GB Write Position @ {pos} ", chunk max, mb max.CommitPosition/1024., max.CommitPosition) + return max } + let establishMax (conn : IEventStoreConnection) = async { + let mutable max = None + while Option.isNone max do + try let! max_ = fetchMax conn + max <- Some max_ + with e -> + Log.Warning(e,"Could not establish max position") + do! Async.Sleep 5000 + return Option.get max } + let pullStream (conn : IEventStoreConnection, batchSize) stream (postBatch : CosmosIngester.Batch -> unit) = + let rec fetchFrom pos = async { + let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect + if currentSlice.IsEndOfStream then return () else + let events = + [| for x in currentSlice.Events -> + let e = x.Event + Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] + postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } + return! fetchFrom currentSlice.NextEventNumber } + fetchFrom 0L + + type [] PullResult = Exn of exn: exn | Eof | EndOfTranche + type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : CosmosIngester.Batch -> unit) = + member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?ignoreEmptyEof) = + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let rec loop () = async { + let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let postSw = Stopwatch.StartNew() + let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) + let streams = + enumEvents currentSlice.Events + |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) + |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) + |> Array.ofSeq + let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length + let mutable usedEvents = 0 + for stream,streamEvents in streams do + for pos, item in streamEvents do + usedEvents <- usedEvents + 1 + postBatch { stream = stream; span = { index = pos; events = [| item |]}} + if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( + Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, + batchEvents, usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds) + let shouldLoop = range.TryNext currentSlice.NextPosition + if shouldLoop && not currentSlice.IsEndOfStream then + sw.Restart() // restart the clock as we hand off back to the Reader + return! loop () + else + return currentSlice.IsEndOfStream } + async { + try let! eof = loop () + return if eof then Eof else EndOfTranche + with e -> return Exn e } + + type [] Work = + | Stream of name: string * batchSize: int + | Tranche of range: Range * batchSize : int + | Tail of pos: Position * interval: TimeSpan * batchSize : int + type FeedQueue(batchSize, minBatchSize, max, ?statsInterval) = + let work = System.Collections.Concurrent.ConcurrentQueue() + member val OverallStats = OverallStats(?statsInterval=statsInterval) + member val SlicesStats = SliceStatsBuffer() + member __.AddTranche(range, ?batchSizeOverride) = + work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) + member __.AddTranche(pos, nextPos, ?batchSizeOverride) = + __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) + member __.AddStream(name, ?batchSizeOverride) = + work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + member __.AddTail(pos, interval, ?batchSizeOverride) = + work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) + member __.TryDequeue () = + work.TryDequeue() + member __.Process(conn, enumEvents, postBatch, work) = async { + let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize + match work with + | Stream (name,batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) + Log.Warning("Reading stream; batch size {bs}", batchSize) + try do! pullStream (conn, batchSize) name postBatch + Log.Warning("completed stream") + with e -> + let bs = adjust batchSize + Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + __.AddStream(name, bs) + return false + | Tranche (range, batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) + Log.Warning("Commencing tranche, batch size {bs}", batchSize) + let reader = ReaderGroup(conn, enumEvents, postBatch) + let! res = reader.Pump(range, batchSize, __.SlicesStats, __.OverallStats) + match res with + | PullResult.EndOfTranche -> + Log.Warning("Completed tranche") + __.OverallStats.DumpIfIntervalExpired() + return false + | PullResult.Eof -> + Log.Warning("REACHED THE END!") + __.OverallStats.DumpIfIntervalExpired(true) + return true + | PullResult.Exn e -> + let bs = adjust batchSize + Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) + __.OverallStats.DumpIfIntervalExpired() + __.AddTranche(range, bs) + return false + | Tail (pos, interval, batchSize) -> + let mutable first, count, batchSize, range = true, 0, batchSize, Range(pos,None, Position.Start) + let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) + let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds + let progressSw, tailSw = Stopwatch.StartNew(), Stopwatch.StartNew() + let reader = ReaderGroup(conn, enumEvents, postBatch) + let slicesStats, stats = SliceStatsBuffer(), OverallStats() + while true do + let currentPos = range.Current + use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") + if first then + first <- false + Log.Warning("Tailing at {interval}s interval", interval.TotalSeconds) + elif progressSw.ElapsedMilliseconds > progressIntervalMs then + Log.Warning("Performed {count} tails to date @ {pos} chunk {chunk}", count, currentPos.CommitPosition, chunk currentPos) + progressSw.Restart() + count <- count + 1 + let! res = reader.Pump(range,batchSize,slicesStats,stats,ignoreEmptyEof=true) + stats.DumpIfIntervalExpired() + match tailIntervalMs - tailSw.ElapsedMilliseconds with + | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) + | _ -> () + tailSw.Restart() + match res with + | PullResult.EndOfTranche | PullResult.Eof -> () + | PullResult.Exn e -> + batchSize <- adjust batchSize + Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) + return true } + + type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : CosmosIngester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = + let work = FeedQueue(spec.batchSize, spec.minBatchSize, max, ?statsInterval=statsInterval) + do match spec.tailInterval with + | Some interval -> work.AddTail(max, interval) + | None -> () + for s in spec.streams do + work.AddStream s + let mutable remainder = + let startPos = + match spec.start with + | StartPos.Start -> Position.Start + | Absolute p -> Position(p, 0L) + | Chunk c -> posFromChunk c + | Percentage pct -> posFromPercentage (pct, max) + | Ignore -> max + Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", + startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) + if spec.start = Ignore then None + else + let nextPos = posFromChunkAfter startPos + work.AddTranche(startPos, nextPos) + Some nextPos + + member __.Pump () = async { + (*if spec.tail then enqueue tail work*) + let maxDop = spec.stripes + Option.count spec.tailInterval + let dop = new SemaphoreSlim(maxDop) + let mutable finished = false + while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do + let! _ = dop.Await() + work.OverallStats.DumpIfIntervalExpired() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let! eof = work.Process(conn, enumEvents, postBatch, task) + if eof then remainder <- None + finally dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + match remainder with + | Some pos -> + let nextPos = posFromChunkAfter pos + remainder <- Some nextPos + do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) + | None -> + if finished then do! Async.Sleep 1000 + else Log.Warning("No further ingestion work to commence") + finished <- true } + + let start log (leaseId, startFromHere, batchSize) (conn, spec, enumEvents, cosmosContext) = async { + let! ct = Async.CancellationToken + let! max = establishMax conn + let writer = CosmosIngester.SynchronousWriter(cosmosContext, log) + let reader = Reader(conn, spec, enumEvents, writer.Add, max, ct) + let! _ = Async.StartChild <| reader.Pump() + return () + } + + //let run (ctx : Equinox.Cosmos.Core.CosmosContext) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { + // let! ingester = EventStoreSource.start(source.ReadConnection, writerQueueLen, writerCount, readerQueueLen) + // let! _feeder = EventStoreSource.start(source.ReadConnection, spec, enumEvents, ingester.Add) + // do! Async.AwaitKeyboardInterrupt() } +let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { + for e in xs -> + let eb = EventStoreSource.esPayloadBytes e + match e.Event with + | e when not e.IsJson + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.StartsWith("$") + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || not (catFilter e.EventStreamId) -> + Choice2Of2 e + | e when eb > EventStoreSource.cosmosPayloadLimit -> + Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", + e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, EventStoreSource.cosmosPayloadLimit) + Choice2Of2 e + | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) +} +#else module CosmosSource = open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing @@ -415,6 +821,7 @@ module CosmosSource = // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level yield { stream = e.Stream; span = { index = e.Index; events = [| e |] } } } //#endif +#endif // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog @@ -445,6 +852,7 @@ let main argv = let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.ForContext()) +#if cosmos let log = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() let auxDiscovery, aux, leaseId, startFromHere, batchSize, lagFrequency = args.BuildChangeFeedParams() @@ -458,6 +866,12 @@ let main argv = CosmosSource.run (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, batchSize, lagFrequency) createSyncHandler +#else + let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint + let esConnection, catFilter = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) + let spec = args.BuildFeedParams() + EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter, target) +#endif |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 8f654d456..60efad412 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -17,6 +17,7 @@ + From ff0c8327f26e8e4b86a77929d5ce0a6034546a2e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 10 Apr 2019 11:50:57 +0100 Subject: [PATCH 039/353] First cut EventStore tailing and progress writing --- equinox-sync/Sync/Program.fs | 281 +++++++++++++++++++++-------------- 1 file changed, 166 insertions(+), 115 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 168bf6684..56a2869e2 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -13,6 +13,24 @@ open System.Collections.Generic open System.Diagnostics open System.Threading +//#if eventStore +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Tail +type ReaderSpec = + { /// Identifier for this projection and it's state + groupName: string + /// Start position from which forward reading is to commence // Assuming no stored position + start: StartPos + /// Additional streams with which to seed the reading + streams: string list + /// Delay when reading yields an empty batch + tailInterval: TimeSpan + /// Maximum number of stream readers to permit + stripes: int + /// Initial batch size to use when commencing reading + batchSize: int + /// Smallest batch size to degrade to in the presence of failures + minBatchSize: int } +//#endif let mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] let every ms f = @@ -22,10 +40,6 @@ let every ms f = f () timer.Restart() -//#if eventStore -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start | Ignore -type ReaderSpec = { start: StartPos; streams: string list; tailInterval: TimeSpan option; stripes: int; batchSize: int; minBatchSize: int } -//#endif module CmdParser = open Argu @@ -48,7 +62,6 @@ module CmdParser = #else | [] MinBatchSize of int | [] Stream of string - | [] All | [] Offset of int64 | [] Chunk of int | [] Percent of float @@ -76,31 +89,31 @@ module CmdParser = | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" | Stream _ -> "specific stream(s) to read" - | All -> "traverse EventStore $all from Start" | Offset _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Stripes _ -> "number of concurrent readers" - | Tail _ -> "attempt to read from tail at specified interval in Seconds" + | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | VerboseConsole -> "request Verbose Console Logging. Default: off" #endif | Verbose -> "request Verbose Logging. Default: off" | Source _ -> "CosmosDb input parameters." and Arguments(a : ParseResults) = - member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None - member __.LeaseId = a.GetResult ConsumerGroupName member __.BatchSize = a.GetResult(BatchSize,1000) member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None #if cosmos + member __.LeaseId = a.GetResult ConsumerGroupName + member __.StartFromHere = a.Contains ForceStartFromHere member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose #else + member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.VerboseConsole = a.Contains VerboseConsole member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.Stripes = a.GetResult(Stripes,1) - member __.TailInterval = match a.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None + member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds #endif member __.Verbose = a.Contains Verbose @@ -123,16 +136,15 @@ module CmdParser = member x.BuildFeedParams() : ReaderSpec = Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = - match a.TryGetResult Offset, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains All with - | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some c, _, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c - | _, _, Some p, _ -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p - | None, None, None, true -> Log.Warning "Processing will commence at $all Start"; Start - | None, None, None, false ->Log.Warning "No $all processing requested"; Ignore - match x.TailInterval with - | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) - | None -> Log.Warning "Not following tail" - { start = startPos; streams = a.GetResults Stream; tailInterval = x.TailInterval + match a.TryGetResult Offset, a.TryGetResult Chunk, a.TryGetResult Percent with + | Some p, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some c, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c + | _, _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | None, None, None -> Log.Warning "Processing will commence at $all Tail"; StartPos.Tail + Log.Information("Processing ConsumerGroupName {groupName} in Database {db} Collection {coll} in batches of {batchSize}", + x.ConsumerGroupName, x.Destination.Database, x.Destination.Collection, x.BatchSize) + Log.Warning("Following tail at {seconds}s interval", let i = x.TailInterval in i.TotalSeconds) + { groupName = x.ConsumerGroupName; start = startPos; streams = a.GetResults Stream; tailInterval = x.TailInterval batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } #endif and [] SourceParameters = @@ -352,9 +364,6 @@ module EventStoreSource = let posFromPercentage (pct,max : Position) = let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L - let posFromChunkAfter (pos: Position) = - let nextChunk = 1 + int (chunk pos) - posFromChunk nextChunk let fetchMax (conn : IEventStoreConnection) = async { let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -383,7 +392,7 @@ module EventStoreSource = fetchFrom 0L type [] PullResult = Exn of exn: exn | Eof | EndOfTranche - type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : CosmosIngester.Batch -> unit) = + type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : Position -> CosmosIngester.Batch[] -> unit) = member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?ignoreEmptyEof) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { @@ -398,15 +407,15 @@ module EventStoreSource = |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - let mutable usedEvents = 0 - for stream,streamEvents in streams do - for pos, item in streamEvents do - usedEvents <- usedEvents + 1 - postBatch { stream = stream; span = { index = pos; events = [| item |]}} + let events : CosmosIngester.Batch[] = + [| for stream,streamEvents in streams do + for pos, item in streamEvents do + yield { stream = stream; span = { index = pos; events = [| item |]}} |] + postBatch currentSlice.NextPosition events if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds) + batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) let shouldLoop = range.TryNext currentSlice.NextPosition if shouldLoop && not currentSlice.IsEndOfStream then sw.Restart() // restart the clock as we hand off back to the Reader @@ -420,149 +429,190 @@ module EventStoreSource = type [] Work = | Stream of name: string * batchSize: int - | Tranche of range: Range * batchSize : int | Tail of pos: Position * interval: TimeSpan * batchSize : int - type FeedQueue(batchSize, minBatchSize, max, ?statsInterval) = + type FeedQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) member val SlicesStats = SliceStatsBuffer() - member __.AddTranche(range, ?batchSizeOverride) = - work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) - member __.AddTranche(pos, nextPos, ?batchSizeOverride) = - __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) member __.AddStream(name, ?batchSizeOverride) = work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) member __.AddTail(pos, interval, ?batchSizeOverride) = work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() - member __.Process(conn, enumEvents, postBatch, work) = async { + member __.Process(conn, enumEvents, postItem, shouldTail, postTail, work) = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize match work with | Stream (name,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream; batch size {bs}", batchSize) - try do! pullStream (conn, batchSize) name postBatch + try do! pullStream (conn, batchSize) name postItem Log.Warning("completed stream") with e -> let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) __.AddStream(name, bs) return false - | Tranche (range, batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) - Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let reader = ReaderGroup(conn, enumEvents, postBatch) - let! res = reader.Pump(range, batchSize, __.SlicesStats, __.OverallStats) - match res with - | PullResult.EndOfTranche -> - Log.Warning("Completed tranche") - __.OverallStats.DumpIfIntervalExpired() - return false - | PullResult.Eof -> - Log.Warning("REACHED THE END!") - __.OverallStats.DumpIfIntervalExpired(true) - return true - | PullResult.Exn e -> - let bs = adjust batchSize - Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) - __.OverallStats.DumpIfIntervalExpired() - __.AddTranche(range, bs) - return false | Tail (pos, interval, batchSize) -> - let mutable first, count, batchSize, range = true, 0, batchSize, Range(pos,None, Position.Start) + let mutable first, count, pauses, batchSize, range = true, 0, 0, batchSize, Range(pos,None, Position.Start) let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds let progressSw, tailSw = Stopwatch.StartNew(), Stopwatch.StartNew() - let reader = ReaderGroup(conn, enumEvents, postBatch) + let awaitInterval = async { + match tailIntervalMs - tailSw.ElapsedMilliseconds with + | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) + | _ -> () + tailSw.Restart() } + let reader = ReaderGroup(conn, enumEvents, postTail) let slicesStats, stats = SliceStatsBuffer(), OverallStats() + use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") while true do let currentPos = range.Current - use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") if first then first <- false Log.Warning("Tailing at {interval}s interval", interval.TotalSeconds) elif progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Warning("Performed {count} tails to date @ {pos} chunk {chunk}", count, currentPos.CommitPosition, chunk currentPos) + Log.Warning("Performed {count} tails ({pauses} pauses) to date @ {pos} chunk {chunk}", count, pauses, currentPos.CommitPosition, chunk currentPos) progressSw.Restart() count <- count + 1 - let! res = reader.Pump(range,batchSize,slicesStats,stats,ignoreEmptyEof=true) + if shouldTail () then + let! res = reader.Pump(range,batchSize,slicesStats,stats,ignoreEmptyEof=true) + do! awaitInterval + match res with + | PullResult.EndOfTranche | PullResult.Eof -> () + | PullResult.Exn e -> + batchSize <- adjust batchSize + Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) + else + pauses <- pauses + 1 + do! awaitInterval stats.DumpIfIntervalExpired() - match tailIntervalMs - tailSw.ElapsedMilliseconds with - | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) - | _ -> () - tailSw.Restart() - match res with - | PullResult.EndOfTranche | PullResult.Eof -> () - | PullResult.Exn e -> - batchSize <- adjust batchSize - Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) return true } - type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : CosmosIngester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = - let work = FeedQueue(spec.batchSize, spec.minBatchSize, max, ?statsInterval=statsInterval) - do match spec.tailInterval with - | Some interval -> work.AddTail(max, interval) - | None -> () + type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, max, ?statsInterval) = + let work = FeedQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) + do work.AddTail(max, spec.tailInterval) for s in spec.streams do work.AddStream s - let mutable remainder = let startPos = match spec.start with - | StartPos.Start -> Position.Start + | StartPos.Tail -> max | Absolute p -> Position(p, 0L) | Chunk c -> posFromChunk c | Percentage pct -> posFromPercentage (pct, max) - | Ignore -> max Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) - if spec.start = Ignore then None - else - let nextPos = posFromChunkAfter startPos - work.AddTranche(startPos, nextPos) - Some nextPos - member __.Pump () = async { - (*if spec.tail then enqueue tail work*) - let maxDop = spec.stripes + Option.count spec.tailInterval + member __.Pump(postItem, shouldTail, postTail) = async { + let maxDop = spec.stripes + 1 let dop = new SemaphoreSlim(maxDop) let mutable finished = false - while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do let! _ = dop.Await() work.OverallStats.DumpIfIntervalExpired() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - try let! eof = work.Process(conn, enumEvents, postBatch, task) - if eof then remainder <- None + try let! _ = work.Process(conn, enumEvents, postItem, shouldTail, postTail, task) in () finally dop.Release() |> ignore } return () } match work.TryDequeue() with | true, task -> do! forkRunRelease task + | false, _ when not finished-> + Log.Warning("No further ingestion work to commence") + finished <- true + | _ -> () } + + type [] CoordinationWork<'Pos> = + | Result of CosmosIngester.Writer.Result + | Unbatched of CosmosIngester.Batch + | BatchWithTracking of 'Pos * CosmosIngester.Batch[] + + let every ms f = + let timer = Stopwatch.StartNew() + fun () -> + if timer.ElapsedMilliseconds > ms then + f () + timer.Restart() + + type Coordinator(log : Serilog.ILogger, reader : Reader, cosmosContext, ?maxWriters, ?interval) = + let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 + let sleepIntervalMs = 100 + let work = System.Collections.Concurrent.ConcurrentQueue() + let buffer = CosmosIngester.Queue.StreamStates() + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, defaultArg maxWriters 32) + let tailSyncState = Progress.State() + let pumpReaders = + let postWrite = work.Enqueue << CoordinationWork.Unbatched + let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) + // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here + let shouldTailNow () = let _, pendingBatchCount = tailSyncState.Validate(fun _ -> None) in pendingBatchCount < 10 // TODO remove 10 + reader.Pump(postWrite, shouldTailNow, postBatch) + let postWriteResult = work.Enqueue << CoordinationWork.Result + + member __.Pump () = async { + use _ = writers.Result.Subscribe postWriteResult + let! _ = Async.StartChild pumpReaders + let! _ = Async.StartChild <| writers.Pump() + let! ct = Async.CancellationToken + let mutable bytesPended = 0L + let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 + let mutable rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let dumpStats () = + log.Warning("Writer Exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed",!rateLimited, !timedOut, !malformed) + rateLimited := 0; timedOut := 0; malformed := 0 + Log.Warning("Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", + !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) + ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 + buffer.Dump log + let tryDumpStats = every statsIntervalMs dumpStats + let handle = function + | CoordinationWork.Unbatched item -> + buffer.Add item |> ignore + | CoordinationWork.BatchWithTracking(pos, items) -> + for item in items do + buffer.Add item |> ignore + tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) + | CoordinationWork.Result res -> + incr resultsHandled + let (stream, updatedState), kind = buffer.HandleWriteResult res + match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) + res.WriteTo log + match kind with + | CosmosIngester.Queue.Ok -> res.WriteTo log + | CosmosIngester.Queue.RateLimited -> incr rateLimited + | CosmosIngester.Queue.TimedOut -> incr timedOut + | CosmosIngester.Queue.Malformed -> incr malformed + let queueWrite (w : CosmosIngester.Batch) = + incr workPended + eventsPended := !eventsPended + w.span.events.Length + bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) + writers.Enqueue w + while not ct.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write results to buffer + Progress + match work.TryDequeue() with + | true, item -> + handle item | false, _ -> - match remainder with - | Some pos -> - let nextPos = posFromChunkAfter pos - remainder <- Some nextPos - do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) - | None -> - if finished then do! Async.Sleep 1000 - else Log.Warning("No further ingestion work to commence") - finished <- true } - - let start log (leaseId, startFromHere, batchSize) (conn, spec, enumEvents, cosmosContext) = async { - let! ct = Async.CancellationToken + // 2. After that, [over] provision writers queue + let mutable more = writers.HasCapacity + while more do + match buffer.TryReady() with + | Some w -> queueWrite w; more <- writers.HasCapacity + | None -> (); more <- false + // 3. Periodically emit status info + tryDumpStats () + // TODO trigger periodic progress writing + // 5. Sleep if + do! Async.Sleep sleepIntervalMs } + + let start (log : Serilog.ILogger) (conn, spec, enumEvents) (maxWriters, cosmosContext) = async { let! max = establishMax conn - let writer = CosmosIngester.SynchronousWriter(cosmosContext, log) - let reader = Reader(conn, spec, enumEvents, writer.Add, max, ct) - let! _ = Async.StartChild <| reader.Pump() - return () - } - - //let run (ctx : Equinox.Cosmos.Core.CosmosContext) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { - // let! ingester = EventStoreSource.start(source.ReadConnection, writerQueueLen, writerCount, readerQueueLen) - // let! _feeder = EventStoreSource.start(source.ReadConnection, spec, enumEvents, ingester.Add) - // do! Async.AwaitKeyboardInterrupt() } + let reader = Reader(conn, spec, enumEvents, max) + let coordinator = Coordinator(log, reader, cosmosContext, maxWriters) + do! coordinator.Pump () } + let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { for e in xs -> let eb = EventStoreSource.esPayloadBytes e @@ -580,7 +630,8 @@ let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) } -#else + +//#else module CosmosSource = open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing @@ -870,7 +921,7 @@ let main argv = let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let esConnection, catFilter = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) let spec = args.BuildFeedParams() - EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter, target) + EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter) (256, target) #endif |> Async.RunSynchronously 0 From e4471f277ea53ace716768a9e59d6001cac6f22d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 10 Apr 2019 16:06:55 +0100 Subject: [PATCH 040/353] Bug fixes etc --- equinox-sync/Sync/Program.fs | 98 ++++++++++++++++++++---------------- 1 file changed, 54 insertions(+), 44 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 56a2869e2..2338d0f34 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -69,7 +69,7 @@ module CmdParser = | [] Tail of intervalS: float | [] VerboseConsole #endif - | [] ForceStartFromHere + | [] ForceStartFromHere | [] BatchSize of int | [] Verbose | [] Source of ParseResults @@ -85,6 +85,7 @@ module CmdParser = | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" + | Source _ -> "CosmosDb input parameters." #else | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" @@ -95,14 +96,14 @@ module CmdParser = | Stripes _ -> "number of concurrent readers" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | VerboseConsole -> "request Verbose Console Logging. Default: off" + | Source _ -> "EventStore input parameters." #endif | Verbose -> "request Verbose Logging. Default: off" - | Source _ -> "CosmosDb input parameters." and Arguments(a : ParseResults) = - member __.BatchSize = a.GetResult(BatchSize,1000) member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None #if cosmos member __.LeaseId = a.GetResult ConsumerGroupName + member __.BatchSize = a.GetResult(BatchSize,1000) member __.StartFromHere = a.Contains ForceStartFromHere member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose @@ -137,13 +138,14 @@ module CmdParser = Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = match a.TryGetResult Offset, a.TryGetResult Chunk, a.TryGetResult Percent with - | Some p, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some c, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c - | _, _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p - | None, None, None -> Log.Warning "Processing will commence at $all Tail"; StartPos.Tail - Log.Information("Processing ConsumerGroupName {groupName} in Database {db} Collection {coll} in batches of {batchSize}", - x.ConsumerGroupName, x.Destination.Database, x.Destination.Collection, x.BatchSize) - Log.Warning("Following tail at {seconds}s interval", let i = x.TailInterval in i.TotalSeconds) + | Some p, _, _ -> Absolute p + | _, Some c, _ -> StartPos.Chunk c + | _, _, Some p -> Percentage p + | None, None, None -> StartPos.Tail + Log.Information("Syncing Consumer Group {groupName} in Database {db} Collection {coll}", + x.ConsumerGroupName, x.Destination.Database, x.Destination.Collection) + Log.Information("Ingesting from {startPos} in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", + startPos, x.MinBatchSize, x.StartingBatchSize, x.Stripes) { groupName = x.ConsumerGroupName; start = startPos; streams = a.GetResults Stream; tailInterval = x.TailInterval batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } #endif @@ -233,9 +235,8 @@ module CmdParser = log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s __.Heartbeat, s __.Timeout, __.Retries) let log=if storeLog.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose storeLog else Logger.SerilogNormal storeLog let tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string] - let catFilter = __.CategoryFilterFunction GesConnector(__.User, __.Password, __.Timeout, __.Retries, log=log, heartbeatTimeout=__.Heartbeat, tags=tags) - .Establish("SyncTemplate", __.Discovery, connectionStrategy) |> Async.RunSynchronously, catFilter + .Establish("SyncTemplate", __.Discovery, connectionStrategy) |> Async.RunSynchronously #endif and [] DestinationParameters = | [] Connection of string @@ -322,7 +323,7 @@ module EventStoreSource = |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) |> Seq.truncate 10 |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) - |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) + |> fun rendered -> Log.Information("EventStore categories {@cats} (MB/cat/count)", rendered) recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log recentCats.Clear() @@ -340,10 +341,11 @@ module EventStoreSource = member __.DumpIfIntervalExpired(?force) = if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then let totalMb = mb totalBytes - Log.Warning("Traversed {events} events {gb:n1}GB {mbs:n2}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + Log.Information("EventStore throughput {events} events {gb:n1}GB {mbs:n2}MB/s", + totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) progressStart.Restart() - type Range(start, sliceEnd : Position option, max : Position) = + type Range(start, sliceEnd : Position option, ?max : Position) = member val Current = start with get, set member __.TryNext(pos: Position) = __.Current <- pos @@ -353,8 +355,9 @@ module EventStoreSource = | Some send when __.Current.CommitPosition >= send.CommitPosition -> false | _ -> true member __.PositionAsRangePercentage = - if max.CommitPosition=0L then Double.NaN - else float __.Current.CommitPosition/float max.CommitPosition + match max with + | None -> Double.NaN + | Some max -> float __.Current.CommitPosition/float max.CommitPosition // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 @@ -367,14 +370,14 @@ module EventStoreSource = let fetchMax (conn : IEventStoreConnection) = async { let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let max = lastItemBatch.NextPosition - Log.Warning("EventStore {chunks} chunks, ~{gb:n1}GB Write Position @ {pos} ", chunk max, mb max.CommitPosition/1024., max.CommitPosition) + let max = lastItemBatch.FromPosition + Log.Information("EventStore Write @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) return max } let establishMax (conn : IEventStoreConnection) = async { let mutable max = None while Option.isNone max do - try let! max_ = fetchMax conn - max <- Some max_ + try let! currentMax = fetchMax conn + max <- Some currentMax with e -> Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 @@ -413,7 +416,7 @@ module EventStoreSource = yield { stream = stream; span = { index = pos; events = [| item |]}} |] postBatch currentSlice.NextPosition events if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( - Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) let shouldLoop = range.TryNext currentSlice.NextPosition @@ -454,10 +457,10 @@ module EventStoreSource = __.AddStream(name, bs) return false | Tail (pos, interval, batchSize) -> - let mutable first, count, pauses, batchSize, range = true, 0, 0, batchSize, Range(pos,None, Position.Start) + let mutable count, pauses, batchSize, range = 0, 0, batchSize, Range(pos, None) let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds - let progressSw, tailSw = Stopwatch.StartNew(), Stopwatch.StartNew() + let tailSw = Stopwatch.StartNew() let awaitInterval = async { match tailIntervalMs - tailSw.ElapsedMilliseconds with | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) @@ -466,13 +469,12 @@ module EventStoreSource = let reader = ReaderGroup(conn, enumEvents, postTail) let slicesStats, stats = SliceStatsBuffer(), OverallStats() use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") + let progressSw = Stopwatch.StartNew() while true do let currentPos = range.Current - if first then - first <- false - Log.Warning("Tailing at {interval}s interval", interval.TotalSeconds) - elif progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Warning("Performed {count} tails ({pauses} pauses) to date @ {pos} chunk {chunk}", count, pauses, currentPos.CommitPosition, chunk currentPos) + if progressSw.ElapsedMilliseconds > progressIntervalMs then + Log.Information("Tailed {count} times ({pauses} pauses @ {pos} (chunk {chunk})", + count, pauses, currentPos.CommitPosition, chunk currentPos) progressSw.Restart() count <- count + 1 if shouldTail () then @@ -491,17 +493,21 @@ module EventStoreSource = type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, max, ?statsInterval) = let work = FeedQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) - do work.AddTail(max, spec.tailInterval) - for s in spec.streams do - work.AddStream s - let startPos = + do let startPos = match spec.start with | StartPos.Tail -> max | Absolute p -> Position(p, 0L) | Chunk c -> posFromChunk c | Percentage pct -> posFromPercentage (pct, max) - Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", - startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) + work.AddTail(startPos, spec.tailInterval) + match spec.streams with + | [] -> () + | streams -> + Log.Information("EventStore Additional Streams {streams}", streams) + for s in streams do + work.AddStream s + Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", + startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) member __.Pump(postItem, shouldTail, postTail) = async { let maxDop = spec.stripes + 1 @@ -520,7 +526,7 @@ module EventStoreSource = | true, task -> do! forkRunRelease task | false, _ when not finished-> - Log.Warning("No further ingestion work to commence") + if spec.streams <> [] then Log.Information("Initial streams seeded") finished <- true | _ -> () } @@ -557,14 +563,17 @@ module EventStoreSource = let! _ = Async.StartChild <| writers.Pump() let! ct = Async.CancellationToken let mutable bytesPended = 0L - let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let mutable rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let resultsHandled, workPended, eventsPended = ref 0, ref 0, ref 0 + let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let badCats = CosmosIngester.Queue.CatStats() let dumpStats () = - log.Warning("Writer Exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed",!rateLimited, !timedOut, !malformed) - rateLimited := 0; timedOut := 0; malformed := 0 - Log.Warning("Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", + if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then + Log.Warning("Writer exceptions {rateLimited} Rate-limited, {timedOut} Timed out, {malformed}", !rateLimited, !timedOut, !malformed) + rateLimited := 0; timedOut := 0; malformed := 0 + if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + Log.Information("Writer throughput {queued} req {events} events Completed {completed} reqs Egress {gb:n3}GB", !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) - ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 + workPended := 0; eventsPended := 0; resultsHandled := 0 buffer.Dump log let tryDumpStats = every statsIntervalMs dumpStats let handle = function @@ -583,7 +592,7 @@ module EventStoreSource = | CosmosIngester.Queue.Ok -> res.WriteTo log | CosmosIngester.Queue.RateLimited -> incr rateLimited | CosmosIngester.Queue.TimedOut -> incr timedOut - | CosmosIngester.Queue.Malformed -> incr malformed + | CosmosIngester.Queue.Malformed -> category stream |> badCats.Ingest; incr malformed let queueWrite (w : CosmosIngester.Batch) = incr workPended eventsPended := !eventsPended + w.span.events.Length @@ -919,7 +928,8 @@ let main argv = createSyncHandler #else let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint - let esConnection, catFilter = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) + let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) + let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter) (256, target) #endif From ee2516fbeb392aeb98bd2b1d7a2ad41894136119 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 10 Apr 2019 20:04:25 +0100 Subject: [PATCH 041/353] Polishing --- equinox-sync/Sync/Program.fs | 76 ++++++++++++++++++++---------------- 1 file changed, 43 insertions(+), 33 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2338d0f34..245821d9f 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -62,7 +62,7 @@ module CmdParser = #else | [] MinBatchSize of int | [] Stream of string - | [] Offset of int64 + | [] Position of int64 | [] Chunk of int | [] Percent of float | [] Stripes of int @@ -90,7 +90,7 @@ module CmdParser = | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" | Stream _ -> "specific stream(s) to read" - | Offset _ -> "EventStore $all Stream Position to commence from" + | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Stripes _ -> "number of concurrent readers" @@ -137,7 +137,7 @@ module CmdParser = member x.BuildFeedParams() : ReaderSpec = Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = - match a.TryGetResult Offset, a.TryGetResult Chunk, a.TryGetResult Percent with + match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent with | Some p, _, _ -> Absolute p | _, Some c, _ -> StartPos.Chunk c | _, _, Some p -> Percentage p @@ -292,11 +292,8 @@ type EventStore.ClientAPI.RecordedEvent with module EventStoreSource = open EventStore.ClientAPI - let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length - let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata - let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 - let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 - let mb x = float x / 1024. / 1024. + let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata + let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 @@ -306,7 +303,7 @@ module EventStoreSource = let mutable batchBytes = 0 for x in slice.Events do let cat = category x.OriginalStreamId - let eventBytes = esPayloadBytes x + let eventBytes = payloadBytes x match recentCats.TryGetValue cat with | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) | false, _ -> recentCats.[cat] <- (1, eventBytes) @@ -396,7 +393,7 @@ module EventStoreSource = type [] PullResult = Exn of exn: exn | Eof | EndOfTranche type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : Position -> CosmosIngester.Batch[] -> unit) = - member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?ignoreEmptyEof) = + member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?once) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -415,12 +412,10 @@ module EventStoreSource = for pos, item in streamEvents do yield { stream = stream; span = { index = pos; events = [| item |]}} |] postBatch currentSlice.NextPosition events - if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", - range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) - let shouldLoop = range.TryNext currentSlice.NextPosition - if shouldLoop && not currentSlice.IsEndOfStream then + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, + batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) + if range.TryNext currentSlice.NextPosition && once <> Some true && not currentSlice.IsEndOfStream then sw.Restart() // restart the clock as we hand off back to the Reader return! loop () else @@ -470,15 +465,17 @@ module EventStoreSource = let slicesStats, stats = SliceStatsBuffer(), OverallStats() use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let progressSw = Stopwatch.StartNew() + let mutable paused = false while true do let currentPos = range.Current if progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Information("Tailed {count} times ({pauses} pauses @ {pos} (chunk {chunk})", + Log.Information("Tailed {count} times ({pauses} waits) @ {pos} (chunk {chunk})", count, pauses, currentPos.CommitPosition, chunk currentPos) progressSw.Restart() count <- count + 1 if shouldTail () then - let! res = reader.Pump(range,batchSize,slicesStats,stats,ignoreEmptyEof=true) + paused <- false + let! res = reader.Pump(range,batchSize,slicesStats,stats,once=true) do! awaitInterval match res with | PullResult.EndOfTranche | PullResult.Eof -> () @@ -486,6 +483,8 @@ module EventStoreSource = batchSize <- adjust batchSize Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) else + if not paused then Log.Information("Pausing...") + paused <- true pauses <- pauses + 1 do! awaitInterval stats.DumpIfIntervalExpired() @@ -509,7 +508,7 @@ module EventStoreSource = Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) - member __.Pump(postItem, shouldTail, postTail) = async { + member __.Pump(postItem, shouldTail, postTailBatch) = async { let maxDop = spec.stripes + 1 let dop = new SemaphoreSlim(maxDop) let mutable finished = false @@ -519,7 +518,7 @@ module EventStoreSource = work.OverallStats.DumpIfIntervalExpired() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, enumEvents, postItem, shouldTail, postTail, task) in () + try let! _ = work.Process(conn, enumEvents, postItem, shouldTail, postTailBatch, task) in () finally dop.Release() |> ignore } return () } match work.TryDequeue() with @@ -553,14 +552,21 @@ module EventStoreSource = let postWrite = work.Enqueue << CoordinationWork.Unbatched let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here - let shouldTailNow () = let _, pendingBatchCount = tailSyncState.Validate(fun _ -> None) in pendingBatchCount < 10 // TODO remove 10 + let shouldTailNow () = + let _, pendingBatchCount = tailSyncState.Validate(fun _ -> None) + let res = pendingBatchCount < 10 // TODO remove 10 + let level = if res then Events.LogEventLevel.Debug else Events.LogEventLevel.Information + Log.Write(level, "Pending Batches {pb}", pendingBatchCount) + res reader.Pump(postWrite, shouldTailNow, postBatch) + let pumpWriters = + writers.Pump() let postWriteResult = work.Enqueue << CoordinationWork.Result member __.Pump () = async { use _ = writers.Result.Subscribe postWriteResult let! _ = Async.StartChild pumpReaders - let! _ = Async.StartChild <| writers.Pump() + let! _ = Async.StartChild pumpWriters let! ct = Async.CancellationToken let mutable bytesPended = 0L let resultsHandled, workPended, eventsPended = ref 0, ref 0, ref 0 @@ -568,10 +574,10 @@ module EventStoreSource = let badCats = CosmosIngester.Queue.CatStats() let dumpStats () = if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then - Log.Warning("Writer exceptions {rateLimited} Rate-limited, {timedOut} Timed out, {malformed}", !rateLimited, !timedOut, !malformed) + Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", !rateLimited, !timedOut, !malformed) rateLimited := 0; timedOut := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - Log.Information("Writer throughput {queued} req {events} events Completed {completed} reqs Egress {gb:n3}GB", + Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) workPended := 0; eventsPended := 0; resultsHandled := 0 buffer.Dump log @@ -587,7 +593,6 @@ module EventStoreSource = incr resultsHandled let (stream, updatedState), kind = buffer.HandleWriteResult res match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) - res.WriteTo log match kind with | CosmosIngester.Queue.Ok -> res.WriteTo log | CosmosIngester.Queue.RateLimited -> incr rateLimited @@ -604,16 +609,18 @@ module EventStoreSource = | true, item -> handle item | false, _ -> - // 2. After that, [over] provision writers queue +// for ps in tailSyncState.PeekPendingStreams do +// buffer.IsReady + // 3. After that, [over] provision writers queue let mutable more = writers.HasCapacity while more do match buffer.TryReady() with | Some w -> queueWrite w; more <- writers.HasCapacity | None -> (); more <- false - // 3. Periodically emit status info + // 4. Periodically emit status info tryDumpStats () // TODO trigger periodic progress writing - // 5. Sleep if + // 6. Sleep if do! Async.Sleep sleepIntervalMs } let start (log : Serilog.ILogger) (conn, spec, enumEvents) (maxWriters, cosmosContext) = async { @@ -624,7 +631,7 @@ module EventStoreSource = let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { for e in xs -> - let eb = EventStoreSource.esPayloadBytes e + let eb = EventStoreSource.payloadBytes e match e.Event with | e when not e.IsJson || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) @@ -633,13 +640,16 @@ let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> Choice2Of2 e - | e when eb > EventStoreSource.cosmosPayloadLimit -> + | e when eb > CosmosIngester.cosmosPayloadLimit -> Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", - e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, EventStoreSource.cosmosPayloadLimit) + e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, CosmosIngester.cosmosPayloadLimit) Choice2Of2 e - | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) + | e -> + //if category e.EventStreamId = "ReloadBatchId" then Log.Information("RBID {s}", System.Text.Encoding.UTF8.GetString(e.Data)) + let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata + let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data + Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp)) } - //#else module CosmosSource = open Microsoft.Azure.Documents From 8d17ffabfedeae7c729b6124b0b935b8d402d379 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 11 Apr 2019 10:50:25 +0100 Subject: [PATCH 042/353] Fix prefix handling --- equinox-sync/Sync/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 245821d9f..6628952ec 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -483,14 +483,14 @@ module EventStoreSource = batchSize <- adjust batchSize Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) else - if not paused then Log.Information("Pausing...") + if not paused then Log.Information("Pausing due to backlog of incomplete batches...") paused <- true pauses <- pauses + 1 do! awaitInterval stats.DumpIfIntervalExpired() return true } - type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, max, ?statsInterval) = + type Reader(conn : IEventStoreConnection, spec : ReaderSpec, enumEvents, max, ?statsInterval) = let work = FeedQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) do let startPos = match spec.start with From b26f5c44cfa7bc7a0beb3a2ced87132087f126af Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 11 Apr 2019 16:18:26 +0100 Subject: [PATCH 043/353] Add gap reading --- equinox-sync/Sync/Program.fs | 49 ++++++++++++++++++++++++++---------- 1 file changed, 36 insertions(+), 13 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 6628952ec..d3a973106 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -379,17 +379,21 @@ module EventStoreSource = Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 return Option.get max } - let pullStream (conn : IEventStoreConnection, batchSize) stream (postBatch : CosmosIngester.Batch -> unit) = - let rec fetchFrom pos = async { - let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect - if currentSlice.IsEndOfStream then return () else + let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : CosmosIngester.Batch -> unit) = + let rec fetchFrom pos limit = async { + let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize + let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect let events = [| for x in currentSlice.Events -> let e = x.Event Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } - return! fetchFrom currentSlice.NextEventNumber } - fetchFrom 0L + match limit with + | None when currentSlice.IsEndOfStream -> return () + | Some limit when events.Length >= limit -> return () + | None -> return! fetchFrom currentSlice.NextEventNumber None + | Some limit -> return! fetchFrom currentSlice.NextEventNumber (Some (limit - events.Length)) } + fetchFrom pos limit type [] PullResult = Exn of exn: exn | Eof | EndOfTranche type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : Position -> CosmosIngester.Batch[] -> unit) = @@ -427,6 +431,7 @@ module EventStoreSource = type [] Work = | Stream of name: string * batchSize: int + | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int | Tail of pos: Position * interval: TimeSpan * batchSize : int type FeedQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() @@ -434,6 +439,8 @@ module EventStoreSource = member val SlicesStats = SliceStatsBuffer() member __.AddStream(name, ?batchSizeOverride) = work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = + work.Enqueue <| Work.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) member __.AddTail(pos, interval, ?batchSizeOverride) = work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = @@ -441,11 +448,21 @@ module EventStoreSource = member __.Process(conn, enumEvents, postItem, shouldTail, postTail, work) = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize match work with + | StreamPrefix (name,pos,len,batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) + Log.Information("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) + try do! pullStream (conn, batchSize) (name, pos, Some len) postItem + Log.Information("completed stream") + with e -> + let bs = adjust batchSize + Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + __.AddStreamPrefix(name, pos, len, bs) + return false | Stream (name,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream; batch size {bs}", batchSize) - try do! pullStream (conn, batchSize) name postItem - Log.Warning("completed stream") + try do! pullStream (conn, batchSize) (name,0L,None) postItem + Log.Information("completed stream") with e -> let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) @@ -491,6 +508,8 @@ module EventStoreSource = return true } type Reader(conn : IEventStoreConnection, spec : ReaderSpec, enumEvents, max, ?statsInterval) = + let maxDop = spec.stripes + 1 + let dop = new SemaphoreSlim(maxDop) let work = FeedQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) do let startPos = match spec.start with @@ -507,10 +526,10 @@ module EventStoreSource = work.AddStream s Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) - + /// Computes number of streams that can be admitted to the stream loading queue by AddStreamPrefix + member __.ReadersHeadroom = dop.CurrentCount * 2 + member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) member __.Pump(postItem, shouldTail, postTailBatch) = async { - let maxDop = spec.stripes + 1 - let dop = new SemaphoreSlim(maxDop) let mutable finished = false let! ct = Async.CancellationToken while not ct.IsCancellationRequested do @@ -609,8 +628,12 @@ module EventStoreSource = | true, item -> handle item | false, _ -> -// for ps in tailSyncState.PeekPendingStreams do -// buffer.IsReady + // 2. Enqueue streams with gaps if there is capacity (not too early to avoid redundant work) + let mutable capacity = reader.ReadersHeadroom + while capacity > 0 do + match buffer.TryGap() with + | None -> capacity <- 0 + | Some (stream,pos,len) -> reader.AddStreamPrefix(stream,pos,len); capacity <- capacity - 1 // 3. After that, [over] provision writers queue let mutable more = writers.HasCapacity while more do From dd62644a91cc7a06df0ff2d35a39aedfecaac20e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 13 Apr 2019 05:27:50 +0100 Subject: [PATCH 044/353] Fix throttling and stream algorithms --- equinox-sync/Sync/Infrastructure.fs | 38 ++++- equinox-sync/Sync/Program.fs | 234 +++++++++++++++------------- 2 files changed, 160 insertions(+), 112 deletions(-) diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 21fbdb72a..3cb6f7255 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -5,6 +5,7 @@ open Equinox.Store // AwaitTaskCorrect open System open System.Threading open System.Threading.Tasks +open System.Collections.Generic #nowarn "21" // re AwaitKeyboardInterrupt #nowarn "40" // re AwaitKeyboardInterrupt @@ -44,4 +45,39 @@ type SemaphoreSlim with let! _ = semaphore.Await() try return! workflow finally semaphore.Release() |> ignore - } \ No newline at end of file + } + +type RefCounted<'T> = { mutable refCount: int; value: 'T } + +// via https://stackoverflow.com/a/31194647/11635 +type SemaphorePool(gen : unit -> SemaphoreSlim) = + let inners: Dictionary> = Dictionary() + + let getOrCreateSlot key = + lock inners <| fun () -> + match inners.TryGetValue key with + | true, inner -> + inner.refCount <- inner.refCount + 1 + inner.value + | false, _ -> + let value = gen () + inners.[key] <- { refCount = 1; value = value } + value + let slotReleaseGuard key : IDisposable = + { new System.IDisposable with + member __.Dispose() = + lock inners <| fun () -> + let item = inners.[key] + match item.refCount with + | 1 -> inners.Remove key |> ignore + | current -> item.refCount <- current - 1 } + + member __.ExecuteAsync(k,f) = async { + let x = getOrCreateSlot k + use _ = slotReleaseGuard k + return! f x } + + member __.Execute(k,f) = + let x = getOrCreateSlot k + use _l = slotReleaseGuard k + f x \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index d3a973106..1822baaa4 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -40,7 +40,6 @@ let every ms f = f () timer.Restart() - module CmdParser = open Argu @@ -93,7 +92,7 @@ module CmdParser = | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | Stripes _ -> "number of concurrent readers" + | Stripes _ -> "number of concurrent readers. Default: 8" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | VerboseConsole -> "request Verbose Console Logging. Default: off" | Source _ -> "EventStore input parameters." @@ -113,7 +112,7 @@ module CmdParser = member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) member __.MinBatchSize = a.GetResult(MinBatchSize,512) - member __.Stripes = a.GetResult(Stripes,1) + member __.Stripes = a.GetResult(Stripes,8) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds #endif @@ -135,7 +134,7 @@ module CmdParser = disco, db, x.LeaseId, x.StartFromHere, x.BatchSize, x.LagFrequency #else member x.BuildFeedParams() : ReaderSpec = - Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) + Log.Information("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent with | Some p, _, _ -> Absolute p @@ -295,9 +294,25 @@ module EventStoreSource = let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + type OverallStats(?statsInterval) = + let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() + let mutable totalEvents, totalBytes = 0L, 0L + member __.Ingest(batchEvents, batchBytes) = + Interlocked.Add(&totalEvents,batchEvents) |> ignore + Interlocked.Add(&totalBytes,batchBytes) |> ignore + member __.Bytes = totalBytes + member __.Events = totalEvents + member __.DumpIfIntervalExpired(?force) = + if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then + let totalMb = mb totalBytes + Log.Information("EventStore throughput {events} events {gb:n1}GB {mbs:n2}MB/s", + totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + progressStart.Restart() + type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() + let recentCats, accStart = Dictionary(), Stopwatch.StartNew() member __.Ingest(slice: AllEventsSlice) = lock recentCats <| fun () -> let mutable batchBytes = 0 @@ -326,22 +341,6 @@ module EventStoreSource = recentCats.Clear() accStart.Restart() - type OverallStats(?statsInterval) = - let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() - let mutable totalEvents, totalBytes = 0L, 0L - member __.Ingest(batchEvents, batchBytes) = - Interlocked.Add(&totalEvents,batchEvents) |> ignore - Interlocked.Add(&totalBytes,batchBytes) |> ignore - member __.Bytes = totalBytes - member __.Events = totalEvents - member __.DumpIfIntervalExpired(?force) = - if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then - let totalMb = mb totalBytes - Log.Information("EventStore throughput {events} events {gb:n1}GB {mbs:n2}MB/s", - totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) - progressStart.Restart() - type Range(start, sliceEnd : Position option, ?max : Position) = member val Current = start with get, set member __.TryNext(pos: Position) = @@ -382,7 +381,7 @@ module EventStoreSource = let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : CosmosIngester.Batch -> unit) = let rec fetchFrom pos limit = async { let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize - let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect + let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect let events = [| for x in currentSlice.Events -> let e = x.Event @@ -390,53 +389,55 @@ module EventStoreSource = postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } match limit with | None when currentSlice.IsEndOfStream -> return () - | Some limit when events.Length >= limit -> return () | None -> return! fetchFrom currentSlice.NextEventNumber None + | Some limit when events.Length >= limit -> return () | Some limit -> return! fetchFrom currentSlice.NextEventNumber (Some (limit - events.Length)) } fetchFrom pos limit type [] PullResult = Exn of exn: exn | Eof | EndOfTranche - type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : Position -> CosmosIngester.Batch[] -> unit) = - member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?once) = - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let rec loop () = async { - let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let postSw = Stopwatch.StartNew() - let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) - let streams = - enumEvents currentSlice.Events - |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) - |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) - |> Array.ofSeq - let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - let events : CosmosIngester.Batch[] = - [| for stream,streamEvents in streams do - for pos, item in streamEvents do + let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) + (range:Range, once) enumEvents (postBatch : Position -> CosmosIngester.Batch[] -> unit) = + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let rec aux () = async { + let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let postSw = Stopwatch.StartNew() + let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) + let streams = + enumEvents currentSlice.Events + |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) + |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) + |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) + |> Array.ofSeq + let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length + let events : CosmosIngester.Batch[] = + [| for stream,streamEvents in streams do + for pos, item in streamEvents do yield { stream = stream; span = { index = pos; events = [| item |]}} |] - postBatch currentSlice.NextPosition events - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", - range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) - if range.TryNext currentSlice.NextPosition && once <> Some true && not currentSlice.IsEndOfStream then - sw.Restart() // restart the clock as we hand off back to the Reader - return! loop () - else - return currentSlice.IsEndOfStream } - async { - try let! eof = loop () - return if eof then Eof else EndOfTranche - with e -> return Exn e } + //Log.Information("Bat {bat}", events) + postBatch currentSlice.NextPosition events + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, + batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) + if range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream then + sw.Restart() // restart the clock as we hand off back to the Reader + return! aux () + else + return currentSlice.IsEndOfStream } + async { + try let! eof = aux () + return if eof then Eof else EndOfTranche + with e -> return Exn e } type [] Work = | Stream of name: string * batchSize: int | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int | Tail of pos: Position * interval: TimeSpan * batchSize : int - type FeedQueue(batchSize, minBatchSize, ?statsInterval) = + type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) member val SlicesStats = SliceStatsBuffer() + member __.QueueCount = work.Count member __.AddStream(name, ?batchSizeOverride) = work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = @@ -450,9 +451,9 @@ module EventStoreSource = match work with | StreamPrefix (name,pos,len,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - Log.Information("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) + Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) try do! pullStream (conn, batchSize) (name, pos, Some len) postItem - Log.Information("completed stream") + Log.Information("completed stream prefix") with e -> let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) @@ -478,7 +479,6 @@ module EventStoreSource = | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) | _ -> () tailSw.Restart() } - let reader = ReaderGroup(conn, enumEvents, postTail) let slicesStats, stats = SliceStatsBuffer(), OverallStats() use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let progressSw = Stopwatch.StartNew() @@ -492,7 +492,7 @@ module EventStoreSource = count <- count + 1 if shouldTail () then paused <- false - let! res = reader.Pump(range,batchSize,slicesStats,stats,once=true) + let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) enumEvents postTail do! awaitInterval match res with | PullResult.EndOfTranche | PullResult.Eof -> () @@ -507,10 +507,11 @@ module EventStoreSource = stats.DumpIfIntervalExpired() return true } - type Reader(conn : IEventStoreConnection, spec : ReaderSpec, enumEvents, max, ?statsInterval) = + type Readers(conn : IEventStoreConnection, spec : ReaderSpec, enumEvents, max, ?statsInterval) = + let sleepIntervalMs = 100 let maxDop = spec.stripes + 1 let dop = new SemaphoreSlim(maxDop) - let work = FeedQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) + let work = ReadQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) do let startPos = match spec.start with | StartPos.Tail -> max @@ -526,15 +527,15 @@ module EventStoreSource = work.AddStream s Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) - /// Computes number of streams that can be admitted to the stream loading queue by AddStreamPrefix - member __.ReadersHeadroom = dop.CurrentCount * 2 - member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) + member __.HasCapacity = work.QueueCount < dop.CurrentCount + member __.AddStreamPrefix(stream, pos, len) = + work.AddStreamPrefix(stream, pos, len) + __.HasCapacity member __.Pump(postItem, shouldTail, postTailBatch) = async { - let mutable finished = false let! ct = Async.CancellationToken while not ct.IsCancellationRequested do - let! _ = dop.Await() work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() let forkRunRelease task = async { let! _ = Async.StartChild <| async { try let! _ = work.Process(conn, enumEvents, postItem, shouldTail, postTailBatch, task) in () @@ -543,62 +544,62 @@ module EventStoreSource = match work.TryDequeue() with | true, task -> do! forkRunRelease task - | false, _ when not finished-> - if spec.streams <> [] then Log.Information("Initial streams seeded") - finished <- true - | _ -> () } + | false, _ -> + dop.Release() |> ignore + do! Async.Sleep sleepIntervalMs } type [] CoordinationWork<'Pos> = | Result of CosmosIngester.Writer.Result | Unbatched of CosmosIngester.Batch | BatchWithTracking of 'Pos * CosmosIngester.Batch[] - let every ms f = - let timer = Stopwatch.StartNew() - fun () -> - if timer.ElapsedMilliseconds > ms then - f () - timer.Restart() - - type Coordinator(log : Serilog.ILogger, reader : Reader, cosmosContext, ?maxWriters, ?interval) = + type Coordinator(log : Serilog.ILogger, readers : Readers, cosmosContext, maxWriters, ?interval) = let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let sleepIntervalMs = 100 let work = System.Collections.Concurrent.ConcurrentQueue() let buffer = CosmosIngester.Queue.StreamStates() - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, defaultArg maxWriters 32) + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) let tailSyncState = Progress.State() + // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here + let mutable pendingBatchCount = 0 + let shouldThrottle () = pendingBatchCount > 32 + let mutable progressEpoch = None let pumpReaders = let postWrite = work.Enqueue << CoordinationWork.Unbatched let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) - // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here - let shouldTailNow () = - let _, pendingBatchCount = tailSyncState.Validate(fun _ -> None) - let res = pendingBatchCount < 10 // TODO remove 10 - let level = if res then Events.LogEventLevel.Debug else Events.LogEventLevel.Information - Log.Write(level, "Pending Batches {pb}", pendingBatchCount) - res - reader.Pump(postWrite, shouldTailNow, postBatch) - let pumpWriters = - writers.Pump() + readers.Pump(postWrite, not << shouldThrottle, postBatch) + let pumpWriters = writers.Pump() let postWriteResult = work.Enqueue << CoordinationWork.Result - member __.Pump () = async { use _ = writers.Result.Subscribe postWriteResult let! _ = Async.StartChild pumpReaders let! _ = Async.StartChild pumpWriters let! ct = Async.CancellationToken + let writerResultLog = log.ForContext() let mutable bytesPended = 0L - let resultsHandled, workPended, eventsPended = ref 0, ref 0, ref 0 + let workPended, eventsPended = ref 0, ref 0 let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 let badCats = CosmosIngester.Queue.CatStats() let dumpStats () = if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then - Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", !rateLimited, !timedOut, !malformed) + Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", + !rateLimited, !timedOut, !malformed) rateLimited := 0; timedOut := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", - !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) - workPended := 0; eventsPended := 0; resultsHandled := 0 + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn + Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} ok {ok} redundant {dup} partial {partial} Missing {prefix} Exceptions {exns} reqs; Egress {gb:n3}GB", + !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) + workPended := 0; eventsPended := 0 + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + + let throttle = shouldThrottle () + let level = if not throttle then Events.LogEventLevel.Debug else Events.LogEventLevel.Information + Log.Write(level, "Pending Batches {pb}", pendingBatchCount) + match progressEpoch with + | None -> () + | Some (epoch : Position) -> log.Information("Progress Epoch @ {epoch}", epoch.CommitPosition) + buffer.Dump log let tryDumpStats = every statsIntervalMs dumpStats let handle = function @@ -609,47 +610,58 @@ module EventStoreSource = buffer.Add item |> ignore tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) | CoordinationWork.Result res -> - incr resultsHandled + match res with + | CosmosIngester.Writer.Result.Ok _ -> incr resultOk + | CosmosIngester.Writer.Result.Duplicate _ -> incr resultDup + | CosmosIngester.Writer.Result.PartialDuplicate _ -> incr resultPartialDup + | CosmosIngester.Writer.Result.PrefixMissing _ -> incr resultPrefix + | CosmosIngester.Writer.Result.Exn _ -> incr resultExn + let (stream, updatedState), kind = buffer.HandleWriteResult res match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) match kind with - | CosmosIngester.Queue.Ok -> res.WriteTo log + | CosmosIngester.Queue.Ok -> res.WriteTo writerResultLog | CosmosIngester.Queue.RateLimited -> incr rateLimited - | CosmosIngester.Queue.TimedOut -> incr timedOut | CosmosIngester.Queue.Malformed -> category stream |> badCats.Ingest; incr malformed + | CosmosIngester.Queue.TimedOut -> incr timedOut let queueWrite (w : CosmosIngester.Batch) = incr workPended eventsPended := !eventsPended + w.span.events.Length bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) writers.Enqueue w + writers.HasCapacity while not ct.IsCancellationRequested do // 1. propagate read items to buffer; propagate write results to buffer + Progress match work.TryDequeue() with | true, item -> handle item | false, _ -> - // 2. Enqueue streams with gaps if there is capacity (not too early to avoid redundant work) - let mutable capacity = reader.ReadersHeadroom - while capacity > 0 do + // 2. Mark off any progress achieved + let _validatedPos, _pendingBatchCount = tailSyncState.Validate buffer.TryGetStreamWritePos + pendingBatchCount <- _pendingBatchCount + progressEpoch <- _validatedPos + // 3. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + let mutable more = readers.HasCapacity + while more do match buffer.TryGap() with - | None -> capacity <- 0 - | Some (stream,pos,len) -> reader.AddStreamPrefix(stream,pos,len); capacity <- capacity - 1 - // 3. After that, [over] provision writers queue + | Some (stream,pos,len) -> more <- readers.AddStreamPrefix(stream,pos,len) + | None -> more <- false + // 4. After that, [over] provision writers queue let mutable more = writers.HasCapacity while more do - match buffer.TryReady() with - | Some w -> queueWrite w; more <- writers.HasCapacity + match buffer.TryReady(writers.IsStreamBusy) with + | Some w -> more <- queueWrite w | None -> (); more <- false - // 4. Periodically emit status info + // 5. Periodically emit status info tryDumpStats () // TODO trigger periodic progress writing - // 6. Sleep if + // 7. Sleep if do! Async.Sleep sleepIntervalMs } let start (log : Serilog.ILogger) (conn, spec, enumEvents) (maxWriters, cosmosContext) = async { let! max = establishMax conn - let reader = Reader(conn, spec, enumEvents, max) - let coordinator = Coordinator(log, reader, cosmosContext, maxWriters) + let readers = Readers(conn, spec, enumEvents, max) + let coordinator = Coordinator(log, readers, cosmosContext, maxWriters) do! coordinator.Pump () } let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { @@ -964,7 +976,7 @@ let main argv = let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() - EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter) (256, target) + EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter) (16, target) #endif |> Async.RunSynchronously 0 From d5f28be324a52a011eaff874d1bcef0518898848 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 14 Apr 2019 14:22:44 +0100 Subject: [PATCH 045/353] Separate CosmosIngester --- equinox-sync/Sync/Program.fs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 1822baaa4..aaef25328 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -553,16 +553,17 @@ module EventStoreSource = | Unbatched of CosmosIngester.Batch | BatchWithTracking of 'Pos * CosmosIngester.Batch[] - type Coordinator(log : Serilog.ILogger, readers : Readers, cosmosContext, maxWriters, ?interval) = + type Coordinator(log : Serilog.ILogger, readers : Readers, cosmosContext, maxWriters, ?interval, ?maxPendingBatches) = + let maxPendingBatches = defaultArg maxPendingBatches 32 let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let sleepIntervalMs = 100 let work = System.Collections.Concurrent.ConcurrentQueue() - let buffer = CosmosIngester.Queue.StreamStates() + let buffer = CosmosIngester.StreamStates() let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) - let tailSyncState = Progress.State() + let tailSyncState = ProgressBatcher.State() // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here let mutable pendingBatchCount = 0 - let shouldThrottle () = pendingBatchCount > 32 + let shouldThrottle () = pendingBatchCount > maxPendingBatches let mutable progressEpoch = None let pumpReaders = let postWrite = work.Enqueue << CoordinationWork.Unbatched @@ -580,7 +581,7 @@ module EventStoreSource = let workPended, eventsPended = ref 0, ref 0 let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 - let badCats = CosmosIngester.Queue.CatStats() + let badCats = CosmosIngester.CatStats() let dumpStats () = if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", @@ -620,10 +621,10 @@ module EventStoreSource = let (stream, updatedState), kind = buffer.HandleWriteResult res match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) match kind with - | CosmosIngester.Queue.Ok -> res.WriteTo writerResultLog - | CosmosIngester.Queue.RateLimited -> incr rateLimited - | CosmosIngester.Queue.Malformed -> category stream |> badCats.Ingest; incr malformed - | CosmosIngester.Queue.TimedOut -> incr timedOut + | CosmosIngester.Ok -> res.WriteTo writerResultLog + | CosmosIngester.RateLimited -> incr rateLimited + | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed + | CosmosIngester.TimedOut -> incr timedOut let queueWrite (w : CosmosIngester.Batch) = incr workPended eventsPended := !eventsPended + w.span.events.Length From 6eeb4f9cb009c3f48fbf01fecbc8f50eb6a99d67 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 14 Apr 2019 15:47:31 +0100 Subject: [PATCH 046/353] Rebase Ingest on Sync --- equinox-ingest/Ingest/Infrastructure.fs | 18 +- equinox-ingest/Ingest/Ingest.fsproj | 1 + equinox-ingest/Ingest/Program.fs | 438 +++++------------------- equinox-sync/Sync/CosmosIngester.fs | 11 + equinox-sync/Sync/Infrastructure.fs | 40 +-- 5 files changed, 102 insertions(+), 406 deletions(-) diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs index 6c4cd7e28..816652591 100644 --- a/equinox-ingest/Ingest/Infrastructure.fs +++ b/equinox-ingest/Ingest/Infrastructure.fs @@ -1,5 +1,5 @@ [] -module private IngestTemplate.Infrastructure +module private Infrastructure open Equinox.Store // AwaitTaskCorrect open System @@ -15,16 +15,12 @@ type SemaphoreSlim with return! Async.AwaitTaskCorrect task } -module Queue = - let tryDequeue (x : System.Collections.Generic.Queue<'T>) = -#if NET461 - if x.Count = 0 then None - else x.Dequeue() |> Some -#else - match x.TryDequeue() with - | false, _ -> None - | true, res -> Some res -#endif + /// Throttling wrapper which waits asynchronously until the semaphore has available capacity + member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { + let! _ = semaphore.Await() + try return! workflow + finally semaphore.Release() |> ignore + } #nowarn "21" // re AwaitKeyboardInterrupt #nowarn "40" // re AwaitKeyboardInterrupt diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj index a6e638dd7..094ebc154 100644 --- a/equinox-ingest/Ingest/Ingest.fsproj +++ b/equinox-ingest/Ingest/Ingest.fsproj @@ -9,6 +9,7 @@ + diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-ingest/Ingest/Program.fs index bd529e2b8..7b41878be 100644 --- a/equinox-ingest/Ingest/Program.fs +++ b/equinox-ingest/Ingest/Program.fs @@ -1,6 +1,7 @@ module IngestTemplate.Program open Equinox.Store // Infra +open SyncTemplate open FSharp.Control open Serilog open System @@ -8,8 +9,8 @@ open System.Collections.Concurrent open System.Diagnostics open System.Threading -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start | Ignore -type ReaderSpec = { start: StartPos; streams: string list; tailInterval: TimeSpan option; stripes: int; batchSize: int; minBatchSize: int } +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start +type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; minBatchSize: int } let mb x = float x / 1024. / 1024. module CmdParser = @@ -119,13 +120,10 @@ module CmdParser = | [] Verbose | [] VerboseConsole | [] LocalSeq - | [] Stream of string - | [] All - | [] Offset of int64 + | [] Position of int64 | [] Chunk of int | [] Percent of float | [] Stripes of int - | [] Tail of intervalS: float | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = @@ -135,13 +133,10 @@ module CmdParser = | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | Stream _ -> "specific stream(s) to read" - | All -> "traverse EventStore $all from Start" - | Offset _ -> "EventStore $all Stream Position to commence from" + | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Stripes _ -> "number of concurrent readers" - | Tail _ -> "attempt to read from tail at specified interval in Seconds" | Es _ -> "specify EventStore parameters" and Arguments(args : ParseResults) = member val EventStore = EventStore.Arguments(args.GetResult Es) @@ -151,21 +146,15 @@ module CmdParser = member __.StartingBatchSize = args.GetResult(BatchSize,4096) member __.MinBatchSize = args.GetResult(MinBatchSize,512) member __.Stripes = args.GetResult(Stripes,1) - member __.TailInterval = match args.TryGetResult Tail with Some s -> TimeSpan.FromSeconds s |> Some | None -> None member x.BuildFeedParams() : ReaderSpec = Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = - match args.TryGetResult Offset, args.TryGetResult Chunk, args.TryGetResult Percent, args.Contains All with - | Some p, _, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some c, _, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c - | _, _, Some p, _ -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p - | None, None, None, true -> Log.Warning "Processing will commence at $all Start"; Start - | None, None, None, false ->Log.Warning "No $all processing requested"; Ignore - match x.TailInterval with - | Some interval -> Log.Warning("Following tail at {seconds}s interval", interval.TotalSeconds) - | None -> Log.Warning "Not following tail" - { start = startPos; streams = args.GetResults Stream; tailInterval = x.TailInterval - batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } + match args.TryGetResult Position, args.TryGetResult Chunk, args.TryGetResult Percent with + | Some p, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p + | _, Some c, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c + | _, _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p + | None, None, None -> Log.Warning "Processing will commence at $all Start"; Start + { start = startPos; batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args let parse argv : Arguments = @@ -187,270 +176,67 @@ module Logging = |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() -module Ingester = - open Equinox.Cosmos.Core - open Equinox.Cosmos.Store - - type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } - type [] Batch = { stream: string; span: Span } - - module Writer = - type [] Result = - | Ok of stream: string * updatedPos: int64 - | Duplicate of stream: string * updatedPos: int64 - | Conflict of overage: Batch - | Exn of exn: exn * batch: Batch - member __.WriteTo(log: ILogger) = - match __ with - | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) - | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) - | Conflict overage -> log.Information("Requeing {stream} {pos} ({count} events)", overage.stream, overage.span.index, overage.span.events.Length) - | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) - let private write (ctx : CosmosContext) ({ stream = s; span={ index = p; events = e}} as batch) = async { - let stream = ctx.CreateStream s - Log.Information("Writing {s}@{i}x{n}",s,p,e.Length) - try let! res = ctx.Sync(stream, { index = p; etag = None }, e) - match res with - | AppendResult.Ok pos -> return Ok (s, pos.index) - | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> - match pos.index, p + e.LongLength with - | actual, expectedMax when actual >= expectedMax -> return Duplicate (s, pos.index) - | actual, _ when p >= actual -> return Conflict batch - | actual, _ -> - Log.Debug("pos {pos} batch.pos {bpos} len {blen} skip {skip}", actual, p, e.LongLength, actual-p) - return Conflict { stream = s; span = { index = actual; events = e |> Array.skip (actual-p |> int) } } - with e -> return Exn (e, batch) } - - /// Manages distribution of work across a specified number of concurrent writers - type WriteQueue(ctx : CosmosContext, queueLen, ct : CancellationToken) = - let buffer = new BlockingCollection<_>(ConcurrentQueue(), queueLen) - let result = Event<_>() - let child = async { - let! ct = Async.CancellationToken // i.e. cts.Token - for item in buffer.GetConsumingEnumerable(ct) do - let! res = write ctx item - result.Trigger res } - member internal __.StartConsumers n = - for _ in 1..n do - Async.StartAsTask(child, cancellationToken=ct) |> ignore - - /// Supply an item to be processed - member __.TryAdd(item, timeout : TimeSpan) = buffer.TryAdd(item, int timeout.TotalMilliseconds, ct) - [] member __.Result = result.Publish - - let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length - - type [] StreamState = { read: int64 option; write: int64 option; isMalformed : bool; queue: Span[] } with - /// Determines whether the head is ready to write (either write position is unknown, or matches) - member __.IsHeady = Array.tryHead __.queue |> Option.exists (fun x -> __.write |> Option.forall (fun w -> w = x.index)) - member __.IsReady = __.queue <> null && not __.isMalformed && __.IsHeady - member __.Size = - if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length*2 + 16) - module StreamState = - module Span = - let private (|Max|) x = x.index + x.events.LongLength - let private trim min (Max m as x) = - // Full remove - if m <= min then { index = min; events = [||] } - // Trim until min - elif m > min && x.index < min then { index = min; events = x.events |> Array.skip (min - x.index |> int) } - // Leave it - else x - let merge min (xs : Span seq) = - let buffer = ResizeArray() - let mutable curr = { index = min; events = [||]} - for x in xs |> Seq.sortBy (fun x -> x.index) do - match curr, trim min x with - // no data incoming, skip - | _, x when x.events.Length = 0 -> - () - // Not overlapping, no data buffered -> buffer - | c, x when c.events.Length = 0 -> - curr <- x - // Overlapping, join - | Max cMax as c, x when cMax >= x.index -> - curr <- { c with events = Array.append c.events (trim cMax x).events } - // Not overlapping, new data - | c, x -> - buffer.Add c - curr <- x - if curr.events.Length <> 0 then buffer.Add curr - if buffer.Count = 0 then null else buffer.ToArray() - - let inline optionCombine f (r1: int64 option) (r2: int64 option) = - match r1, r2 with - | Some x, Some y -> f x y |> Some - | None, None -> None - | None, x | x, None -> x - let combine (s1: StreamState) (s2: StreamState) : StreamState = - let writePos = optionCombine max s1.write s2.write - let items = seq { if s1.queue <> null then yield! s1.queue; if s2.queue <> null then yield! s2.queue } - { read = optionCombine max s1.read s2.read; write = writePos; isMalformed = s1.isMalformed || s2.isMalformed; queue = Span.merge (defaultArg writePos 0L) items} - - /// Gathers stats relating to how many items of a given category have been observed - type CatStats() = - let cats = System.Collections.Generic.Dictionary() - member __.Ingest cat = - match cats.TryGetValue cat with - | true, catCount -> cats.[cat] <- catCount + 1 - | false, _ -> cats.[cat] <- 1 - member __.Any = cats.Count <> 0 - member __.Clear() = cats.Clear() - member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd - - let category (s : string) = s.Split([|'-'|], 2, StringSplitOptions.RemoveEmptyEntries) |> Array.head - let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 - let inline cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 - let (|TimedOutMessage|RateLimitedMessage|MalformedMessage|Other|) (e: exn) = - match string e with - | m when m.Contains "Microsoft.Azure.Documents.RequestTimeoutException" -> TimedOutMessage - | m when m.Contains "Microsoft.Azure.Documents.RequestRateTooLargeException" -> RateLimitedMessage - | m when m.Contains "SyntaxError: JSON.parse Error: Unexpected input at position" - || m.Contains "SyntaxError: JSON.parse Error: Invalid character at position" -> MalformedMessage - | _ -> Other - - type Result = TimedOut | RateLimited | Malformed of category: string | Ok - type StreamStates() = - let states = System.Collections.Generic.Dictionary() - let dirty = System.Collections.Generic.Queue() - let markDirty stream = if dirty.Contains stream |> not then dirty.Enqueue stream - - let update stream (state : StreamState) = - Log.Debug("Updated {s} r{r} w{w}", stream, state.read, state.write) - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - markDirty stream |> ignore - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - if updated.IsReady then markDirty stream |> ignore - let updateWritePos stream pos isMalformed span = - update stream { read = None; write = pos; isMalformed = isMalformed; queue = span } - - member __.Add (item: Batch, ?isMalformed) = updateWritePos item.stream None (defaultArg isMalformed false) [|item.span|] - member __.HandleWriteResult = function - | Writer.Result.Ok (stream, pos) -> updateWritePos stream (Some pos) false null; Ok - | Writer.Result.Duplicate (stream, pos) -> updateWritePos stream (Some pos) false null; Ok - | Writer.Result.Conflict overage -> updateWritePos overage.stream (Some overage.span.index) false [|overage.span|]; Ok - | Writer.Result.Exn (exn, batch) -> - let r, malformed = - match exn with - | RateLimitedMessage -> RateLimited, false - | TimedOutMessage -> TimedOut, false - | MalformedMessage -> Malformed (category batch.stream), true - | Other -> Ok, false - __.Add(batch,malformed) - r - member __.TryPending() = - match dirty |> Queue.tryDequeue with - | None -> None - | Some stream -> - let state = states.[stream] - - if not state.IsReady then None else - - let x = state.queue |> Array.head - - let mutable bytesBudget = cosmosPayloadLimit - let mutable count = 0 - let max2MbMax1000EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = - bytesBudget <- bytesBudget - cosmosPayloadBytes y - count <- count + 1 - // Reduce the item count when we don't yet know the write position - count <= (if Option.isNone state.write then 10 else 100) && (bytesBudget >= 0 || count = 1) - Some { stream = stream; span = { index = x.index; events = x.events |> Array.takeWhile max2MbMax1000EventsMax10EventsFirstTranche } } - member __.Dump() = - let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 - let mutable readyB, waitingB, malformedB = 0L, 0L, 0L - let waitCats = CatStats() - for KeyValue (stream,state) in states do - match int64 state.Size with - | 0L -> synced <- synced + 1 - | sz when state.isMalformed -> malformed <- malformed + 1; malformedB <- malformedB + sz - | sz when state.IsReady -> ready <- ready + 1; readyB <- readyB + sz - | sz -> waitCats.Ingest(category stream); waiting <- waiting + 1; waitingB <- waitingB + sz - Log.Warning("Syncing {dirty} Ready {ready}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", - dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB, synced) - if waitCats.Any then Log.Warning("Waiting {waitCats}", waitCats.StatsDescending) - - type SyncQueue(log : Serilog.ILogger, writer : Writer.WriteQueue, cancellationToken: CancellationToken, readerQueueLen, ?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let states = StreamStates() - let results = ConcurrentQueue<_>() - let work = new BlockingCollection<_>(ConcurrentQueue<_>(), readerQueueLen) - - member __.Add item = work.Add item - member __.HandleWriteResult = results.Enqueue - member __.Pump() = - let fiveMs = TimeSpan.FromMilliseconds 5. - let mutable pendingWriterAdd = None - let mutable bytesPended = 0L - let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let badCats = CatStats() - let progressTimer = Stopwatch.StartNew() - while not cancellationToken.IsCancellationRequested do - let mutable moreResults, rateLimited, timedOut = true, 0, 0 - while moreResults do - match results.TryDequeue() with - | true, res -> - incr resultsHandled - match states.HandleWriteResult res with - | Malformed cat -> badCats.Ingest cat - | RateLimited -> rateLimited <- rateLimited + 1 - | TimedOut -> timedOut <- timedOut + 1 - | Ok -> res.WriteTo log - | false, _ -> moreResults <- false - if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) - let mutable t = Unchecked.defaultof<_> - let mutable toIngest = 4096 * 5 - while work.TryTake(&t,fiveMs) && toIngest > 0 do - incr ingestionsHandled - toIngest <- toIngest - 1 - states.Add t - let mutable moreWork = true - while moreWork do - let wrk = - match pendingWriterAdd with - | Some w -> - pendingWriterAdd <- None - Some w - | None -> - let pending = states.TryPending() - match pending with - | Some p -> Some p - | None -> - moreWork <- false - None - match wrk with - | None -> () - | Some w -> - if not (writer.TryAdd(w,fiveMs)) then - moreWork <- false - pendingWriterAdd <- Some w - else - incr workPended - eventsPended := !eventsPended + w.span.events.Length - bytesPended <- bytesPended + int64 (Array.sumBy cosmosPayloadBytes w.span.events) - - if progressTimer.ElapsedMilliseconds > intervalMs then - progressTimer.Restart() - Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", - !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) - if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() - ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 - states.Dump() +type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancellationToken: CancellationToken, readerQueueLen, ?interval) = + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 + let states = CosmosIngester.StreamStates() + let results = ConcurrentQueue<_>() + let work = new BlockingCollection<_>(ConcurrentQueue<_>(), readerQueueLen) + + member __.Add item = work.Add item + member __.HandleWriteResult = results.Enqueue + member __.Pump() = + let _ = writers.Result.Subscribe __.HandleWriteResult // codependent, wont worry about unsubcribing + let fiveMs = TimeSpan.FromMilliseconds 5. + let mutable pendingWriterAdd = None + let mutable bytesPended = 0L + let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 + let badCats = CosmosIngester.CatStats() + let progressTimer = Stopwatch.StartNew() + while not cancellationToken.IsCancellationRequested do + let mutable moreResults, rateLimited, timedOut = true, 0, 0 + while moreResults do + match results.TryDequeue() with + | true, res -> + incr resultsHandled + match states.HandleWriteResult res with + | (stream, _), CosmosIngester.Malformed -> CosmosIngester.category stream |> badCats.Ingest + | _, CosmosIngester.RateLimited -> rateLimited <- rateLimited + 1 + | _, CosmosIngester.TimedOut -> timedOut <- timedOut + 1 + | _, CosmosIngester.Ok -> res.WriteTo log + | false, _ -> moreResults <- false + if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) + let mutable t = Unchecked.defaultof<_> + let mutable toIngest = 4096 * 5 + while work.TryTake(&t,fiveMs) && toIngest > 0 do + incr ingestionsHandled + toIngest <- toIngest - 1 + states.Add t |> ignore + let mutable moreWork = true + while writers.HasCapacity && moreWork do + let pending = states.TryReady(writers.IsStreamBusy) + match pending with + | None -> moreWork <- false + | Some w -> + incr workPended + eventsPended := !eventsPended + w.span.events.Length + bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) + + if progressTimer.ElapsedMilliseconds > intervalMs then + progressTimer.Restart() + Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", + !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) + if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() + ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 + states.Dump log /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does - let start(ctx : CosmosContext, writerQueueLen, writerCount, readerQueueLen) = async { + static member Start(log, ctx, writerQueueLen, writerCount, readerQueueLen) = async { let! ct = Async.CancellationToken - let writer = Writer.WriteQueue(ctx, writerQueueLen, ct) - let queue = SyncQueue(Log.Logger, writer, ct, readerQueueLen) - let _ = writer.Result.Subscribe queue.HandleWriteResult // codependent, wont worry about unsubcribing - writer.StartConsumers writerCount - let! _ = Async.StartChild(async { queue.Pump() }) - return queue + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount, writerQueueLen) + let instance = Coordinator(log, writers, ct, readerQueueLen) + let! _ = Async.StartChild <| writers.Pump() + let! _ = Async.StartChild(async { instance.Pump() }) + return instance } type EventStore.ClientAPI.RecordedEvent with @@ -459,7 +245,7 @@ type EventStore.ClientAPI.RecordedEvent with module EventStoreReader = open EventStore.ClientAPI - let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = Ingester.arrayBytes x.Data + Ingester.arrayBytes x.Metadata + let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = CosmosIngester.arrayBytes x.Data + CosmosIngester.arrayBytes x.Metadata let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 type SliceStatsBuffer(?interval) = @@ -469,7 +255,7 @@ module EventStoreReader = lock recentCats <| fun () -> let mutable batchBytes = 0 for x in slice.Events do - let cat = Ingester.category x.OriginalStreamId + let cat = CosmosIngester.category x.OriginalStreamId let eventBytes = esPayloadBytes x match recentCats.TryGetValue cat with | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) @@ -547,20 +333,9 @@ module EventStoreReader = Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 return Option.get max } - let pullStream (conn : IEventStoreConnection, batchSize) stream (postBatch : Ingester.Batch -> unit) = - let rec fetchFrom pos = async { - let! currentSlice = conn.ReadStreamEventsBackwardAsync(stream, pos, batchSize, resolveLinkTos=true) |> Async.AwaitTaskCorrect - if currentSlice.IsEndOfStream then return () else - let events = - [| for x in currentSlice.Events -> - let e = x.Event - Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] - postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } - return! fetchFrom currentSlice.NextEventNumber } - fetchFrom 0L type [] PullResult = Exn of exn: exn | Eof | EndOfTranche - type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : Ingester.Batch -> unit) = + type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : CosmosIngester.Batch -> unit) = member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?ignoreEmptyEof) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec loop () = async { @@ -596,9 +371,7 @@ module EventStoreReader = with e -> return Exn e } type [] Work = - | Stream of name: string * batchSize: int | Tranche of range: Range * batchSize : int - | Tail of pos: Position * interval: TimeSpan * batchSize : int type FeedQueue(batchSize, minBatchSize, max, ?statsInterval) = let work = ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) @@ -607,25 +380,11 @@ module EventStoreReader = work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) member __.AddTranche(pos, nextPos, ?batchSizeOverride) = __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) - member __.AddStream(name, ?batchSizeOverride) = - work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) - member __.AddTail(pos, interval, ?batchSizeOverride) = - work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() member __.Process(conn, enumEvents, postBatch, work) = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize match work with - | Stream (name,batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - Log.Warning("Reading stream; batch size {bs}", batchSize) - try do! pullStream (conn, batchSize) name postBatch - Log.Warning("completed stream") - with e -> - let bs = adjust batchSize - Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - __.AddStream(name, bs) - return false | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Commencing tranche, batch size {bs}", batchSize) @@ -645,44 +404,10 @@ module EventStoreReader = Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) __.OverallStats.DumpIfIntervalExpired() __.AddTranche(range, bs) - return false - | Tail (pos, interval, batchSize) -> - let mutable first, count, batchSize, range = true, 0, batchSize, Range(pos,None, Position.Start) - let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) - let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds - let progressSw, tailSw = Stopwatch.StartNew(), Stopwatch.StartNew() - let reader = ReaderGroup(conn, enumEvents, postBatch) - let slicesStats, stats = SliceStatsBuffer(), OverallStats() - while true do - let currentPos = range.Current - use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") - if first then - first <- false - Log.Warning("Tailing at {interval}s interval", interval.TotalSeconds) - elif progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Warning("Performed {count} tails to date @ {pos} chunk {chunk}", count, currentPos.CommitPosition, chunk currentPos) - progressSw.Restart() - count <- count + 1 - let! res = reader.Pump(range,batchSize,slicesStats,stats,ignoreEmptyEof=true) - stats.DumpIfIntervalExpired() - match tailIntervalMs - tailSw.ElapsedMilliseconds with - | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) - | _ -> () - tailSw.Restart() - match res with - | PullResult.EndOfTranche | PullResult.Eof -> () - | PullResult.Exn e -> - batchSize <- adjust batchSize - Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) - return true } + return false } - type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : Ingester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = + type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : CosmosIngester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = let work = FeedQueue(spec.batchSize, spec.minBatchSize, max, ?statsInterval=statsInterval) - do match spec.tailInterval with - | Some interval -> work.AddTail(max, interval) - | None -> () - for s in spec.streams do - work.AddStream s let mutable remainder = let startPos = match spec.start with @@ -690,18 +415,16 @@ module EventStoreReader = | Absolute p -> Position(p, 0L) | Chunk c -> posFromChunk c | Percentage pct -> posFromPercentage (pct, max) - | Ignore -> max Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) - if spec.start = Ignore then None + if spec.start = StartPos.Start then None else let nextPos = posFromChunkAfter startPos work.AddTranche(startPos, nextPos) Some nextPos member __.Pump () = async { - (*if spec.tail then enqueue tail work*) - let maxDop = spec.stripes + Option.count spec.tailInterval + let maxDop = spec.stripes let dop = new SemaphoreSlim(maxDop) let mutable finished = false while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do @@ -748,15 +471,16 @@ let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { || e.EventStreamId.EndsWith("_checkpoints") || e.EventStreamId.EndsWith("_checkpoint") -> Choice2Of2 e - | e when eb > Ingester.cosmosPayloadLimit -> - Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, Ingester.cosmosPayloadLimit) + | e when eb > CosmosIngester.cosmosPayloadLimit -> + Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", + e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, CosmosIngester.cosmosPayloadLimit) Choice2Of2 e | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) } let run (ctx : Equinox.Cosmos.Core.CosmosContext) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { - let! ingester = Ingester.start(ctx, writerQueueLen, writerCount, readerQueueLen) - let! _feeder = EventStoreReader.start(source.ReadConnection, spec, enumEvents, ingester.Add) + let! coodinator = Coordinator.Start(Log.Logger, ctx, writerQueueLen, writerCount, readerQueueLen) + let! _ = EventStoreReader.start(source.ReadConnection, spec, enumEvents, coodinator.Add) do! Async.AwaitKeyboardInterrupt() } [] diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 049678b08..46e03143a 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -129,6 +129,17 @@ type CatStats() = member __.Clear() = cats.Clear() member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd +module Queue = + let tryDequeue (x : System.Collections.Generic.Queue<'T>) = +#if NET461 + if x.Count = 0 then None + else x.Dequeue() |> Some +#else + match x.TryDequeue() with + | false, _ -> None + | true, res -> Some res +#endif + type ResultKind = TimedOut | RateLimited | Malformed | Ok type StreamStates() = diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 3cb6f7255..30f313320 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -5,7 +5,6 @@ open Equinox.Store // AwaitTaskCorrect open System open System.Threading open System.Threading.Tasks -open System.Collections.Generic #nowarn "21" // re AwaitKeyboardInterrupt #nowarn "40" // re AwaitKeyboardInterrupt @@ -33,7 +32,7 @@ module Queue = type SemaphoreSlim with /// F# friendly semaphore await function - member semaphore.Await(?timeout : TimeSpan) = async { + member semaphore.Await(?timeout : System.TimeSpan) = async { let! ct = Async.CancellationToken let timeout = defaultArg timeout Timeout.InfiniteTimeSpan let task = semaphore.WaitAsync(timeout, ct) @@ -45,39 +44,4 @@ type SemaphoreSlim with let! _ = semaphore.Await() try return! workflow finally semaphore.Release() |> ignore - } - -type RefCounted<'T> = { mutable refCount: int; value: 'T } - -// via https://stackoverflow.com/a/31194647/11635 -type SemaphorePool(gen : unit -> SemaphoreSlim) = - let inners: Dictionary> = Dictionary() - - let getOrCreateSlot key = - lock inners <| fun () -> - match inners.TryGetValue key with - | true, inner -> - inner.refCount <- inner.refCount + 1 - inner.value - | false, _ -> - let value = gen () - inners.[key] <- { refCount = 1; value = value } - value - let slotReleaseGuard key : IDisposable = - { new System.IDisposable with - member __.Dispose() = - lock inners <| fun () -> - let item = inners.[key] - match item.refCount with - | 1 -> inners.Remove key |> ignore - | current -> item.refCount <- current - 1 } - - member __.ExecuteAsync(k,f) = async { - let x = getOrCreateSlot k - use _ = slotReleaseGuard k - return! f x } - - member __.Execute(k,f) = - let x = getOrCreateSlot k - use _l = slotReleaseGuard k - f x \ No newline at end of file + } \ No newline at end of file From 55cd0e746862bf2e2ea6a2b4bf0076758e566502 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 14 Apr 2019 15:57:13 +0100 Subject: [PATCH 047/353] Move Ingest to Sync Template --- equinox-ingest/Ingest/Infrastructure.fs | 36 ------------------- equinox-ingest/Ingest/Ingest.fsproj | 26 -------------- equinox-sync/Ingest/Ingest.fsproj | 17 +++++++++ .../Ingest/Program.fs | 0 equinox-sync/Sync/CosmosIngester.fs | 11 ------ equinox-sync/equinox-sync.sln | 6 ++++ 6 files changed, 23 insertions(+), 73 deletions(-) delete mode 100644 equinox-ingest/Ingest/Infrastructure.fs delete mode 100644 equinox-ingest/Ingest/Ingest.fsproj create mode 100644 equinox-sync/Ingest/Ingest.fsproj rename {equinox-ingest => equinox-sync}/Ingest/Program.fs (100%) diff --git a/equinox-ingest/Ingest/Infrastructure.fs b/equinox-ingest/Ingest/Infrastructure.fs deleted file mode 100644 index 816652591..000000000 --- a/equinox-ingest/Ingest/Infrastructure.fs +++ /dev/null @@ -1,36 +0,0 @@ -[] -module private Infrastructure - -open Equinox.Store // AwaitTaskCorrect -open System -open System.Threading -open System.Threading.Tasks - -type SemaphoreSlim with - /// F# friendly semaphore await function - member semaphore.Await(?timeout : TimeSpan) = async { - let! ct = Async.CancellationToken - let timeout = defaultArg timeout Timeout.InfiniteTimeSpan - let task = semaphore.WaitAsync(timeout, ct) - return! Async.AwaitTaskCorrect task - } - - /// Throttling wrapper which waits asynchronously until the semaphore has available capacity - member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { - let! _ = semaphore.Await() - try return! workflow - finally semaphore.Release() |> ignore - } - -#nowarn "21" // re AwaitKeyboardInterrupt -#nowarn "40" // re AwaitKeyboardInterrupt - -type Async with - static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) - /// Asynchronously awaits the next keyboard interrupt event - static member AwaitKeyboardInterrupt () : Async = - Async.FromContinuations(fun (sc,_,_) -> - let isDisposed = ref 0 - let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore - and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) \ No newline at end of file diff --git a/equinox-ingest/Ingest/Ingest.fsproj b/equinox-ingest/Ingest/Ingest.fsproj deleted file mode 100644 index 094ebc154..000000000 --- a/equinox-ingest/Ingest/Ingest.fsproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - Exe - netcoreapp2.1;net461 - 5 - $(DefineConstants);NET461 - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/equinox-sync/Ingest/Ingest.fsproj b/equinox-sync/Ingest/Ingest.fsproj new file mode 100644 index 000000000..9f8ae20d5 --- /dev/null +++ b/equinox-sync/Ingest/Ingest.fsproj @@ -0,0 +1,17 @@ + + + + Exe + netcoreapp2.1 + 5 + + + + + + + + + + + \ No newline at end of file diff --git a/equinox-ingest/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs similarity index 100% rename from equinox-ingest/Ingest/Program.fs rename to equinox-sync/Ingest/Program.fs diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 46e03143a..049678b08 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -129,17 +129,6 @@ type CatStats() = member __.Clear() = cats.Clear() member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd -module Queue = - let tryDequeue (x : System.Collections.Generic.Queue<'T>) = -#if NET461 - if x.Count = 0 then None - else x.Dequeue() |> Some -#else - match x.TryDequeue() with - | false, _ -> None - | true, res -> Some res -#endif - type ResultKind = TimedOut | RateLimited | Malformed | Ok type StreamStates() = diff --git a/equinox-sync/equinox-sync.sln b/equinox-sync/equinox-sync.sln index 857f5d668..be656b901 100644 --- a/equinox-sync/equinox-sync.sln +++ b/equinox-sync/equinox-sync.sln @@ -11,6 +11,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "Ingest\Ingest.fsproj", "{BB7079A7-53E8-4843-8981-78DD025F8C91}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -25,6 +27,10 @@ Global {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Debug|Any CPU.Build.0 = Debug|Any CPU {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Release|Any CPU.ActiveCfg = Release|Any CPU {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Release|Any CPU.Build.0 = Release|Any CPU + {BB7079A7-53E8-4843-8981-78DD025F8C91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {BB7079A7-53E8-4843-8981-78DD025F8C91}.Debug|Any CPU.Build.0 = Debug|Any CPU + {BB7079A7-53E8-4843-8981-78DD025F8C91}.Release|Any CPU.ActiveCfg = Release|Any CPU + {BB7079A7-53E8-4843-8981-78DD025F8C91}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From d04917daee24abc438b5ab776e61af3c6d896194 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 14 Apr 2019 16:11:51 +0100 Subject: [PATCH 048/353] remove redundant solution folder --- equinox-ingest/equinox-ingest.sln | 24 ------------------------ 1 file changed, 24 deletions(-) delete mode 100644 equinox-ingest/equinox-ingest.sln diff --git a/equinox-ingest/equinox-ingest.sln b/equinox-ingest/equinox-ingest.sln deleted file mode 100644 index 61e2c6bef..000000000 --- a/equinox-ingest/equinox-ingest.sln +++ /dev/null @@ -1,24 +0,0 @@ -Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio Version 16 -VisualStudioVersion = 16.0.28714.193 -MinimumVisualStudioVersion = 10.0.40219.1 -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "Ingest\Ingest.fsproj", "{AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}" -EndProject -Global - GlobalSection(SolutionConfigurationPlatforms) = preSolution - Debug|Any CPU = Debug|Any CPU - Release|Any CPU = Release|Any CPU - EndGlobalSection - GlobalSection(ProjectConfigurationPlatforms) = postSolution - {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AFEB8BA3-F506-4137-A7C2-DBE2922C9C91}.Release|Any CPU.Build.0 = Release|Any CPU - EndGlobalSection - GlobalSection(SolutionProperties) = preSolution - HideSolutionNode = FALSE - EndGlobalSection - GlobalSection(ExtensibilityGlobals) = postSolution - SolutionGuid = {8D7BF395-42EA-45B4-B353-040636AD73CE} - EndGlobalSection -EndGlobal From d45cc3b317e89e67a03dd6149589b13d4a53a109 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 15 Apr 2019 13:12:50 +0100 Subject: [PATCH 049/353] Rebase Ingester on Sync EventStoreSource --- equinox-sync/Ingest/Program.fs | 356 ++++------------ equinox-sync/Sync/EventStoreSource.fs | 270 ++++++++++++ equinox-sync/Sync/Program.fs | 591 ++++++++------------------ equinox-sync/Sync/Sync.fsproj | 1 + 4 files changed, 520 insertions(+), 698 deletions(-) create mode 100644 equinox-sync/Sync/EventStoreSource.fs diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 7b41878be..52901a59d 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -1,15 +1,15 @@ module IngestTemplate.Program -open Equinox.Store // Infra -open SyncTemplate -open FSharp.Control open Serilog open System open System.Collections.Concurrent open System.Diagnostics open System.Threading -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Start +open SyncTemplate +open EventStoreSource + +type StartPos = Position of int64 | Chunk of int | Percentage of float | StreamList of string list | Start type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; minBatchSize: int } let mb x = float x / 1024. / 1024. @@ -124,6 +124,7 @@ module CmdParser = | [] Chunk of int | [] Percent of float | [] Stripes of int + | [] Stream of string | [] Es of ParseResults interface IArgParserTemplate with member a.Usage = @@ -133,10 +134,11 @@ module CmdParser = | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | Position _ -> "EventStore $all Stream Position to commence from" + | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | Stripes _ -> "number of concurrent readers" + | Stream _ -> "specific stream(s) to read" | Es _ -> "specify EventStore parameters" and Arguments(args : ParseResults) = member val EventStore = EventStore.Arguments(args.GetResult Es) @@ -147,13 +149,15 @@ module CmdParser = member __.MinBatchSize = args.GetResult(MinBatchSize,512) member __.Stripes = args.GetResult(Stripes,1) member x.BuildFeedParams() : ReaderSpec = - Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = match args.TryGetResult Position, args.TryGetResult Chunk, args.TryGetResult Percent with - | Some p, _, _ -> Log.Warning("Processing will commence at $all Position {p}", p); Absolute p - | _, Some c, _ -> Log.Warning("Processing will commence at $all Chunk {c}", c); StartPos.Chunk c - | _, _, Some p -> Log.Warning("Processing will commence at $all Percentage {pct:P}", p/100.); Percentage p - | None, None, None -> Log.Warning "Processing will commence at $all Start"; Start + | Some p, _, _ -> StartPos.Position p + | _, Some c, _ -> StartPos.Chunk c + | _, _, Some p -> Percentage p + | None, None, None when args.GetResults Stream <> [] -> StreamList (args.GetResults Stream) + | None, None, None -> Start + Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes covering from {startPos}", + x.MinBatchSize, x.StartingBatchSize, x.Stripes, startPos) { start = startPos; batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args @@ -162,19 +166,53 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -// Illustrates how to emit direct to the Console using Serilog -// Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog -module Logging = - let initialize verbose consoleMinLevel maybeSeqEndpoint = - Log.Logger <- - LoggerConfiguration() - .Destructure.FSharpTypes() - .Enrich.FromLogContext() - |> fun c -> if verbose then c.MinimumLevel.Debug() else c - |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {NewLine}{Exception}" - c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) - |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) - |> fun c -> c.CreateLogger() +type Readers(conn, spec : ReaderSpec, tryMapEvent, postBatch, max : EventStore.ClientAPI.Position, ct : CancellationToken, ?statsInterval) = + let work = ReadQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) + let posFromChunkAfter (pos: EventStore.ClientAPI.Position) = + let nextChunk = 1 + int (chunk pos) + posFromChunk nextChunk + let mutable remainder = + let startAt (startPos : EventStore.ClientAPI.Position) = + Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", + startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition) + let nextPos = posFromChunkAfter startPos + work.AddTranche(startPos, nextPos, max) + Some nextPos + match spec.start with + | Start -> startAt <| EventStore.ClientAPI.Position.Start + | Position p -> startAt <| EventStore.ClientAPI.Position(p, 0L) + | Chunk c -> startAt <| posFromChunk c + | Percentage pct -> startAt <| posFromPercentage (pct, max) + | StreamList streams -> + for s in streams do + work.AddStream s + None + member __.Pump () = async { + let maxDop = spec.stripes + let dop = new SemaphoreSlim(maxDop) + let mutable finished = false + while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do + let! _ = dop.Await() + work.OverallStats.DumpIfIntervalExpired() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let! eof = work.Process(conn, tryMapEvent, postBatch, (fun _ -> true), (fun _pos -> Seq.iter postBatch), task) + if eof then remainder <- None + finally dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + match remainder with + | Some pos -> + let nextPos = posFromChunkAfter pos + remainder <- Some nextPos + do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) + | None -> + if finished then do! Async.Sleep 1000 + else Log.Warning("No further ingestion work to commence") + finished <- true } type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancellationToken: CancellationToken, readerQueueLen, ?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 @@ -187,7 +225,6 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel member __.Pump() = let _ = writers.Result.Subscribe __.HandleWriteResult // codependent, wont worry about unsubcribing let fiveMs = TimeSpan.FromMilliseconds 5. - let mutable pendingWriterAdd = None let mutable bytesPended = 0L let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let badCats = CosmosIngester.CatStats() @@ -220,7 +257,6 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel incr workPended eventsPended := !eventsPended + w.span.events.Length bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) - if progressTimer.ElapsedMilliseconds > intervalMs then progressTimer.Restart() Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", @@ -229,260 +265,33 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 states.Dump log - /// Manages establishing of the writer 'threads' - can be Stop()ped explicitly and/or will stop when caller does - static member Start(log, ctx, writerQueueLen, writerCount, readerQueueLen) = async { + static member Run log conn (spec : ReaderSpec, tryMapEvent) (ctx : Equinox.Cosmos.Core.CosmosContext) (writerQueueLen, writerCount, readerQueueLen) = async { let! ct = Async.CancellationToken + let! max = establishMax conn let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount, writerQueueLen) + let readers = Readers(conn, spec, tryMapEvent, writers.Enqueue, max, ct) let instance = Coordinator(log, writers, ct, readerQueueLen) let! _ = Async.StartChild <| writers.Pump() + let! _ = Async.StartChild <| readers.Pump() let! _ = Async.StartChild(async { instance.Pump() }) - return instance - } - -type EventStore.ClientAPI.RecordedEvent with - member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) - -module EventStoreReader = - open EventStore.ClientAPI - - let inline esRecPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = CosmosIngester.arrayBytes x.Data + CosmosIngester.arrayBytes x.Metadata - let inline esPayloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = esRecPayloadBytes x.Event + x.OriginalStreamId.Length * 2 - - type SliceStatsBuffer(?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let recentCats, accStart = System.Collections.Generic.Dictionary(), Stopwatch.StartNew() - member __.Ingest(slice: AllEventsSlice) = - lock recentCats <| fun () -> - let mutable batchBytes = 0 - for x in slice.Events do - let cat = CosmosIngester.category x.OriginalStreamId - let eventBytes = esPayloadBytes x - match recentCats.TryGetValue cat with - | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) - | false, _ -> recentCats.[cat] <- (1, eventBytes) - batchBytes <- batchBytes + eventBytes - __.DumpIfIntervalExpired() - slice.Events.Length, int64 batchBytes - member __.DumpIfIntervalExpired(?force) = - if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then - lock recentCats <| fun () -> - let log = function - | [||] -> () - | xs -> - xs - |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) - |> Seq.truncate 10 - |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) - |> fun rendered -> Log.Warning("Processed {@cats} (MB/cat/count)", rendered) - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log - recentCats.Clear() - accStart.Restart() - - type OverallStats(?statsInterval) = - let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() - let mutable totalEvents, totalBytes = 0L, 0L - member __.Ingest(batchEvents, batchBytes) = - Interlocked.Add(&totalEvents,batchEvents) |> ignore - Interlocked.Add(&totalBytes,batchBytes) |> ignore - member __.Bytes = totalBytes - member __.Events = totalEvents - member __.DumpIfIntervalExpired(?force) = - if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then - let totalMb = mb totalBytes - Log.Warning("Traversed {events} events {gb:n1}GB {mbs:n2}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) - progressStart.Restart() - - type Range(start, sliceEnd : Position option, max : Position) = - member val Current = start with get, set - member __.TryNext(pos: Position) = - __.Current <- pos - __.IsCompleted - member __.IsCompleted = - match sliceEnd with - | Some send when __.Current.CommitPosition >= send.CommitPosition -> false - | _ -> true - member __.PositionAsRangePercentage = - if max.CommitPosition=0L then Double.NaN - else float __.Current.CommitPosition/float max.CommitPosition + do! Async.AwaitKeyboardInterrupt() } - // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk - let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 - let posFromChunk (chunk: int) = - let chunkBase = int64 chunk * 1024L * 1024L * 256L - Position(chunkBase,0L) - let posFromPercentage (pct,max : Position) = - let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) - let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L - let posFromChunkAfter (pos: Position) = - let nextChunk = 1 + int (chunk pos) - posFromChunk nextChunk - - let fetchMax (conn : IEventStoreConnection) = async { - let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let max = lastItemBatch.NextPosition - Log.Warning("EventStore {chunks} chunks, ~{gb:n1}GB Write Position @ {pos} ", chunk max, mb max.CommitPosition/1024., max.CommitPosition) - return max } - let establishMax (conn : IEventStoreConnection) = async { - let mutable max = None - while Option.isNone max do - try let! max_ = fetchMax conn - max <- Some max_ - with e -> - Log.Warning(e,"Could not establish max position") - do! Async.Sleep 5000 - return Option.get max } - - type [] PullResult = Exn of exn: exn | Eof | EndOfTranche - type ReaderGroup(conn : IEventStoreConnection, enumEvents, postBatch : CosmosIngester.Batch -> unit) = - member __.Pump(range : Range, batchSize, slicesStats : SliceStatsBuffer, overallStats : OverallStats, ?ignoreEmptyEof) = - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let rec loop () = async { - let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let postSw = Stopwatch.StartNew() - let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) - let streams = - enumEvents currentSlice.Events - |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) - |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) - |> Array.ofSeq - let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - let mutable usedEvents = 0 - for stream,streamEvents in streams do - for pos, item in streamEvents do - usedEvents <- usedEvents + 1 - postBatch { stream = stream; span = { index = pos; events = [| item |]}} - if not(ignoreEmptyEof = Some true && batchEvents = 0 && not currentSlice.IsEndOfStream) then // ES doesnt report EOF on the first call :( - Log.Warning("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", - range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, usedEvents, postSw.ElapsedMilliseconds) - let shouldLoop = range.TryNext currentSlice.NextPosition - if shouldLoop && not currentSlice.IsEndOfStream then - sw.Restart() // restart the clock as we hand off back to the Reader - return! loop () - else - return currentSlice.IsEndOfStream } - async { - try let! eof = loop () - return if eof then Eof else EndOfTranche - with e -> return Exn e } - - type [] Work = - | Tranche of range: Range * batchSize : int - type FeedQueue(batchSize, minBatchSize, max, ?statsInterval) = - let work = ConcurrentQueue() - member val OverallStats = OverallStats(?statsInterval=statsInterval) - member val SlicesStats = SliceStatsBuffer() - member __.AddTranche(range, ?batchSizeOverride) = - work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) - member __.AddTranche(pos, nextPos, ?batchSizeOverride) = - __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) - member __.TryDequeue () = - work.TryDequeue() - member __.Process(conn, enumEvents, postBatch, work) = async { - let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize - match work with - | Tranche (range, batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) - Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let reader = ReaderGroup(conn, enumEvents, postBatch) - let! res = reader.Pump(range, batchSize, __.SlicesStats, __.OverallStats) - match res with - | PullResult.EndOfTranche -> - Log.Warning("Completed tranche") - __.OverallStats.DumpIfIntervalExpired() - return false - | PullResult.Eof -> - Log.Warning("REACHED THE END!") - __.OverallStats.DumpIfIntervalExpired(true) - return true - | PullResult.Exn e -> - let bs = adjust batchSize - Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) - __.OverallStats.DumpIfIntervalExpired() - __.AddTranche(range, bs) - return false } - - type Reader(conn : IEventStoreConnection, spec: ReaderSpec, enumEvents, postBatch : CosmosIngester.Batch -> unit, max, ct : CancellationToken, ?statsInterval) = - let work = FeedQueue(spec.batchSize, spec.minBatchSize, max, ?statsInterval=statsInterval) - let mutable remainder = - let startPos = - match spec.start with - | StartPos.Start -> Position.Start - | Absolute p -> Position(p, 0L) - | Chunk c -> posFromChunk c - | Percentage pct -> posFromPercentage (pct, max) - Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", - startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition) - if spec.start = StartPos.Start then None - else - let nextPos = posFromChunkAfter startPos - work.AddTranche(startPos, nextPos) - Some nextPos - - member __.Pump () = async { - let maxDop = spec.stripes - let dop = new SemaphoreSlim(maxDop) - let mutable finished = false - while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do - let! _ = dop.Await() - work.OverallStats.DumpIfIntervalExpired() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! eof = work.Process(conn, enumEvents, postBatch, task) - if eof then remainder <- None - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - match remainder with - | Some pos -> - let nextPos = posFromChunkAfter pos - remainder <- Some nextPos - do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) - | None -> - if finished then do! Async.Sleep 1000 - else Log.Warning("No further ingestion work to commence") - finished <- true } - - let start (conn, spec, enumEvents, postBatch) = async { - let! ct = Async.CancellationToken - let! max = establishMax conn - let reader = Reader(conn, spec, enumEvents, postBatch, max, ct) - let! _ = Async.StartChild <| reader.Pump() - return () - } +// Illustrates how to emit direct to the Console using Serilog +// Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog +module Logging = + let initialize verbose consoleMinLevel maybeSeqEndpoint = + Log.Logger <- + LoggerConfiguration() + .Destructure.FSharpTypes() + .Enrich.FromLogContext() + |> fun c -> if verbose then c.MinimumLevel.Debug() else c + |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {NewLine}{Exception}" + c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) + |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) + |> fun c -> c.CreateLogger() -open Equinox.Cosmos open Equinox.EventStore -let enumEvents (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { - for e in xs -> - let eb = EventStoreReader.esPayloadBytes e - match e.Event with - | e when not e.IsJson - || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.StartsWith("$") - || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") -> - Choice2Of2 e - | e when eb > CosmosIngester.cosmosPayloadLimit -> - Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", - e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, CosmosIngester.cosmosPayloadLimit) - Choice2Of2 e - | e -> Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp)) -} - -let run (ctx : Equinox.Cosmos.Core.CosmosContext) (source : GesConnection) (spec: ReaderSpec) (writerQueueLen, writerCount, readerQueueLen) = async { - let! coodinator = Coordinator.Start(Log.Logger, ctx, writerQueueLen, writerCount, readerQueueLen) - let! _ = EventStoreReader.start(source.ReadConnection, spec, enumEvents, coodinator.Add) - do! Async.AwaitKeyboardInterrupt() } - [] let main argv = try let args = CmdParser.parse argv @@ -492,11 +301,10 @@ let main argv = let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let ctx = - let destination = cosmos.Connect "IngestTemplate" |> Async.RunSynchronously - let colls = CosmosCollections(cosmos.Database, cosmos.Collection) + let destination = cosmos.Connect "SyncTemplate.Ingester" |> Async.RunSynchronously + let colls = Equinox.Cosmos.CosmosCollections(cosmos.Database, cosmos.Collection) Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) - Thread.Sleep(1000) // https://github.com/EventStore/EventStore/issues/1899 - run ctx source readerSpec (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously + Coordinator.Run Log.Logger source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs new file mode 100644 index 000000000..900b58380 --- /dev/null +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -0,0 +1,270 @@ +module SyncTemplate.EventStoreSource + +open Equinox.Store // AwaitTaskCorrect +open EventStore.ClientAPI +open System +open Serilog // NB Needs to shadow ILogger +open System.Diagnostics +open System.Threading +open System.Collections.Generic + +type EventStore.ClientAPI.RecordedEvent with + member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) + +let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = CosmosIngester.arrayBytes x.Data + CosmosIngester.arrayBytes x.Metadata +let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + +let tryToBatch (e : RecordedEvent) : CosmosIngester.Batch option = + let eb = recPayloadBytes e + if eb > CosmosIngester.cosmosPayloadLimit then + Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", + e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, CosmosIngester.cosmosPayloadLimit) + None + else + let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata + let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data + let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ + Some { stream = e.EventStreamId; span = { index = e.EventNumber; events = [| event |]} } + +let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = + match x.Event with + | e when not e.IsJson + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.StartsWith("$") + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || not (catFilter e.EventStreamId) -> None + | e -> tryToBatch e + +let private mb x = float x / 1024. / 1024. + +let category (streamName : string) = streamName.Split([|'-'|],2).[0] + +type OverallStats(?statsInterval) = + let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() + let mutable totalEvents, totalBytes = 0L, 0L + member __.Ingest(batchEvents, batchBytes) = + Interlocked.Add(&totalEvents,batchEvents) |> ignore + Interlocked.Add(&totalBytes,batchBytes) |> ignore + member __.Bytes = totalBytes + member __.Events = totalEvents + member __.DumpIfIntervalExpired(?force) = + if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then + let totalMb = mb totalBytes + Log.Information("EventStore throughput {events} events {gb:n1}GB {mbs:n2}MB/s", + totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + progressStart.Restart() + +type SliceStatsBuffer(?interval) = + let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 + let recentCats, accStart = Dictionary(), Stopwatch.StartNew() + member __.Ingest(slice: AllEventsSlice) = + lock recentCats <| fun () -> + let mutable batchBytes = 0 + for x in slice.Events do + let cat = category x.OriginalStreamId + let eventBytes = payloadBytes x + match recentCats.TryGetValue cat with + | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) + | false, _ -> recentCats.[cat] <- (1, eventBytes) + batchBytes <- batchBytes + eventBytes + __.DumpIfIntervalExpired() + slice.Events.Length, int64 batchBytes + member __.DumpIfIntervalExpired(?force) = + if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then + lock recentCats <| fun () -> + let log = function + | [||] -> () + | xs -> + xs + |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) + |> Seq.truncate 10 + |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) + |> fun rendered -> Log.Information("EventStore categories {@cats} (MB/cat/count)", rendered) + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log + recentCats.Clear() + accStart.Restart() + +type Range(start, sliceEnd : Position option, ?max : Position) = + member val Current = start with get, set + member __.TryNext(pos: Position) = + __.Current <- pos + __.IsCompleted + member __.IsCompleted = + match sliceEnd with + | Some send when __.Current.CommitPosition >= send.CommitPosition -> false + | _ -> true + member __.PositionAsRangePercentage = + match max with + | None -> Double.NaN + | Some max -> float __.Current.CommitPosition/float max.CommitPosition + +// @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk +let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 +let posFromChunk (chunk: int) = + let chunkBase = int64 chunk * 1024L * 1024L * 256L + Position(chunkBase,0L) +let posFromPercentage (pct,max : Position) = + let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) + let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L + +let fetchMax (conn : IEventStoreConnection) = async { + let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect + let max = lastItemBatch.FromPosition + Log.Information("EventStore Write @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) + return max } +let establishMax (conn : IEventStoreConnection) = async { + let mutable max = None + while Option.isNone max do + try let! currentMax = fetchMax conn + max <- Some currentMax + with e -> + Log.Warning(e,"Could not establish max position") + do! Async.Sleep 5000 + return Option.get max } +let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : CosmosIngester.Batch -> unit) = + let rec fetchFrom pos limit = async { + let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize + let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect + let events = + [| for x in currentSlice.Events -> + let e = x.Event + Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] + postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } + match limit with + | None when currentSlice.IsEndOfStream -> return () + | None -> return! fetchFrom currentSlice.NextEventNumber None + | Some limit when events.Length >= limit -> return () + | Some limit -> return! fetchFrom currentSlice.NextEventNumber (Some (limit - events.Length)) } + fetchFrom pos limit + +type [] PullResult = Exn of exn: exn | Eof | EndOfTranche +let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) + (range:Range, once) (tryMapEvent : ResolvedEvent -> CosmosIngester.Batch option) (postBatch : Position -> CosmosIngester.Batch[] -> unit) = + let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch + let rec aux () = async { + let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let postSw = Stopwatch.StartNew() + let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) + let batches = currentSlice.Events |> Seq.choose tryMapEvent |> Array.ofSeq + let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq + let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length + postBatch currentSlice.NextPosition batches + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, + batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) + if range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream then + sw.Restart() // restart the clock as we hand off back to the Reader + return! aux () + else + return currentSlice.IsEndOfStream } + async { + try let! eof = aux () + return if eof then Eof else EndOfTranche + with e -> return Exn e } + +type [] Work = + | Stream of name: string * batchSize: int + | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int + | Tranche of range: Range * batchSize : int + | Tail of pos: Position * interval: TimeSpan * batchSize : int + +type ReadQueue(batchSize, minBatchSize, ?statsInterval) = + let work = System.Collections.Concurrent.ConcurrentQueue() + member val OverallStats = OverallStats(?statsInterval=statsInterval) + member val SlicesStats = SliceStatsBuffer() + member __.QueueCount = work.Count + member __.AddStream(name, ?batchSizeOverride) = + work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = + work.Enqueue <| Work.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) + member __.AddTranche(range, ?batchSizeOverride) = + work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) + member __.AddTranche(pos, nextPos, max, ?batchSizeOverride) = + __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) + member __.AddTail(pos, interval, ?batchSizeOverride) = + work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) + member __.TryDequeue () = + work.TryDequeue() + member __.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, work) = async { + let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize + match work with + | StreamPrefix (name,pos,len,batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) + Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) + try do! pullStream (conn, batchSize) (name, pos, Some len) postItem + Log.Information("completed stream prefix") + with e -> + let bs = adjust batchSize + Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + __.AddStreamPrefix(name, pos, len, bs) + return false + | Stream (name,batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) + Log.Warning("Reading stream; batch size {bs}", batchSize) + try do! pullStream (conn, batchSize) (name,0L,None) postItem + Log.Information("completed stream") + with e -> + let bs = adjust batchSize + Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + __.AddStream(name, bs) + return false + | Tranche (range, batchSize) -> + use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) + Log.Warning("Commencing tranche, batch size {bs}", batchSize) + let! res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch + match res with + | PullResult.EndOfTranche -> + Log.Warning("Completed tranche") + __.OverallStats.DumpIfIntervalExpired() + return false + | PullResult.Eof -> + Log.Warning("REACHED THE END!") + __.OverallStats.DumpIfIntervalExpired(true) + return true + | PullResult.Exn e -> + let bs = adjust batchSize + Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) + __.OverallStats.DumpIfIntervalExpired() + __.AddTranche(range, bs) + return false + | Tail (pos, interval, batchSize) -> + let mutable count, pauses, batchSize, range = 0, 0, batchSize, Range(pos, None) + let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) + let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds + let tailSw = Stopwatch.StartNew() + let awaitInterval = async { + match tailIntervalMs - tailSw.ElapsedMilliseconds with + | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) + | _ -> () + tailSw.Restart() } + let slicesStats, stats = SliceStatsBuffer(), OverallStats() + use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") + let progressSw = Stopwatch.StartNew() + let mutable paused = false + while true do + let currentPos = range.Current + if progressSw.ElapsedMilliseconds > progressIntervalMs then + Log.Information("Tailed {count} times ({pauses} waits) @ {pos} (chunk {chunk})", + count, pauses, currentPos.CommitPosition, chunk currentPos) + progressSw.Restart() + count <- count + 1 + if shouldTail () then + paused <- false + let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch + do! awaitInterval + match res with + | PullResult.EndOfTranche | PullResult.Eof -> () + | PullResult.Exn e -> + batchSize <- adjust batchSize + Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) + else + if not paused then Log.Information("Pausing due to backlog of incomplete batches...") + paused <- true + pauses <- pauses + 1 + do! awaitInterval + stats.DumpIfIntervalExpired() + return true } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index aaef25328..acfe58785 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -13,32 +13,32 @@ open System.Collections.Generic open System.Diagnostics open System.Threading +let mb x = float x / 1024. / 1024. +let category (streamName : string) = streamName.Split([|'-'|],2).[0] +let every ms f = + let timer = Stopwatch.StartNew() + fun () -> + if timer.ElapsedMilliseconds > ms then + f () + timer.Restart() + //#if eventStore -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | Tail +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint + type ReaderSpec = { /// Identifier for this projection and it's state groupName: string /// Start position from which forward reading is to commence // Assuming no stored position start: StartPos - /// Additional streams with which to seed the reading - streams: string list /// Delay when reading yields an empty batch tailInterval: TimeSpan /// Maximum number of stream readers to permit - stripes: int + streamReaders: int /// Initial batch size to use when commencing reading batchSize: int /// Smallest batch size to degrade to in the presence of failures minBatchSize: int } //#endif -let mb x = float x / 1024. / 1024. -let category (streamName : string) = streamName.Split([|'-'|],2).[0] -let every ms f = - let timer = Stopwatch.StartNew() - fun () -> - if timer.ElapsedMilliseconds > ms then - f () - timer.Restart() module CmdParser = open Argu @@ -60,11 +60,10 @@ module CmdParser = | [] ChangeFeedVerbose #else | [] MinBatchSize of int - | [] Stream of string | [] Position of int64 | [] Chunk of int | [] Percent of float - | [] Stripes of int + | [] StreamReaders of int | [] Tail of intervalS: float | [] VerboseConsole #endif @@ -88,11 +87,10 @@ module CmdParser = #else | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | Stream _ -> "specific stream(s) to read" | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | Stripes _ -> "number of concurrent readers. Default: 8" + | StreamReaders _ -> "number of concurrent readers. Default: 8" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | VerboseConsole -> "request Verbose Console Logging. Default: off" | Source _ -> "EventStore input parameters." @@ -112,7 +110,7 @@ module CmdParser = member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) member __.MinBatchSize = a.GetResult(MinBatchSize,512) - member __.Stripes = a.GetResult(Stripes,8) + member __.StreamReaders = a.GetResult(StreamReaders,8) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds #endif @@ -134,19 +132,18 @@ module CmdParser = disco, db, x.LeaseId, x.StartFromHere, x.BatchSize, x.LagFrequency #else member x.BuildFeedParams() : ReaderSpec = - Log.Information("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes", x.MinBatchSize, x.StartingBatchSize, x.Stripes) let startPos = match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent with | Some p, _, _ -> Absolute p | _, Some c, _ -> StartPos.Chunk c | _, _, Some p -> Percentage p - | None, None, None -> StartPos.Tail - Log.Information("Syncing Consumer Group {groupName} in Database {db} Collection {coll}", - x.ConsumerGroupName, x.Destination.Database, x.Destination.Collection) - Log.Information("Ingesting from {startPos} in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", - startPos, x.MinBatchSize, x.StartingBatchSize, x.Stripes) - { groupName = x.ConsumerGroupName; start = startPos; streams = a.GetResults Stream; tailInterval = x.TailInterval - batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } + | None, None, None -> StartPos.TailOrCheckpoint + Log.Information("Processing Consumer Group {groupName} in Database {db} Collection {coll} starting from {startPos}", + x.ConsumerGroupName, x.Destination.Database, x.Destination.Collection, startPos) + Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", + x.MinBatchSize, x.StartingBatchSize, x.StreamReaders) + { groupName = x.ConsumerGroupName; start = startPos; tailInterval = x.TailInterval + batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; streamReaders = x.StreamReaders } #endif and [] SourceParameters = #if cosmos @@ -285,407 +282,153 @@ module CmdParser = parser.ParseCommandLine argv |> Arguments #if !cosmos -type EventStore.ClientAPI.RecordedEvent with - member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) - -module EventStoreSource = - open EventStore.ClientAPI - - let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata - let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 - - type OverallStats(?statsInterval) = - let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() - let mutable totalEvents, totalBytes = 0L, 0L - member __.Ingest(batchEvents, batchBytes) = - Interlocked.Add(&totalEvents,batchEvents) |> ignore - Interlocked.Add(&totalBytes,batchBytes) |> ignore - member __.Bytes = totalBytes - member __.Events = totalEvents - member __.DumpIfIntervalExpired(?force) = - if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then - let totalMb = mb totalBytes - Log.Information("EventStore throughput {events} events {gb:n1}GB {mbs:n2}MB/s", - totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) - progressStart.Restart() - - type SliceStatsBuffer(?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let recentCats, accStart = Dictionary(), Stopwatch.StartNew() - member __.Ingest(slice: AllEventsSlice) = - lock recentCats <| fun () -> - let mutable batchBytes = 0 - for x in slice.Events do - let cat = category x.OriginalStreamId - let eventBytes = payloadBytes x - match recentCats.TryGetValue cat with - | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) - | false, _ -> recentCats.[cat] <- (1, eventBytes) - batchBytes <- batchBytes + eventBytes - __.DumpIfIntervalExpired() - slice.Events.Length, int64 batchBytes - member __.DumpIfIntervalExpired(?force) = - if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then - lock recentCats <| fun () -> - let log = function - | [||] -> () - | xs -> - xs - |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) - |> Seq.truncate 10 - |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) - |> fun rendered -> Log.Information("EventStore categories {@cats} (MB/cat/count)", rendered) - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log - recentCats.Clear() - accStart.Restart() - - type Range(start, sliceEnd : Position option, ?max : Position) = - member val Current = start with get, set - member __.TryNext(pos: Position) = - __.Current <- pos - __.IsCompleted - member __.IsCompleted = - match sliceEnd with - | Some send when __.Current.CommitPosition >= send.CommitPosition -> false - | _ -> true - member __.PositionAsRangePercentage = - match max with - | None -> Double.NaN - | Some max -> float __.Current.CommitPosition/float max.CommitPosition - - // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk - let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 - let posFromChunk (chunk: int) = - let chunkBase = int64 chunk * 1024L * 1024L * 256L - Position(chunkBase,0L) - let posFromPercentage (pct,max : Position) = - let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) - let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L - - let fetchMax (conn : IEventStoreConnection) = async { - let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let max = lastItemBatch.FromPosition - Log.Information("EventStore Write @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) - return max } - let establishMax (conn : IEventStoreConnection) = async { - let mutable max = None - while Option.isNone max do - try let! currentMax = fetchMax conn - max <- Some currentMax - with e -> - Log.Warning(e,"Could not establish max position") - do! Async.Sleep 5000 - return Option.get max } - let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : CosmosIngester.Batch -> unit) = - let rec fetchFrom pos limit = async { - let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize - let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect - let events = - [| for x in currentSlice.Events -> - let e = x.Event - Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] - postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } - match limit with - | None when currentSlice.IsEndOfStream -> return () - | None -> return! fetchFrom currentSlice.NextEventNumber None - | Some limit when events.Length >= limit -> return () - | Some limit -> return! fetchFrom currentSlice.NextEventNumber (Some (limit - events.Length)) } - fetchFrom pos limit - - type [] PullResult = Exn of exn: exn | Eof | EndOfTranche - let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) - (range:Range, once) enumEvents (postBatch : Position -> CosmosIngester.Batch[] -> unit) = - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let rec aux () = async { - let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let postSw = Stopwatch.StartNew() - let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) - let streams = - enumEvents currentSlice.Events - |> Seq.choose (function Choice1Of2 e -> Some e | Choice2Of2 _ -> None) - |> Seq.groupBy (fun (streamId,_eventNumber,_eventData) -> streamId) - |> Seq.map (fun (streamId,xs) -> streamId, [| for _s, i, e in xs -> i, e |]) - |> Array.ofSeq - let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - let events : CosmosIngester.Batch[] = - [| for stream,streamEvents in streams do - for pos, item in streamEvents do - yield { stream = stream; span = { index = pos; events = [| item |]}} |] - //Log.Information("Bat {bat}", events) - postBatch currentSlice.NextPosition events - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", - range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, events.Length, postSw.ElapsedMilliseconds) - if range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream then - sw.Restart() // restart the clock as we hand off back to the Reader - return! aux () - else - return currentSlice.IsEndOfStream } - async { - try let! eof = aux () - return if eof then Eof else EndOfTranche - with e -> return Exn e } - - type [] Work = - | Stream of name: string * batchSize: int - | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int - | Tail of pos: Position * interval: TimeSpan * batchSize : int - type ReadQueue(batchSize, minBatchSize, ?statsInterval) = - let work = System.Collections.Concurrent.ConcurrentQueue() - member val OverallStats = OverallStats(?statsInterval=statsInterval) - member val SlicesStats = SliceStatsBuffer() - member __.QueueCount = work.Count - member __.AddStream(name, ?batchSizeOverride) = - work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) - member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = - work.Enqueue <| Work.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) - member __.AddTail(pos, interval, ?batchSizeOverride) = - work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) - member __.TryDequeue () = - work.TryDequeue() - member __.Process(conn, enumEvents, postItem, shouldTail, postTail, work) = async { - let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize - match work with - | StreamPrefix (name,pos,len,batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) - try do! pullStream (conn, batchSize) (name, pos, Some len) postItem - Log.Information("completed stream prefix") - with e -> - let bs = adjust batchSize - Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - __.AddStreamPrefix(name, pos, len, bs) - return false - | Stream (name,batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - Log.Warning("Reading stream; batch size {bs}", batchSize) - try do! pullStream (conn, batchSize) (name,0L,None) postItem - Log.Information("completed stream") - with e -> - let bs = adjust batchSize - Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - __.AddStream(name, bs) - return false - | Tail (pos, interval, batchSize) -> - let mutable count, pauses, batchSize, range = 0, 0, batchSize, Range(pos, None) - let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) - let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds - let tailSw = Stopwatch.StartNew() - let awaitInterval = async { - match tailIntervalMs - tailSw.ElapsedMilliseconds with - | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) - | _ -> () - tailSw.Restart() } - let slicesStats, stats = SliceStatsBuffer(), OverallStats() - use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") - let progressSw = Stopwatch.StartNew() - let mutable paused = false - while true do - let currentPos = range.Current - if progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Information("Tailed {count} times ({pauses} waits) @ {pos} (chunk {chunk})", - count, pauses, currentPos.CommitPosition, chunk currentPos) - progressSw.Restart() - count <- count + 1 - if shouldTail () then - paused <- false - let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) enumEvents postTail - do! awaitInterval - match res with - | PullResult.EndOfTranche | PullResult.Eof -> () - | PullResult.Exn e -> - batchSize <- adjust batchSize - Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) - else - if not paused then Log.Information("Pausing due to backlog of incomplete batches...") - paused <- true - pauses <- pauses + 1 - do! awaitInterval - stats.DumpIfIntervalExpired() - return true } - - type Readers(conn : IEventStoreConnection, spec : ReaderSpec, enumEvents, max, ?statsInterval) = - let sleepIntervalMs = 100 - let maxDop = spec.stripes + 1 - let dop = new SemaphoreSlim(maxDop) - let work = ReadQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) - do let startPos = - match spec.start with - | StartPos.Tail -> max - | Absolute p -> Position(p, 0L) - | Chunk c -> posFromChunk c - | Percentage pct -> posFromPercentage (pct, max) - work.AddTail(startPos, spec.tailInterval) - match spec.streams with - | [] -> () - | streams -> - Log.Information("EventStore Additional Streams {streams}", streams) - for s in streams do - work.AddStream s - Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", - startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) - member __.HasCapacity = work.QueueCount < dop.CurrentCount - member __.AddStreamPrefix(stream, pos, len) = - work.AddStreamPrefix(stream, pos, len) - __.HasCapacity - member __.Pump(postItem, shouldTail, postTailBatch) = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - work.OverallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, enumEvents, postItem, shouldTail, postTailBatch, task) in () - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - dop.Release() |> ignore - do! Async.Sleep sleepIntervalMs } - - type [] CoordinationWork<'Pos> = - | Result of CosmosIngester.Writer.Result - | Unbatched of CosmosIngester.Batch - | BatchWithTracking of 'Pos * CosmosIngester.Batch[] - - type Coordinator(log : Serilog.ILogger, readers : Readers, cosmosContext, maxWriters, ?interval, ?maxPendingBatches) = - let maxPendingBatches = defaultArg maxPendingBatches 32 - let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let sleepIntervalMs = 100 - let work = System.Collections.Concurrent.ConcurrentQueue() - let buffer = CosmosIngester.StreamStates() - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) - let tailSyncState = ProgressBatcher.State() - // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here - let mutable pendingBatchCount = 0 - let shouldThrottle () = pendingBatchCount > maxPendingBatches - let mutable progressEpoch = None - let pumpReaders = - let postWrite = work.Enqueue << CoordinationWork.Unbatched - let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) - readers.Pump(postWrite, not << shouldThrottle, postBatch) - let pumpWriters = writers.Pump() - let postWriteResult = work.Enqueue << CoordinationWork.Result - member __.Pump () = async { - use _ = writers.Result.Subscribe postWriteResult - let! _ = Async.StartChild pumpReaders - let! _ = Async.StartChild pumpWriters - let! ct = Async.CancellationToken - let writerResultLog = log.ForContext() - let mutable bytesPended = 0L - let workPended, eventsPended = ref 0, ref 0 - let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 - let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 - let badCats = CosmosIngester.CatStats() - let dumpStats () = - if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then - Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", - !rateLimited, !timedOut, !malformed) - rateLimited := 0; timedOut := 0; malformed := 0 - if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} ok {ok} redundant {dup} partial {partial} Missing {prefix} Exceptions {exns} reqs; Egress {gb:n3}GB", - !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) - workPended := 0; eventsPended := 0 - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; - - let throttle = shouldThrottle () - let level = if not throttle then Events.LogEventLevel.Debug else Events.LogEventLevel.Information - Log.Write(level, "Pending Batches {pb}", pendingBatchCount) - match progressEpoch with - | None -> () - | Some (epoch : Position) -> log.Information("Progress Epoch @ {epoch}", epoch.CommitPosition) - - buffer.Dump log - let tryDumpStats = every statsIntervalMs dumpStats - let handle = function - | CoordinationWork.Unbatched item -> +type [] CoordinationWork<'Pos> = + | Result of CosmosIngester.Writer.Result + | Unbatched of CosmosIngester.Batch + | BatchWithTracking of 'Pos * CosmosIngester.Batch[] + +type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> CosmosIngester.Batch option, maxDop, ?statsInterval) = + let sleepIntervalMs = 100 + let dop = new SemaphoreSlim(maxDop) + let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) + member __.HasCapacity = work.QueueCount < dop.CurrentCount + member __.AddTail(startPos, interval) = work.AddTail(startPos, interval) + member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) + member __.Pump(postItem, shouldTail, postBatch) = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let! _ = work.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, task) in () + finally dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + dop.Release() |> ignore + do! Async.Sleep sleepIntervalMs } + +type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, ?interval, ?maxPendingBatches) = + let maxPendingBatches = defaultArg maxPendingBatches 32 + let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 + let sleepIntervalMs = 100 + let work = System.Collections.Concurrent.ConcurrentQueue() + let buffer = CosmosIngester.StreamStates() + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) + let tailSyncState = ProgressBatcher.State() + // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here + let mutable pendingBatchCount = 0 + let shouldThrottle () = pendingBatchCount > maxPendingBatches + let mutable progressEpoch = None + let pumpReaders = + let postWrite = work.Enqueue << CoordinationWork.Unbatched + let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) + readers.Pump(postWrite, not << shouldThrottle, postBatch) + let pumpWriters = writers.Pump() + let postWriteResult = work.Enqueue << CoordinationWork.Result + member __.Pump () = async { + use _ = writers.Result.Subscribe postWriteResult + let! _ = Async.StartChild pumpReaders + let! _ = Async.StartChild pumpWriters + let! ct = Async.CancellationToken + let writerResultLog = log.ForContext() + let mutable bytesPended = 0L + let workPended, eventsPended = ref 0, ref 0 + let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 + let badCats = CosmosIngester.CatStats() + let dumpStats () = + if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then + Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", + !rateLimited, !timedOut, !malformed) + rateLimited := 0; timedOut := 0; malformed := 0 + if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn + Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} ok {ok} redundant {dup} partial {partial} Missing {prefix} Exceptions {exns} reqs; Egress {gb:n3}GB", + !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) + workPended := 0; eventsPended := 0 + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + + let throttle = shouldThrottle () + let level = if not throttle then Events.LogEventLevel.Debug else Events.LogEventLevel.Information + Log.Write(level, "Pending Batches {pb}", pendingBatchCount) + match progressEpoch with + | None -> () + | Some (epoch : EventStore.ClientAPI.Position) -> log.Information("Progress Epoch @ {epoch}", epoch.CommitPosition) + + buffer.Dump log + let tryDumpStats = every statsIntervalMs dumpStats + let handle = function + | CoordinationWork.Unbatched item -> + buffer.Add item |> ignore + | CoordinationWork.BatchWithTracking(pos, items) -> + for item in items do buffer.Add item |> ignore - | CoordinationWork.BatchWithTracking(pos, items) -> - for item in items do - buffer.Add item |> ignore - tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) - | CoordinationWork.Result res -> - match res with - | CosmosIngester.Writer.Result.Ok _ -> incr resultOk - | CosmosIngester.Writer.Result.Duplicate _ -> incr resultDup - | CosmosIngester.Writer.Result.PartialDuplicate _ -> incr resultPartialDup - | CosmosIngester.Writer.Result.PrefixMissing _ -> incr resultPrefix - | CosmosIngester.Writer.Result.Exn _ -> incr resultExn - - let (stream, updatedState), kind = buffer.HandleWriteResult res - match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) - match kind with - | CosmosIngester.Ok -> res.WriteTo writerResultLog - | CosmosIngester.RateLimited -> incr rateLimited - | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed - | CosmosIngester.TimedOut -> incr timedOut - let queueWrite (w : CosmosIngester.Batch) = - incr workPended - eventsPended := !eventsPended + w.span.events.Length - bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) - writers.Enqueue w - writers.HasCapacity - while not ct.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write results to buffer + Progress - match work.TryDequeue() with - | true, item -> - handle item - | false, _ -> - // 2. Mark off any progress achieved - let _validatedPos, _pendingBatchCount = tailSyncState.Validate buffer.TryGetStreamWritePos - pendingBatchCount <- _pendingBatchCount - progressEpoch <- _validatedPos - // 3. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - let mutable more = readers.HasCapacity - while more do - match buffer.TryGap() with - | Some (stream,pos,len) -> more <- readers.AddStreamPrefix(stream,pos,len) - | None -> more <- false - // 4. After that, [over] provision writers queue - let mutable more = writers.HasCapacity - while more do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> more <- queueWrite w - | None -> (); more <- false - // 5. Periodically emit status info - tryDumpStats () - // TODO trigger periodic progress writing - // 7. Sleep if - do! Async.Sleep sleepIntervalMs } - - let start (log : Serilog.ILogger) (conn, spec, enumEvents) (maxWriters, cosmosContext) = async { - let! max = establishMax conn - let readers = Readers(conn, spec, enumEvents, max) + tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) + | CoordinationWork.Result res -> + match res with + | CosmosIngester.Writer.Result.Ok _ -> incr resultOk + | CosmosIngester.Writer.Result.Duplicate _ -> incr resultDup + | CosmosIngester.Writer.Result.PartialDuplicate _ -> incr resultPartialDup + | CosmosIngester.Writer.Result.PrefixMissing _ -> incr resultPrefix + | CosmosIngester.Writer.Result.Exn _ -> incr resultExn + + let (stream, updatedState), kind = buffer.HandleWriteResult res + match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) + match kind with + | CosmosIngester.Ok -> res.WriteTo writerResultLog + | CosmosIngester.RateLimited -> incr rateLimited + | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed + | CosmosIngester.TimedOut -> incr timedOut + let queueWrite (w : CosmosIngester.Batch) = + incr workPended + eventsPended := !eventsPended + w.span.events.Length + bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) + writers.Enqueue w + while not ct.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write results to buffer + Progress + match work.TryDequeue() with + | true, item -> + handle item + | false, _ -> + // 2. Mark off any progress achieved + let _validatedPos, _pendingBatchCount = tailSyncState.Validate buffer.TryGetStreamWritePos + pendingBatchCount <- _pendingBatchCount + progressEpoch <- _validatedPos + // 3. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + let mutable more = true + while more && readers.HasCapacity do + match buffer.TryGap() with + | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + | None -> more <- false + // 4. After that, [over] provision writers queue + let mutable more = true + while more && writers.HasCapacity do + match buffer.TryReady(writers.IsStreamBusy) with + | Some w -> queueWrite w + | None -> (); more <- false + // 5. Periodically emit status info + tryDumpStats () + // TODO trigger periodic progress writing + // 7. Sleep if + do! Async.Sleep sleepIntervalMs } + + static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext) = async { + let! max = EventStoreSource.establishMax conn + let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) let coordinator = Coordinator(log, readers, cosmosContext, maxWriters) + let startPos = + match spec.start with + | Absolute p -> EventStore.ClientAPI.Position(p, 0L) + | Chunk c -> EventStoreSource.posFromChunk c + | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) + | TailOrCheckpoint -> max + Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", + startPos.CommitPosition, EventStoreSource.chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) do! coordinator.Pump () } - -let enumEvents catFilter (xs : EventStore.ClientAPI.ResolvedEvent[]) = seq { - for e in xs -> - let eb = EventStoreSource.payloadBytes e - match e.Event with - | e when not e.IsJson - || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.StartsWith("$") - || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") - || not (catFilter e.EventStreamId) -> - Choice2Of2 e - | e when eb > CosmosIngester.cosmosPayloadLimit -> - Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", - e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, CosmosIngester.cosmosPayloadLimit) - Choice2Of2 e - | e -> - //if category e.EventStreamId = "ReloadBatchId" then Log.Information("RBID {s}", System.Text.Encoding.UTF8.GetString(e.Data)) - let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata - let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data - Choice1Of2 (e.EventStreamId, e.EventNumber, Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp)) -} //#else module CosmosSource = open Microsoft.Azure.Documents @@ -977,7 +720,7 @@ let main argv = let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() - EventStoreSource.start log (esConnection.ReadConnection, spec, enumEvents catFilter) (16, target) + Coordinator.Run log (esConnection.ReadConnection, spec, EventStoreSource.tryMapEvent catFilter) (16, target) #endif |> Async.RunSynchronously 0 diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 60efad412..da29aa124 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -10,6 +10,7 @@ + From a67f0b592b186ccc6eec50629cdbc7ec04dbde9a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 15 Apr 2019 14:36:36 +0100 Subject: [PATCH 050/353] fix clean diff --- equinox-sync/Sync/Infrastructure.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 30f313320..21fbdb72a 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -32,7 +32,7 @@ module Queue = type SemaphoreSlim with /// F# friendly semaphore await function - member semaphore.Await(?timeout : System.TimeSpan) = async { + member semaphore.Await(?timeout : TimeSpan) = async { let! ct = Async.CancellationToken let timeout = defaultArg timeout Timeout.InfiniteTimeSpan let task = semaphore.WaitAsync(timeout, ct) From ed1d2b108758220b2ecb0595ea07396a6db71d77 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 11:40:55 +0100 Subject: [PATCH 051/353] Add Progress writing for Sync --- equinox-sync/Sync/Checkpoint.fs | 130 +++++++++ equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/EventStoreSource.fs | 23 +- equinox-sync/Sync/Program.fs | 401 +++++++++++++++----------- equinox-sync/Sync/Sync.fsproj | 2 + 5 files changed, 378 insertions(+), 180 deletions(-) create mode 100644 equinox-sync/Sync/Checkpoint.fs diff --git a/equinox-sync/Sync/Checkpoint.fs b/equinox-sync/Sync/Checkpoint.fs new file mode 100644 index 000000000..8a703df01 --- /dev/null +++ b/equinox-sync/Sync/Checkpoint.fs @@ -0,0 +1,130 @@ +module SyncTemplate.Checkpoint + +open FSharp.UMX +open System // must shadow UMX to use DateTimeOffSet + +type CheckpointSeriesId = string +and [] checkpointSeriesId +module CheckpointSeriesId = let ofGroupName (groupName : string) = UMX.tag groupName + +// NB - these schemas reflect the actual storage formats and hence need to be versioned with care +module Events = + type Checkpoint = { at: DateTimeOffset; nextCheckpointDue: DateTimeOffset; pos: int64 } + type Config = { checkpointFreqS: int } + type Started = { config: Config; origin: Checkpoint } + type Checkpointed = { config: Config; pos: Checkpoint } + type Unfolded = { config: Config; state: Checkpoint } + type Event = + | Started of Started + | Checkpointed of Checkpointed + | Overrode of Checkpointed + | [] + Unfolded of Unfolded + interface TypeShape.UnionContract.IUnionContract + +module Folds = + type State = NotStarted | Running of Events.Unfolded + + let initial : State = NotStarted + let private evolve _ignoreState = function + | Events.Started { config = cfg; origin=originState } -> Running { config = cfg; state = originState } + | Events.Checkpointed e | Events.Overrode e -> Running { config = e.config; state = e.pos } + | Events.Unfolded runningState -> Running runningState + let fold (state: State) = Seq.fold evolve state + let isOrigin _state = true // we can build a state from any of the events and/or an unfold + let unfold state = + match state with + | NotStarted -> failwith "should never produce a NotStarted state" + | Running state -> Events.Unfolded {config = state.config; state=state.state} + + /// We only want to generate a first class event every N minutes, while efficiently writing contingent on the current etag value + //let postProcess events state = + // let checkpointEventIsRedundant (e: Events.Checkpointed) (s: Events.Unfolded) = + // s.state.nextCheckpointDue = e.pos.nextCheckpointDue + // && s.state.pos <> e.pos.pos + // match events, state with + // | [Events.Checkpointed e], (Running state as s) when checkpointEventIsRedundant e state -> + // [],unfold s + // | xs, state -> + // xs,unfold state + +type Command = + | Start of at: DateTimeOffset * checkpointFreq: TimeSpan * pos: int64 + | Override of at: DateTimeOffset * checkpointFreq: TimeSpan * pos: int64 + | Update of at: DateTimeOffset * pos: int64 + +module Commands = + let interpret command (state : Folds.State) = + let mkCheckpoint at next pos = { at=at; nextCheckpointDue = next; pos = pos } : Events.Checkpoint + let mk (at : DateTimeOffset) (interval: TimeSpan) pos : Events.Config * Events.Checkpoint= + let freq = int interval.TotalSeconds + let next = at.AddSeconds(float freq) + { checkpointFreqS = freq }, mkCheckpoint at next pos + match command, state with + | Start (at, freq, pos), Folds.NotStarted -> + let config, checkpoint = mk at freq pos + [Events.Started { config = config; origin = checkpoint}] + | Override (at, freq, pos), Folds.Running _ -> + let config, checkpoint = mk at freq pos + [Events.Overrode { config = config; pos = checkpoint}] + | Update (at,pos), Folds.Running state -> + // Force a write every N seconds regardless of whether the position has actually changed + if state.state.pos = pos && at < state.state.nextCheckpointDue then [] else + let freq = TimeSpan.FromSeconds <| float state.config.checkpointFreqS + let config, checkpoint = mk at freq pos + [Events.Checkpointed { config = config; pos = checkpoint}] + | c, s -> failwithf "Command %A invalid when %A" c s + +type Service(log, resolveStream, ?maxAttempts) = + let (|AggregateId|) (id : CheckpointSeriesId) = Equinox.AggregateId ("Sync", % id) + let (|Stream|) (AggregateId id) = Equinox.Stream(log, resolveStream id, defaultArg maxAttempts 3) + let execute (Stream stream) cmd = stream.Transact(Commands.interpret cmd) + + /// Determines the present state of the CheckpointSequence + member __.Read(Stream stream) = + stream.Query id + + /// Start a checkpointing series with the supplied parameters + /// NB will fail if already existing; caller should select to `Start` or `Override` based on whether Read indicates state is Running Or NotStarted + member __.Start(id, freq: TimeSpan, pos: int64) = + execute id <| Command.Start(DateTimeOffset.UtcNow, freq, pos) + + /// Override a checkpointing series with the supplied parameters + /// NB fails if not already initialized; caller should select to `Start` or `Override` based on whether Read indicates state is Running Or NotStarted + member __.Override(id, freq: TimeSpan, pos: int64) = + execute id <| Command.Override(DateTimeOffset.UtcNow, freq, pos) + + /// Ingest a position update + /// NB fails if not already initialized; caller should ensure correct initialization has taken place via Read -> Start + member __.Commit(id, pos: int64) = + execute id <| Command.Update(DateTimeOffset.UtcNow, pos) + +// General pattern is that an Equinox Service is a singleton and calls pass an inentifier for a stream per call +// This light wrapper means we can adhere to that general pattern yet still end up with lef=gible code while we in practice only maintain a single checkpoint series per running app +type CheckpointSeries(name, log, resolveStream) = + let seriesId = CheckpointSeriesId.ofGroupName name + let inner = Service(log, resolveStream) + member __.Read = inner.Read seriesId + member __.Start(freq, pos) = inner.Start(seriesId, freq, pos) + member __.Override(freq, pos) = inner.Override(seriesId, freq, pos) + member __.Commit(pos) = inner.Commit(seriesId, pos) + +/// Manages writing of progress +/// - Each write attempt is of the newest token +/// - retries until success or a new item is posted +type ProgressWriter(sync, ?interval) = + let pumpSleepMs = let interval = defaultArg interval TimeSpan.FromSeconds 5. in interval.TotalMilliseconds |> int + let mutable latest = None + let result = Event>() + [] member __.Result = result.Publish + member __.Post(value : int64) = latest <- Some value + member __.Pump() = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + match latest with + | Some value -> + try do! sync value + result.Trigger (Choice1Of2 value) + with e -> result.Trigger (Choice2Of2 e) + | _ -> () + do! Async.Sleep pumpSleepMs } \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 049678b08..39dd64154 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -230,7 +230,7 @@ type StreamStates() = waitCats.Ingest(category stream) waiting <- waiting + 1 waitingB <- waitingB + sz - log.Information("Streams Synced {synced} Dirty {dirty} Ready {ready}/{readyMb:n1}MB Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + log.Information("Synced {synced} Dirty {dirty} Ready {ready}/{readyMb:n1}MB Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) if readyCats.Any then log.Information("Ready {readyCats} {readyStreams}", readyCats.StatsDescending, readyStreams.StatsDescending) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 900b58380..c6bac504c 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -52,7 +52,7 @@ type OverallStats(?statsInterval) = member __.DumpIfIntervalExpired(?force) = if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then let totalMb = mb totalBytes - Log.Information("EventStore throughput {events} events {gb:n1}GB {mbs:n2}MB/s", + Log.Information("Reader Throughput {events} events {gb:n1}GB {mbs:n2}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) progressStart.Restart() @@ -74,16 +74,15 @@ type SliceStatsBuffer(?interval) = member __.DumpIfIntervalExpired(?force) = if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then lock recentCats <| fun () -> - let log = function - | [||] -> () - | xs -> - xs - |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) - |> Seq.truncate 10 - |> Seq.map (fun (KeyValue (s,(c,b))) -> b/1024/1024, s, c) - |> fun rendered -> Log.Information("EventStore categories {@cats} (MB/cat/count)", rendered) - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> Array.ofSeq |> log - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> Array.ofSeq |> log + let log kind xs = + let cats = + [| for KeyValue (s,(c,b)) in xs |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) -> + mb (int64 b) |> round, s, c |] + if (not << Array.isEmpty) cats then + let mb, events, top = Array.sumBy (fun (mb, _, _) -> mb) cats, Array.sumBy (fun (_, _, c) -> c) cats, Seq.truncate 100 cats + Log.Information("Reader {kind} {mb:n1}MB {events} events categories: {@cats} (MB/cat/count)", kind, mb, events, top) + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> log "payload" + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> log "meta" recentCats.Clear() accStart.Restart() @@ -113,7 +112,7 @@ let posFromPercentage (pct,max : Position) = let fetchMax (conn : IEventStoreConnection) = async { let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.FromPosition - Log.Information("EventStore Write @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) + Log.Information("EventStore Tail Position: @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) return max } let establishMax (conn : IEventStoreConnection) = async { let mutable max = None diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index acfe58785..6b9468855 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -2,14 +2,20 @@ open Equinox.Cosmos open Equinox.Cosmos.Core +//#if !eventStore open Equinox.Cosmos.Projection +//#endif //#if eventStore open Equinox.EventStore //#endif +//#if !eventStore open Equinox.Store +//#endif open Serilog open System +//#if !eventStore open System.Collections.Generic +//#endif open System.Diagnostics open System.Threading @@ -28,8 +34,11 @@ type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrC type ReaderSpec = { /// Identifier for this projection and it's state groupName: string + /// Indicates user has specified that they wish to restart from the indicated position as opposed to resuming from the checkpoint position + forceRestart: bool /// Start position from which forward reading is to commence // Assuming no stored position start: StartPos + checkpointInterval: TimeSpan /// Delay when reading yields an empty batch tailInterval: TimeSpan /// Maximum number of stream readers to permit @@ -67,7 +76,7 @@ module CmdParser = | [] Tail of intervalS: float | [] VerboseConsole #endif - | [] ForceStartFromHere + | [] ForceRestart | [] BatchSize of int | [] Verbose | [] Source of ParseResults @@ -76,8 +85,8 @@ module CmdParser = match a with | ConsumerGroupName _ -> "Projector consumer group name." | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | ForceStartFromHere _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." #if cosmos + | ForceStartFromHere _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | BatchSize _ -> "maximum item count to request from feed. Default: 1000" | LeaseCollectionSource _ ->"specify Collection Name for Leases collection, within `source` connection/database (default: `source`'s `collection` + `-aux`)." | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." @@ -85,6 +94,7 @@ module CmdParser = | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | Source _ -> "CosmosDb input parameters." #else + | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" | Position _ -> "EventStore $all Stream Position to commence from" @@ -112,6 +122,8 @@ module CmdParser = member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.StreamReaders = a.GetResult(StreamReaders,8) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds + member __.CheckpointInterval = TimeSpan.FromHours 1. + member __.ForceRestart = a.Contains ForceRestart #endif member __.Verbose = a.Contains Verbose @@ -138,11 +150,11 @@ module CmdParser = | _, Some c, _ -> StartPos.Chunk c | _, _, Some p -> Percentage p | None, None, None -> StartPos.TailOrCheckpoint - Log.Information("Processing Consumer Group {groupName} in Database {db} Collection {coll} starting from {startPos}", - x.ConsumerGroupName, x.Destination.Database, x.Destination.Collection, startPos) + Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", + x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", x.MinBatchSize, x.StartingBatchSize, x.StreamReaders) - { groupName = x.ConsumerGroupName; start = startPos; tailInterval = x.TailInterval + { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; streamReaders = x.StreamReaders } #endif and [] SourceParameters = @@ -282,154 +294,198 @@ module CmdParser = parser.ParseCommandLine argv |> Arguments #if !cosmos -type [] CoordinationWork<'Pos> = - | Result of CosmosIngester.Writer.Result - | Unbatched of CosmosIngester.Batch - | BatchWithTracking of 'Pos * CosmosIngester.Batch[] - -type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> CosmosIngester.Batch option, maxDop, ?statsInterval) = - let sleepIntervalMs = 100 - let dop = new SemaphoreSlim(maxDop) - let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) - member __.HasCapacity = work.QueueCount < dop.CurrentCount - member __.AddTail(startPos, interval) = work.AddTail(startPos, interval) - member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) - member __.Pump(postItem, shouldTail, postBatch) = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - work.OverallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, task) in () - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - dop.Release() |> ignore - do! Async.Sleep sleepIntervalMs } - -type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, ?interval, ?maxPendingBatches) = - let maxPendingBatches = defaultArg maxPendingBatches 32 - let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let sleepIntervalMs = 100 - let work = System.Collections.Concurrent.ConcurrentQueue() - let buffer = CosmosIngester.StreamStates() - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) - let tailSyncState = ProgressBatcher.State() - // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here - let mutable pendingBatchCount = 0 - let shouldThrottle () = pendingBatchCount > maxPendingBatches - let mutable progressEpoch = None - let pumpReaders = - let postWrite = work.Enqueue << CoordinationWork.Unbatched - let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) - readers.Pump(postWrite, not << shouldThrottle, postBatch) - let pumpWriters = writers.Pump() - let postWriteResult = work.Enqueue << CoordinationWork.Result - member __.Pump () = async { - use _ = writers.Result.Subscribe postWriteResult - let! _ = Async.StartChild pumpReaders - let! _ = Async.StartChild pumpWriters - let! ct = Async.CancellationToken - let writerResultLog = log.ForContext() - let mutable bytesPended = 0L - let workPended, eventsPended = ref 0, ref 0 - let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 - let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 - let badCats = CosmosIngester.CatStats() - let dumpStats () = - if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then - Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", - !rateLimited, !timedOut, !malformed) - rateLimited := 0; timedOut := 0; malformed := 0 - if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} ok {ok} redundant {dup} partial {partial} Missing {prefix} Exceptions {exns} reqs; Egress {gb:n3}GB", - !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) - workPended := 0; eventsPended := 0 - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; - - let throttle = shouldThrottle () - let level = if not throttle then Events.LogEventLevel.Debug else Events.LogEventLevel.Information - Log.Write(level, "Pending Batches {pb}", pendingBatchCount) - match progressEpoch with - | None -> () - | Some (epoch : EventStore.ClientAPI.Position) -> log.Information("Progress Epoch @ {epoch}", epoch.CommitPosition) - - buffer.Dump log - let tryDumpStats = every statsIntervalMs dumpStats - let handle = function - | CoordinationWork.Unbatched item -> - buffer.Add item |> ignore - | CoordinationWork.BatchWithTracking(pos, items) -> - for item in items do +module EventStoreSource = + type [] CoordinationWork<'Pos> = + | Result of CosmosIngester.Writer.Result + | ProgressResult of Choice + | Unbatched of CosmosIngester.Batch + | BatchWithTracking of 'Pos * CosmosIngester.Batch[] + + type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> CosmosIngester.Batch option, maxDop, ?statsInterval) = + let sleepIntervalMs = 100 + let dop = new SemaphoreSlim(maxDop) + let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) + member __.HasCapacity = work.QueueCount < dop.CurrentCount + member __.AddTail(startPos, interval) = work.AddTail(startPos, interval) + member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) + member __.Pump(postItem, shouldTail, postBatch) = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let! _ = work.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, task) in () + finally dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + dop.Release() |> ignore + do! Async.Sleep sleepIntervalMs } + + type StartMode = Starting | Resuming | Overridding + type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, ?interval, ?maxPendingBatches) = + let maxPendingBatches = defaultArg maxPendingBatches 32 + let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 + let sleepIntervalMs = 100 + let work = System.Collections.Concurrent.ConcurrentQueue() + let buffer = CosmosIngester.StreamStates() + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) + let tailSyncState = ProgressBatcher.State() + // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here + let mutable pendingBatchCount = 0 + let shouldThrottle () = pendingBatchCount > maxPendingBatches + let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None + let pumpReaders = + let postWrite = work.Enqueue << CoordinationWork.Unbatched + let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) + readers.Pump(postWrite, not << shouldThrottle, postBatch) + let postWriteResult = work.Enqueue << CoordinationWork.Result + let postProgressResult = work.Enqueue << CoordinationWork.ProgressResult + member __.Pump() = async { + use _ = writers.Result.Subscribe postWriteResult + use _ = progressWriter.Result.Subscribe postProgressResult + let! _ = Async.StartChild pumpReaders + let! _ = Async.StartChild <| writers.Pump() + let! _ = Async.StartChild <| progressWriter.Pump() + let! ct = Async.CancellationToken + let writerResultLog = log.ForContext() + let mutable bytesPended = 0L + let workPended, eventsPended = ref 0, ref 0 + let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 + let progCommitFails, progCommits = ref 0, ref 0 + let badCats = CosmosIngester.CatStats() + let dumpStats () = + if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then + Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", + !rateLimited, !timedOut, !malformed) + rateLimited := 0; timedOut := 0; malformed := 0 + if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn + Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} ok {ok} redundant {dup} partial {partial} Missing {prefix} Exceptions {exns} reqs; Egress {gb:n3}GB", + !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) + workPended := 0; eventsPended := 0 + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + + let throttle = shouldThrottle () + let level = if not throttle then Events.LogEventLevel.Debug else Events.LogEventLevel.Information + Log.Write(level, "Pending Batches {pb}", pendingBatchCount) + + if !progCommitFails <> 0 || !progCommits <> 0 then + match comittedEpoch with + | None -> + log.Error("Progress Epoch @ {validated}; writing failing: {failures} failures ({commits} successful commits)", + Option.toNullable validatedEpoch, !progCommitFails, !progCommits) + | Some committed when !progCommitFails <> 0 -> + log.Warning("Progress Epoch @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", + Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) + | Some committed -> + log.Information("Progress Epoch @ {validated} (committed: {committed}, {commits} commits)", + Option.toNullable validatedEpoch, committed, !progCommits) + progCommits := 0; progCommitFails := 0 + else + log.Information("Progress Epoch @ {validated} (committed: {committed})", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + buffer.Dump log + let tryDumpStats = every statsIntervalMs dumpStats + let handle = function + | CoordinationWork.Unbatched item -> buffer.Add item |> ignore - tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) - | CoordinationWork.Result res -> - match res with - | CosmosIngester.Writer.Result.Ok _ -> incr resultOk - | CosmosIngester.Writer.Result.Duplicate _ -> incr resultDup - | CosmosIngester.Writer.Result.PartialDuplicate _ -> incr resultPartialDup - | CosmosIngester.Writer.Result.PrefixMissing _ -> incr resultPrefix - | CosmosIngester.Writer.Result.Exn _ -> incr resultExn - - let (stream, updatedState), kind = buffer.HandleWriteResult res - match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) - match kind with - | CosmosIngester.Ok -> res.WriteTo writerResultLog - | CosmosIngester.RateLimited -> incr rateLimited - | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed - | CosmosIngester.TimedOut -> incr timedOut - let queueWrite (w : CosmosIngester.Batch) = - incr workPended - eventsPended := !eventsPended + w.span.events.Length - bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) - writers.Enqueue w - while not ct.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write results to buffer + Progress - match work.TryDequeue() with - | true, item -> - handle item - | false, _ -> - // 2. Mark off any progress achieved - let _validatedPos, _pendingBatchCount = tailSyncState.Validate buffer.TryGetStreamWritePos - pendingBatchCount <- _pendingBatchCount - progressEpoch <- _validatedPos - // 3. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - let mutable more = true - while more && readers.HasCapacity do - match buffer.TryGap() with - | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - | None -> more <- false - // 4. After that, [over] provision writers queue - let mutable more = true - while more && writers.HasCapacity do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> queueWrite w - | None -> (); more <- false - // 5. Periodically emit status info - tryDumpStats () - // TODO trigger periodic progress writing - // 7. Sleep if - do! Async.Sleep sleepIntervalMs } - - static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext) = async { - let! max = EventStoreSource.establishMax conn - let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) - let coordinator = Coordinator(log, readers, cosmosContext, maxWriters) - let startPos = - match spec.start with - | Absolute p -> EventStore.ClientAPI.Position(p, 0L) - | Chunk c -> EventStoreSource.posFromChunk c - | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) - | TailOrCheckpoint -> max - Log.Information("EventStore Tailing @ {pos} (chunk {chunk}, {pct:p1}) every {interval}s", - startPos.CommitPosition, EventStoreSource.chunk startPos, float startPos.CommitPosition/ float max.CommitPosition, spec.tailInterval.TotalSeconds) - do! coordinator.Pump () } -//#else + | CoordinationWork.BatchWithTracking(pos, items) -> + for item in items do + buffer.Add item |> ignore + tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) + | CoordinationWork.Result res -> + match res with + | CosmosIngester.Writer.Result.Ok _ -> incr resultOk + | CosmosIngester.Writer.Result.Duplicate _ -> incr resultDup + | CosmosIngester.Writer.Result.PartialDuplicate _ -> incr resultPartialDup + | CosmosIngester.Writer.Result.PrefixMissing _ -> incr resultPrefix + | CosmosIngester.Writer.Result.Exn _ -> incr resultExn + + let (stream, updatedState), kind = buffer.HandleWriteResult res + match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) + match kind with + | CosmosIngester.Ok -> res.WriteTo writerResultLog + | CosmosIngester.RateLimited -> incr rateLimited + | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed + | CosmosIngester.TimedOut -> incr timedOut + | CoordinationWork.ProgressResult (Choice1Of2 epoch) -> + incr progCommits + comittedEpoch <- Some epoch + | CoordinationWork.ProgressResult (Choice2Of2 (_exn : exn)) -> + incr progCommitFails + let queueWrite (w : CosmosIngester.Batch) = + incr workPended + eventsPended := !eventsPended + w.span.events.Length + bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) + writers.Enqueue w + while not ct.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + match work.TryDequeue() with + | true, item -> + handle item + | false, _ -> + // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) + let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos + pendingBatchCount <- _pendingBatchCount + validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) + // 3. Feed latest position to store + validatedEpoch |> Option.iter progressWriter.Post + // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + let mutable more = true + while more && readers.HasCapacity do + match buffer.TryGap() with + | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + | None -> more <- false + // 5. After that, [over] provision writers queue + let mutable more = true + while more && writers.HasCapacity do + match buffer.TryReady(writers.IsStreamBusy) with + | Some w -> queueWrite w + | None -> (); more <- false + // 6. Periodically emit status info + tryDumpStats () + // 7. Sleep if + do! Async.Sleep sleepIntervalMs } + + static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext) resolveCheckpointStream = async { + let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) + let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn + let! initialCheckpointState = checkpoints.Read + let! max = maxInParallel + let! startPos = async { + let mkPos x = EventStore.ClientAPI.Position(x, 0L) + let requestedStartPos = + match spec.start with + | Absolute p -> mkPos p + | Chunk c -> EventStoreSource.posFromChunk c + | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) + | TailOrCheckpoint -> max + let! startMode, startPos, checkpointFreq = async { + match initialCheckpointState, requestedStartPos with + | Checkpoint.Folds.NotStarted, r -> + if spec.forceRestart then raise <| CmdParser.InvalidArguments ("Cannot specify --forceRestart when no progress yet committed") + do! checkpoints.Start(spec.checkpointInterval, r.CommitPosition) + return Starting, r, spec.checkpointInterval + | Checkpoint.Folds.Running s, _ when not spec.forceRestart -> + return Resuming, mkPos s.state.pos, TimeSpan.FromSeconds(float s.config.checkpointFreqS) + | Checkpoint.Folds.Running _, r -> + do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) + return Overridding, r, spec.checkpointInterval + } + Log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m", + startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, + float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) + return startPos } + let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) + readers.AddTail(startPos, spec.tailInterval) + let progress = Checkpoint.ProgressWriter(checkpoints.Commit) + let coordinator = Coordinator(log, readers, cosmosContext, maxWriters, progress) + do! coordinator.Pump() } +#else module CosmosSource = open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing @@ -677,17 +733,19 @@ module CosmosSource = module Logging = let initialize verbose changeLogVerbose maybeSeqEndpoint = Log.Logger <- - LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() + LoggerConfiguration() + .Destructure.FSharpTypes() + .Enrich.FromLogContext() + |> fun c -> // LibLog writes to the global logger, so we need to control the emission if we don't want to pass loggers everywhere + let cfpLevel = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Warning + c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpLevel) + |> fun c -> let ingesterLevel = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Information + c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c - // LibLog writes to the global logger, so we need to control the emission if we don't want to pass loggers everywhere - |> fun c -> let cfpl = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Warning - c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpl) - |> fun c -> let ol = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Information - c.MinimumLevel.Override(typeof.FullName, ol) - |> fun c -> let ol = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning - c.MinimumLevel.Override(typeof.FullName, ol) - |> fun c -> let cl = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning - c.MinimumLevel.Override(typeof.FullName, cl) + |> fun c -> let generalLevel = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning + c.MinimumLevel.Override(typeof.FullName, generalLevel) + .MinimumLevel.Override(typeof.FullName, generalLevel) + .MinimumLevel.Override(typeof.FullName, generalLevel) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) @@ -697,10 +755,19 @@ module Logging = [] let main argv = try let args = CmdParser.parse argv - let target = - let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously - let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) - Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.ForContext()) + let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously + let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) + let resolveCheckpointStream = + let gateway = CosmosGateway(destination, CosmosBatchingPolicy()) + let store = Equinox.Cosmos.CosmosStore(gateway, colls) + let settings = Newtonsoft.Json.JsonSerializerSettings() + let codec = Equinox.Codec.NewtonsoftJson.Json.Create settings + let caching = + let c = Equinox.Cosmos.Caching.Cache("SyncTemplate", sizeMb = 1) + Equinox.Cosmos.CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) + let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) + Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve + let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.ForContext()) #if cosmos let log = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() @@ -720,7 +787,7 @@ let main argv = let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() - Coordinator.Run log (esConnection.ReadConnection, spec, EventStoreSource.tryMapEvent catFilter) (16, target) + EventStoreSource.Coordinator.Run log (esConnection.ReadConnection, spec, EventStoreSource.tryMapEvent catFilter) (16, target) resolveCheckpointStream #endif |> Async.RunSynchronously 0 diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index da29aa124..787bc1463 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -7,6 +7,7 @@ + @@ -19,6 +20,7 @@ + From 6840e720f497c6768697016ec20a7260538f1542 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 11:47:28 +0100 Subject: [PATCH 052/353] Add Total stat --- equinox-sync/Sync/EventStoreSource.fs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index c6bac504c..25a2b0aa9 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -74,15 +74,16 @@ type SliceStatsBuffer(?interval) = member __.DumpIfIntervalExpired(?force) = if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then lock recentCats <| fun () -> - let log kind xs = + let log kind limit xs = let cats = [| for KeyValue (s,(c,b)) in xs |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) -> mb (int64 b) |> round, s, c |] if (not << Array.isEmpty) cats then - let mb, events, top = Array.sumBy (fun (mb, _, _) -> mb) cats, Array.sumBy (fun (_, _, c) -> c) cats, Seq.truncate 100 cats + let mb, events, top = Array.sumBy (fun (mb, _, _) -> mb) cats, Array.sumBy (fun (_, _, c) -> c) cats, Seq.truncate limit cats Log.Information("Reader {kind} {mb:n1}MB {events} events categories: {@cats} (MB/cat/count)", kind, mb, events, top) - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> log "payload" - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> log "meta" + recentCats |> log "Total" 3 + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> log "payload" 100 + recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> log "meta" 100 recentCats.Clear() accStart.Restart() From 2288d4dc4f9797cb9bfd69255a18928ea88b1393 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 13:29:26 +0100 Subject: [PATCH 053/353] Move tryMapEvent to main --- equinox-sync/Ingest/Program.fs | 9 +++++++++ equinox-sync/Sync/EventStoreSource.fs | 10 ---------- equinox-sync/Sync/Program.fs | 11 ++++++++++- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 52901a59d..e509f1be7 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -304,6 +304,15 @@ let main argv = let destination = cosmos.Connect "SyncTemplate.Ingester" |> Async.RunSynchronously let colls = Equinox.Cosmos.CosmosCollections(cosmos.Database, cosmos.Collection) Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) + let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = + match x.Event with + | e when not e.IsJson + || e.EventStreamId.StartsWith("$") + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || not (catFilter e.EventStreamId) -> None + | e -> EventStoreSource.tryToBatch e Coordinator.Run Log.Logger source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 25a2b0aa9..c507cdd0f 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -26,16 +26,6 @@ let tryToBatch (e : RecordedEvent) : CosmosIngester.Batch option = let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ Some { stream = e.EventStreamId; span = { index = e.EventNumber; events = [| event |]} } -let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = - match x.Event with - | e when not e.IsJson - || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.StartsWith("$") - || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") - || not (catFilter e.EventStreamId) -> None - | e -> tryToBatch e - let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 6b9468855..75ba4b968 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -787,7 +787,16 @@ let main argv = let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() - EventStoreSource.Coordinator.Run log (esConnection.ReadConnection, spec, EventStoreSource.tryMapEvent catFilter) (16, target) resolveCheckpointStream + let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = + match x.Event with + | e when not e.IsJson + || e.EventStreamId.StartsWith("$") + || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || not (catFilter e.EventStreamId) -> None + | e -> EventStoreSource.tryToBatch e + EventStoreSource.Coordinator.Run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (16, target) resolveCheckpointStream #endif |> Async.RunSynchronously 0 From 5294f4fe3fe0418ed41d2ecbb4f68f77c15248b4 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 17:38:45 +0100 Subject: [PATCH 054/353] Prod param tweaks --- equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/Program.fs | 17 +++++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 39dd64154..48cc2e539 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -233,7 +233,7 @@ type StreamStates() = log.Information("Synced {synced} Dirty {dirty} Ready {ready}/{readyMb:n1}MB Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) - if readyCats.Any then log.Information("Ready {readyCats} {readyStreams}", readyCats.StatsDescending, readyStreams.StatsDescending) + if readyCats.Any then log.Information("Ready {readyCats} {readyStreams}", readyCats.StatsDescending, Seq.truncate 10 readyStreams.StatsDescending) type RefCounted<'T> = { mutable refCount: int; value: 'T } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 75ba4b968..35c1248cb 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -69,6 +69,8 @@ module CmdParser = | [] ChangeFeedVerbose #else | [] MinBatchSize of int + | [] MaxPending of int + | [] MaxWriters of int | [] Position of int64 | [] Chunk of int | [] Percent of float @@ -97,6 +99,8 @@ module CmdParser = | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" + | MaxPending _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" + | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 64" | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" @@ -119,6 +123,8 @@ module CmdParser = member __.VerboseConsole = a.Contains VerboseConsole member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) + member __.MaxPendingBatches = a.GetResult(MaxPending,64) + member __.MaxWriters = a.GetResult(MaxWriters,64) member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.StreamReaders = a.GetResult(StreamReaders,8) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds @@ -326,8 +332,7 @@ module EventStoreSource = do! Async.Sleep sleepIntervalMs } type StartMode = Starting | Resuming | Overridding - type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, ?interval, ?maxPendingBatches) = - let maxPendingBatches = defaultArg maxPendingBatches 32 + type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let sleepIntervalMs = 100 let work = System.Collections.Concurrent.ConcurrentQueue() @@ -365,7 +370,7 @@ module EventStoreSource = rateLimited := 0; timedOut := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - Log.Information("Writer Throughput {queued} req {events} events; Completed {completed} ok {ok} redundant {dup} partial {partial} Missing {prefix} Exceptions {exns} reqs; Egress {gb:n3}GB", + Log.Information("Writer Throughput {queued} reqs {events} events; Completed {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) workPended := 0; eventsPended := 0 resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; @@ -451,7 +456,7 @@ module EventStoreSource = // 7. Sleep if do! Async.Sleep sleepIntervalMs } - static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext) resolveCheckpointStream = async { + static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn let! initialCheckpointState = checkpoints.Read @@ -483,7 +488,7 @@ module EventStoreSource = let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) readers.AddTail(startPos, spec.tailInterval) let progress = Checkpoint.ProgressWriter(checkpoints.Commit) - let coordinator = Coordinator(log, readers, cosmosContext, maxWriters, progress) + let coordinator = Coordinator(log, readers, cosmosContext, maxWriters, progress, maxPendingBatches=maxPendingBatches) do! coordinator.Pump() } #else module CosmosSource = @@ -796,7 +801,7 @@ let main argv = || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e - EventStoreSource.Coordinator.Run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (16, target) resolveCheckpointStream + EventStoreSource.Coordinator.Run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (args.MaxWriters, target, args.MaxPendingBatches) resolveCheckpointStream #endif |> Async.RunSynchronously 0 From 1762609b760446d021231b25b411a5d3df98b871 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 18:56:01 +0100 Subject: [PATCH 055/353] Try 1000 write len --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 48cc2e539..01cd87e8f 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -205,7 +205,7 @@ type StreamStates() = bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 // Reduce the item count when we don't yet know the write position - count <= (if Option.isNone state.write then 10 else 100) && (bytesBudget >= 0 || count = 1) + count <= (if Option.isNone state.write then 10 else 1000) && (bytesBudget >= 0 || count = 1) Some { stream = stream; span = { index = h.index; events = h.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } let res = aux () for x in blocked do markDirty x From 44cf6329f4dc94337bca9dc22f4071003d34256a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 19:41:23 +0100 Subject: [PATCH 056/353] Slave --- equinox-sync/Sync/EventStoreSource.fs | 2 +- equinox-sync/Sync/Program.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index c507cdd0f..393c06e44 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -143,7 +143,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,3}c {streams,4}s {events,4}e Post {pt:n0}ms", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) if range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream then diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 35c1248cb..7d3e08df7 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -789,7 +789,7 @@ let main argv = createSyncHandler #else let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint - let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.Master) + let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = From df3aa1f8dcc760c8b36c26fe1050a38ab73580e7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 20:43:30 +0100 Subject: [PATCH 057/353] Add egress MB --- equinox-sync/Sync/Program.fs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 7d3e08df7..e3ac7aaeb 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -357,7 +357,7 @@ module EventStoreSource = let! _ = Async.StartChild <| progressWriter.Pump() let! ct = Async.CancellationToken let writerResultLog = log.ForContext() - let mutable bytesPended = 0L + let mutable bytesPended, bytesPendedAgg = 0L, 0L let workPended, eventsPended = ref 0, ref 0 let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 @@ -370,9 +370,10 @@ module EventStoreSource = rateLimited := 0; timedOut := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - Log.Information("Writer Throughput {queued} reqs {events} events; Completed {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", - !workPended, !eventsPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPended / 1024.) - workPended := 0; eventsPended := 0 + bytesPendedAgg <- bytesPendedAgg + bytesPended + Log.Information("Writer Throughput {queued} reqs {events} events {mb:n}MB; Completed {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", + !workPended, !eventsPended, mb bytesPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPendedAgg / 1024.) + workPended := 0; eventsPended := 0; bytesPended <- 0L resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; let throttle = shouldThrottle () From 8bbc7fe212392b7e6809e4f46fed8be0f77f12d0 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 21:24:33 +0100 Subject: [PATCH 058/353] Coordinator thread resilience --- equinox-sync/Sync/Program.fs | 66 +++++++++++++++++++----------------- 1 file changed, 34 insertions(+), 32 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index e3ac7aaeb..43e4a63b3 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -335,7 +335,8 @@ module EventStoreSource = type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let sleepIntervalMs = 100 - let work = System.Collections.Concurrent.ConcurrentQueue() + let work = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), maxPendingBatches*4096) + let addWork = work.Add let buffer = CosmosIngester.StreamStates() let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) let tailSyncState = ProgressBatcher.State() @@ -344,11 +345,11 @@ module EventStoreSource = let shouldThrottle () = pendingBatchCount > maxPendingBatches let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None let pumpReaders = - let postWrite = work.Enqueue << CoordinationWork.Unbatched - let postBatch pos xs = work.Enqueue(CoordinationWork.BatchWithTracking (pos,xs)) + let postWrite = addWork << CoordinationWork.Unbatched + let postBatch pos xs = addWork (CoordinationWork.BatchWithTracking (pos,xs)) readers.Pump(postWrite, not << shouldThrottle, postBatch) - let postWriteResult = work.Enqueue << CoordinationWork.Result - let postProgressResult = work.Enqueue << CoordinationWork.ProgressResult + let postWriteResult = addWork << CoordinationWork.Result + let postProgressResult = addWork << CoordinationWork.ProgressResult member __.Pump() = async { use _ = writers.Result.Subscribe postWriteResult use _ = progressWriter.Result.Subscribe postProgressResult @@ -429,33 +430,34 @@ module EventStoreSource = bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) writers.Enqueue w while not ct.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - match work.TryDequeue() with - | true, item -> - handle item - | false, _ -> - // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos - pendingBatchCount <- _pendingBatchCount - validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) - // 3. Feed latest position to store - validatedEpoch |> Option.iter progressWriter.Post - // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - let mutable more = true - while more && readers.HasCapacity do - match buffer.TryGap() with - | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - | None -> more <- false - // 5. After that, [over] provision writers queue - let mutable more = true - while more && writers.HasCapacity do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> queueWrite w - | None -> (); more <- false - // 6. Periodically emit status info - tryDumpStats () - // 7. Sleep if - do! Async.Sleep sleepIntervalMs } + try // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + match work.TryTake() with + | true, item -> + handle item + | false, _ -> + // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) + let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos + pendingBatchCount <- _pendingBatchCount + validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) + // 3. Feed latest position to store + validatedEpoch |> Option.iter progressWriter.Post + // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + let mutable more = true + while more && readers.HasCapacity do + match buffer.TryGap() with + | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + | None -> more <- false + // 5. After that, [over] provision writers queue + let mutable more = true + while more && writers.HasCapacity do + match buffer.TryReady(writers.IsStreamBusy) with + | Some w -> queueWrite w + | None -> (); more <- false + // 6. Periodically emit status info + tryDumpStats () + // 7. Sleep if + do! Async.Sleep sleepIntervalMs + with e -> log.Fatal(e,"Loop exn") } static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) From 9b6326adeded966932a4b28f441f7c438465d5b7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 21:46:19 +0100 Subject: [PATCH 059/353] Fix coordination loop --- equinox-sync/Sync/Program.fs | 57 ++++++++++++++++++------------------ 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 43e4a63b3..c86dfd57e 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -334,7 +334,7 @@ module EventStoreSource = type StartMode = Starting | Resuming | Overridding type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let sleepIntervalMs = 100 + let sleepIntervalMs = 10 let work = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), maxPendingBatches*4096) let addWork = work.Add let buffer = CosmosIngester.StreamStates() @@ -430,33 +430,34 @@ module EventStoreSource = bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) writers.Enqueue w while not ct.IsCancellationRequested do - try // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - match work.TryTake() with - | true, item -> - handle item - | false, _ -> - // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos - pendingBatchCount <- _pendingBatchCount - validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) - // 3. Feed latest position to store - validatedEpoch |> Option.iter progressWriter.Post - // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - let mutable more = true - while more && readers.HasCapacity do - match buffer.TryGap() with - | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - | None -> more <- false - // 5. After that, [over] provision writers queue - let mutable more = true - while more && writers.HasCapacity do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> queueWrite w - | None -> (); more <- false - // 6. Periodically emit status info - tryDumpStats () - // 7. Sleep if - do! Async.Sleep sleepIntervalMs + try // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + let mutable more = true + while more do + match work.TryTake() with + | true, item -> handle item + | false, _ -> more <- false + // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) + let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos + pendingBatchCount <- _pendingBatchCount + validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) + // 3. Feed latest position to store + validatedEpoch |> Option.iter progressWriter.Post + // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + let mutable more = true + while more && readers.HasCapacity do + match buffer.TryGap() with + | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + | None -> more <- false + // 5. After that, [over] provision writers queue + let mutable more = true + while more && writers.HasCapacity do + match buffer.TryReady(writers.IsStreamBusy) with + | Some w -> queueWrite w + | None -> (); more <- false + // 6. Periodically emit status info + tryDumpStats () + // 7. Sleep if + do! Async.Sleep sleepIntervalMs with e -> log.Fatal(e,"Loop exn") } static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { From b7845677aabe454a7843e6f2bbc64f4703385f24 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 22:44:20 +0100 Subject: [PATCH 060/353] Add write capacity --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 01cd87e8f..59d498dc1 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -276,7 +276,7 @@ type SemaphorePool(gen : unit -> SemaphoreSlim) = let locks = SemaphorePool(fun () -> new SemaphoreSlim 1) [] member __.Result = result.Publish member __.Enqueue item = work.Enqueue item - member __.HasCapacity = work.Count < maxDop && maxQueueLen |> Option.forall (fun max -> work.Count < max) + member __.HasCapacity = work.Count < maxDop * 2 && maxQueueLen |> Option.forall (fun max -> work.Count < max) member __.IsStreamBusy stream = let checkBusy (x : SemaphoreSlim) = x.CurrentCount = 0 locks.Execute(stream,checkBusy) From 5ecea38549cd762c4e09dda81901feae680685ab Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 23:21:43 +0100 Subject: [PATCH 061/353] undo 2x --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 59d498dc1..01cd87e8f 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -276,7 +276,7 @@ type SemaphorePool(gen : unit -> SemaphoreSlim) = let locks = SemaphorePool(fun () -> new SemaphoreSlim 1) [] member __.Result = result.Publish member __.Enqueue item = work.Enqueue item - member __.HasCapacity = work.Count < maxDop * 2 && maxQueueLen |> Option.forall (fun max -> work.Count < max) + member __.HasCapacity = work.Count < maxDop && maxQueueLen |> Option.forall (fun max -> work.Count < max) member __.IsStreamBusy stream = let checkBusy (x : SemaphoreSlim) = x.CurrentCount = 0 locks.Execute(stream,checkBusy) From 39f1f1149dc03c57b4e79ded71b8848ca5cb3f15 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 16 Apr 2019 23:31:47 +0100 Subject: [PATCH 062/353] Add cycle counter --- equinox-sync/Sync/Program.fs | 68 +++++++++++++++++++----------------- 1 file changed, 35 insertions(+), 33 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index c86dfd57e..78f9440f9 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -359,7 +359,7 @@ module EventStoreSource = let! ct = Async.CancellationToken let writerResultLog = log.ForContext() let mutable bytesPended, bytesPendedAgg = 0L, 0L - let workPended, eventsPended = ref 0, ref 0 + let workPended, eventsPended, cycles = ref 0, ref 0, ref 0 let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 let progCommitFails, progCommits = ref 0, ref 0 @@ -372,9 +372,9 @@ module EventStoreSource = if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Writer Throughput {queued} reqs {events} events {mb:n}MB; Completed {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", - !workPended, !eventsPended, mb bytesPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPendedAgg / 1024.) - workPended := 0; eventsPended := 0; bytesPended <- 0L + Log.Information("Writer Throughput {cycles} cycles {queued} reqs {events} events {mb:n}MB; Completed {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", + !cycles, !workPended, !eventsPended, mb bytesPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPendedAgg / 1024.) + cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; let throttle = shouldThrottle () @@ -430,35 +430,37 @@ module EventStoreSource = bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) writers.Enqueue w while not ct.IsCancellationRequested do - try // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - let mutable more = true - while more do - match work.TryTake() with - | true, item -> handle item - | false, _ -> more <- false - // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos - pendingBatchCount <- _pendingBatchCount - validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) - // 3. Feed latest position to store - validatedEpoch |> Option.iter progressWriter.Post - // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - let mutable more = true - while more && readers.HasCapacity do - match buffer.TryGap() with - | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - | None -> more <- false - // 5. After that, [over] provision writers queue - let mutable more = true - while more && writers.HasCapacity do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> queueWrite w - | None -> (); more <- false - // 6. Periodically emit status info - tryDumpStats () - // 7. Sleep if - do! Async.Sleep sleepIntervalMs - with e -> log.Fatal(e,"Loop exn") } + incr cycles + // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + let mutable gotWork = false + let mutable remaining = 1000 + while remaining > 0 do + match work.TryTake() with + | true, item -> handle item; gotWork <- true; remaining <- remaining - 1 + | false, _ -> remaining <- 0 + // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) + let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos + pendingBatchCount <- _pendingBatchCount + validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) + // 3. Feed latest position to store + validatedEpoch |> Option.iter progressWriter.Post + // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + let mutable more = true + while more && readers.HasCapacity do + match buffer.TryGap() with + | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + | None -> more <- false + // 5. After that, [over] provision writers queue + let mutable more = true + while more && writers.HasCapacity do + match buffer.TryReady(writers.IsStreamBusy) with + | Some w -> queueWrite w; gotWork <- true + | None -> (); more <- false + // 6. Periodically emit status info + tryDumpStats () + // 7. Sleep if + if not gotWork then + do! Async.Sleep sleepIntervalMs } static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) From 8a20f68cf05419a77bd2ca054a0481579672aa6f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 00:06:28 +0100 Subject: [PATCH 063/353] Prioritize results processing --- equinox-sync/Sync/Program.fs | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 78f9440f9..908c24812 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -335,21 +335,21 @@ module EventStoreSource = type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let sleepIntervalMs = 10 - let work = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), maxPendingBatches*4096) - let addWork = work.Add + let input = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), 65536) + let results = System.Collections.Concurrent.ConcurrentQueue() let buffer = CosmosIngester.StreamStates() let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) - let tailSyncState = ProgressBatcher.State() + let tailSyncState = ProgressBatcher.State() // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here let mutable pendingBatchCount = 0 let shouldThrottle () = pendingBatchCount > maxPendingBatches let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None let pumpReaders = - let postWrite = addWork << CoordinationWork.Unbatched - let postBatch pos xs = addWork (CoordinationWork.BatchWithTracking (pos,xs)) + let postWrite = input.Add << CoordinationWork.Unbatched + let postBatch pos xs = input.Add (CoordinationWork.BatchWithTracking (pos,xs)) readers.Pump(postWrite, not << shouldThrottle, postBatch) - let postWriteResult = addWork << CoordinationWork.Result - let postProgressResult = addWork << CoordinationWork.ProgressResult + let postWriteResult = results.Enqueue << CoordinationWork.Result + let postProgressResult = results.Enqueue << CoordinationWork.ProgressResult member __.Pump() = async { use _ = writers.Result.Subscribe postWriteResult use _ = progressWriter.Result.Subscribe postProgressResult @@ -432,12 +432,11 @@ module EventStoreSource = while not ct.IsCancellationRequested do incr cycles // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - let mutable gotWork = false - let mutable remaining = 1000 - while remaining > 0 do - match work.TryTake() with - | true, item -> handle item; gotWork <- true; remaining <- remaining - 1 - | false, _ -> remaining <- 0 + let mutable more, gotWork = true, false + while more do + match results.TryDequeue() with + | true, item -> handle item; gotWork <- true + | false, _ -> more <- false // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos pendingBatchCount <- _pendingBatchCount @@ -456,9 +455,15 @@ module EventStoreSource = match buffer.TryReady(writers.IsStreamBusy) with | Some w -> queueWrite w; gotWork <- true | None -> (); more <- false - // 6. Periodically emit status info + // 6. OK, we've stashed and cleaned work; now take some inputs + let mutable remaining = 8192 + while remaining > 0 do + match input.TryTake() with + | true, item -> handle item; gotWork <- true; remaining <- remaining - 1 + | false, _ -> remaining <- 0 + // 7. Periodically emit status info tryDumpStats () - // 7. Sleep if + // 8. Sleep if if not gotWork then do! Async.Sleep sleepIntervalMs } From f8d5a702680cbbcb91a84248f96220a07ab80fbb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 00:27:58 +0100 Subject: [PATCH 064/353] Deprioritize ingestion --- equinox-sync/Sync/Program.fs | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 908c24812..f44048e8c 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -335,7 +335,7 @@ module EventStoreSource = type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 let sleepIntervalMs = 10 - let input = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), 65536) + let input = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), maxPendingBatches) let results = System.Collections.Concurrent.ConcurrentQueue() let buffer = CosmosIngester.StreamStates() let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) @@ -345,7 +345,7 @@ module EventStoreSource = let shouldThrottle () = pendingBatchCount > maxPendingBatches let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None let pumpReaders = - let postWrite = input.Add << CoordinationWork.Unbatched + let postWrite = (*we want to prioritize processing of catchup stream reads*) results.Enqueue << CoordinationWork.Unbatched let postBatch pos xs = input.Add (CoordinationWork.BatchWithTracking (pos,xs)) readers.Pump(postWrite, not << shouldThrottle, postBatch) let postWriteResult = results.Enqueue << CoordinationWork.Result @@ -456,16 +456,18 @@ module EventStoreSource = | Some w -> queueWrite w; gotWork <- true | None -> (); more <- false // 6. OK, we've stashed and cleaned work; now take some inputs - let mutable remaining = 8192 - while remaining > 0 do - match input.TryTake() with - | true, item -> handle item; gotWork <- true; remaining <- remaining - 1 - | false, _ -> remaining <- 0 - // 7. Periodically emit status info - tryDumpStats () - // 8. Sleep if if not gotWork then - do! Async.Sleep sleepIntervalMs } + let x = Stopwatch.StartNew() + let mutable more = true + while more && x.ElapsedMilliseconds < int64 sleepIntervalMs do + match input.TryTake() with + | true, item -> handle item + | false, _ -> more <- false + match sleepIntervalMs - int x.ElapsedMilliseconds with + | d when d > 0 -> do! Async.Sleep d + | _ -> () + // 7. Periodically emit status info + tryDumpStats () } static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) From c916fd63127ea563f142a2710d595760ad0becb7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 00:54:30 +0100 Subject: [PATCH 065/353] No sleeping when writing to do --- equinox-sync/Sync/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index f44048e8c..7d8ce2ce7 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -459,12 +459,12 @@ module EventStoreSource = if not gotWork then let x = Stopwatch.StartNew() let mutable more = true - while more && x.ElapsedMilliseconds < int64 sleepIntervalMs do + while more && not writers.HasCapacity && x.ElapsedMilliseconds < int64 sleepIntervalMs do match input.TryTake() with | true, item -> handle item | false, _ -> more <- false match sleepIntervalMs - int x.ElapsedMilliseconds with - | d when d > 0 -> do! Async.Sleep d + | d when d > 0 && not writers.HasCapacity -> do! Async.Sleep d | _ -> () // 7. Periodically emit status info tryDumpStats () } From dbd2e9ffb57d6b9d4e9f14c7df143f00c2f11272 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 01:00:39 +0100 Subject: [PATCH 066/353] More --- equinox-sync/Sync/Program.fs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 7d8ce2ce7..68097b62a 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -464,7 +464,9 @@ module EventStoreSource = | true, item -> handle item | false, _ -> more <- false match sleepIntervalMs - int x.ElapsedMilliseconds with - | d when d > 0 && not writers.HasCapacity -> do! Async.Sleep d + | d when d > 0 -> + if writers.HasCapacity then do! Async.Sleep 1 + else do! Async.Sleep d | _ -> () // 7. Periodically emit status info tryDumpStats () } From b9bf9f8022e5427d7a8a469a2c6808c89ce6e132 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 01:04:40 +0100 Subject: [PATCH 067/353] Fix denial --- equinox-sync/Sync/Program.fs | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 68097b62a..6b6d6c25b 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -456,18 +456,17 @@ module EventStoreSource = | Some w -> queueWrite w; gotWork <- true | None -> (); more <- false // 6. OK, we've stashed and cleaned work; now take some inputs - if not gotWork then - let x = Stopwatch.StartNew() - let mutable more = true - while more && not writers.HasCapacity && x.ElapsedMilliseconds < int64 sleepIntervalMs do - match input.TryTake() with - | true, item -> handle item - | false, _ -> more <- false - match sleepIntervalMs - int x.ElapsedMilliseconds with - | d when d > 0 -> - if writers.HasCapacity then do! Async.Sleep 1 - else do! Async.Sleep d - | _ -> () + let x = Stopwatch.StartNew() + let mutable more = true + while more && x.ElapsedMilliseconds < int64 sleepIntervalMs do + match input.TryTake() with + | true, item -> handle item + | false, _ -> more <- false + match sleepIntervalMs - int x.ElapsedMilliseconds with + | d when d > 0 -> + if writers.HasCapacity then do! Async.Sleep 1 + else do! Async.Sleep d + | _ -> () // 7. Periodically emit status info tryDumpStats () } From 79e78337935bda22a0b627a3c3566ddbc490e5e8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 01:24:47 +0100 Subject: [PATCH 068/353] Reduce sleeps --- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- equinox-sync/Sync/Program.fs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 01cd87e8f..a1aa1651c 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -270,13 +270,13 @@ type SemaphorePool(gen : unit -> SemaphoreSlim) = use _l = slotReleaseGuard k f x - type Writers(write, maxDop, ?maxQueueLen) = + type Writers(write, maxDop) = let work = ConcurrentQueue() let result = Event() let locks = SemaphorePool(fun () -> new SemaphoreSlim 1) [] member __.Result = result.Publish member __.Enqueue item = work.Enqueue item - member __.HasCapacity = work.Count < maxDop && maxQueueLen |> Option.forall (fun max -> work.Count < max) + member __.HasCapacity = work.Count < maxDop member __.IsStreamBusy stream = let checkBusy (x : SemaphoreSlim) = x.CurrentCount = 0 locks.Execute(stream,checkBusy) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 6b6d6c25b..2b14a3ff9 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -464,7 +464,7 @@ module EventStoreSource = | false, _ -> more <- false match sleepIntervalMs - int x.ElapsedMilliseconds with | d when d > 0 -> - if writers.HasCapacity then do! Async.Sleep 1 + if writers.HasCapacity || gotWork then do! Async.Sleep 1 else do! Async.Sleep d | _ -> () // 7. Periodically emit status info From 831e0a0b67be10bd7d4b96dfdf36398e3822d716 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 01:32:46 +0100 Subject: [PATCH 069/353] 4096 --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index a1aa1651c..91feae4e4 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -205,7 +205,7 @@ type StreamStates() = bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 // Reduce the item count when we don't yet know the write position - count <= (if Option.isNone state.write then 10 else 1000) && (bytesBudget >= 0 || count = 1) + count <= (if Option.isNone state.write then 10 else 4096) && (bytesBudget >= 0 || count = 1) Some { stream = stream; span = { index = h.index; events = h.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } let res = aux () for x in blocked do markDirty x From d21eea26fcd7233c1dbcf92c3568e6c0b81d330c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 01:37:28 +0100 Subject: [PATCH 070/353] Fudge max limits --- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 91feae4e4..50457c45d 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -12,8 +12,8 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] -let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 +let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 (*fudge*) - 2048 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 (* fudge*) + 128 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } From 36d2b29f20edbf7ac72d67a1f9c4893affd08f5f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 02:02:06 +0100 Subject: [PATCH 071/353] Clean logging --- equinox-sync/Sync/CosmosIngester.fs | 10 +++++----- equinox-sync/Sync/EventStoreSource.fs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 50457c45d..cc8501f05 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -13,7 +13,7 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 (*fudge*) - 2048 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 4 (* fudge*) + 128 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 16 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } @@ -119,9 +119,9 @@ module StreamState = /// Gathers stats relating to how many items of a given category have been observed type CatStats() = - let cats = Dictionary() + let cats = Dictionary() member __.Ingest(cat,?weight) = - let weight = defaultArg weight 1 + let weight = defaultArg weight 1L match cats.TryGetValue cat with | true, catCount -> cats.[cat] <- catCount + weight | false, _ -> cats.[cat] <- weight @@ -223,7 +223,7 @@ type StreamStates() = malformedB <- malformedB + sz | sz when state.IsReady -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%A" stream state.write, int sz) + readyStreams.Ingest(sprintf "%s@%A" stream state.write, mb sz |> int64) ready <- ready + 1 readyB <- readyB + sz | sz -> @@ -233,7 +233,7 @@ type StreamStates() = log.Information("Synced {synced} Dirty {dirty} Ready {ready}/{readyMb:n1}MB Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) - if readyCats.Any then log.Information("Ready {readyCats} {readyStreams}", readyCats.StatsDescending, Seq.truncate 10 readyStreams.StatsDescending) + if readyCats.Any then log.Information("Ready {readyCats}\nTop Streams (MB) {readyStreams}", readyCats.StatsDescending, Seq.truncate 10 readyStreams.StatsDescending) type RefCounted<'T> = { mutable refCount: int; value: 'T } diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 393c06e44..a8b3c7bfe 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -42,7 +42,7 @@ type OverallStats(?statsInterval) = member __.DumpIfIntervalExpired(?force) = if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then let totalMb = mb totalBytes - Log.Information("Reader Throughput {events} events {gb:n1}GB {mbs:n2}MB/s", + Log.Information("Reader Throughput {events} events {gb:n1}GB {mb:n2}MB/s", totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) progressStart.Restart() @@ -70,7 +70,7 @@ type SliceStatsBuffer(?interval) = mb (int64 b) |> round, s, c |] if (not << Array.isEmpty) cats then let mb, events, top = Array.sumBy (fun (mb, _, _) -> mb) cats, Array.sumBy (fun (_, _, c) -> c) cats, Seq.truncate limit cats - Log.Information("Reader {kind} {mb:n1}MB {events} events categories: {@cats} (MB/cat/count)", kind, mb, events, top) + Log.Information("Reader {kind} {mb:n0}MB {events:n0} events categories: {@cats} (MB/cat/count)", kind, mb, events, top) recentCats |> log "Total" 3 recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> log "payload" 100 recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> log "meta" 100 From 27485916c4b63cc9f81f2a6d8d92c42574695685 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 02:08:17 +0100 Subject: [PATCH 072/353] More fudge --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index cc8501f05..171811401 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -13,7 +13,7 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 (*fudge*) - 2048 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 16 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 128 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } From 82d33c27f43a5711299f8680fd08a860a25b8a47 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 07:54:00 +0100 Subject: [PATCH 073/353] More fudge --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 171811401..4e3bb16f4 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -12,7 +12,7 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] -let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 (*fudge*) - 2048 +let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 (*fudge*) - 4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 128 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } From bdc7aaa20c66f62cea4c6539bc1f2627d070a9bf Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 08:01:44 +0100 Subject: [PATCH 074/353] Add timing to stream reading --- equinox-sync/Sync/EventStoreSource.fs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index a8b3c7bfe..8cdfcaf56 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -185,8 +185,8 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | StreamPrefix (name,pos,len,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) - try do! pullStream (conn, batchSize) (name, pos, Some len) postItem - Log.Information("completed stream prefix") + try let! t,() = pullStream (conn, batchSize) (name, pos, Some len) postItem |> Stopwatch.Time + Log.Information("completed stream prefix in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) with e -> let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) @@ -195,8 +195,8 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Stream (name,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream; batch size {bs}", batchSize) - try do! pullStream (conn, batchSize) (name,0L,None) postItem - Log.Information("completed stream") + try let! t,() = pullStream (conn, batchSize) (name,0L,None) postItem |> Stopwatch.Time + Log.Information("completed stream in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) with e -> let bs = adjust batchSize Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) @@ -205,14 +205,14 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let! res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch + let! t,res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with | PullResult.EndOfTranche -> - Log.Warning("Completed tranche") + Log.Warning("completed tranche in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) __.OverallStats.DumpIfIntervalExpired() return false | PullResult.Eof -> - Log.Warning("REACHED THE END!") + Log.Warning("completed tranche AND REACHED THE END in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) __.OverallStats.DumpIfIntervalExpired(true) return true | PullResult.Exn e -> From 0d48ddf30176c87cf06fa9270aaba632cdd65f83 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 09:34:35 +0100 Subject: [PATCH 075/353] Exclude ReloadBatchId --- equinox-sync/Sync/Program.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2b14a3ff9..f7e39dae9 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -812,6 +812,7 @@ let main argv = || e.EventStreamId.StartsWith("$") || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId = "ReloadBatchId" || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e From 77327f4006cf1daa28f34853cc43f271e9cc8fae Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 09:46:43 +0100 Subject: [PATCH 076/353] Handle oversive messages --- equinox-sync/Ingest/Program.fs | 9 +++++---- equinox-sync/Sync/CosmosIngester.fs | 8 +++++--- equinox-sync/Sync/Program.fs | 11 ++++++----- 3 files changed, 16 insertions(+), 12 deletions(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index e509f1be7..0751e3d87 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -236,6 +236,7 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel | true, res -> incr resultsHandled match states.HandleWriteResult res with + | (stream, _), CosmosIngester.TooLarge -> CosmosIngester.category stream |> badCats.Ingest | (stream, _), CosmosIngester.Malformed -> CosmosIngester.category stream |> badCats.Ingest | _, CosmosIngester.RateLimited -> rateLimited <- rateLimited + 1 | _, CosmosIngester.TimedOut -> timedOut <- timedOut + 1 @@ -265,10 +266,10 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 states.Dump log - static member Run log conn (spec : ReaderSpec, tryMapEvent) (ctx : Equinox.Cosmos.Core.CosmosContext) (writerQueueLen, writerCount, readerQueueLen) = async { + static member Run log conn (spec : ReaderSpec, tryMapEvent) (ctx : Equinox.Cosmos.Core.CosmosContext) (writerCount, readerQueueLen) = async { let! ct = Async.CancellationToken let! max = establishMax conn - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount, writerQueueLen) + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount) let readers = Readers(conn, spec, tryMapEvent, writers.Enqueue, max, ct) let instance = Coordinator(log, writers, ct, readerQueueLen) let! _ = Async.StartChild <| writers.Pump() @@ -298,7 +299,7 @@ let main argv = Logging.initialize args.Verbose args.ConsoleMinLevel args.MaybeSeqEndpoint let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) |> Async.RunSynchronously let readerSpec = args.BuildFeedParams() - let writerQueueLen, writerCount, readerQueueLen = 2048,64,4096*10*10 + let writerCount, readerQueueLen = 64,4096*10*10 let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let ctx = let destination = cosmos.Connect "SyncTemplate.Ingester" |> Async.RunSynchronously @@ -313,7 +314,7 @@ let main argv = || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e - Coordinator.Run Log.Logger source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerQueueLen, writerCount, readerQueueLen) |> Async.RunSynchronously + Coordinator.Run Log.Logger source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 4e3bb16f4..0591bd601 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -12,7 +12,7 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] -let cosmosPayloadLimit = 2 * 1024 * 1024 - 1024 (*fudge*) - 4096 +let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)2048 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 128 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } @@ -50,10 +50,11 @@ module Writer = log.Debug("Result: {res}",ress) return ress with e -> return Exn (e, batch) } - let (|TimedOutMessage|RateLimitedMessage|MalformedMessage|Other|) (e: exn) = + let (|TimedOutMessage|RateLimitedMessage|TooLargeMessage|MalformedMessage|Other|) (e: exn) = match string e with | m when m.Contains "Microsoft.Azure.Documents.RequestTimeoutException" -> TimedOutMessage | m when m.Contains "Microsoft.Azure.Documents.RequestRateTooLargeException" -> RateLimitedMessage + | m when m.Contains "Microsoft.Azure.Documents.RequestEntityTooLargeException" -> TooLargeMessage | m when m.Contains "SyntaxError: JSON.parse Error: Unexpected input at position" || m.Contains "SyntaxError: JSON.parse Error: Invalid character at position" -> MalformedMessage | _ -> Other @@ -129,7 +130,7 @@ type CatStats() = member __.Clear() = cats.Clear() member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd -type ResultKind = TimedOut | RateLimited | Malformed | Ok +type ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Ok type StreamStates() = let states = Dictionary() @@ -170,6 +171,7 @@ type StreamStates() = match exn with | Writer.RateLimitedMessage -> RateLimited, false | Writer.TimedOutMessage -> TimedOut, false + | Writer.TooLargeMessage -> TooLarge, true | Writer.MalformedMessage -> Malformed, true | Writer.Other -> Ok, false __.Add(batch, malformed), r diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index f7e39dae9..8913d92d8 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -360,15 +360,15 @@ module EventStoreSource = let writerResultLog = log.ForContext() let mutable bytesPended, bytesPendedAgg = 0L, 0L let workPended, eventsPended, cycles = ref 0, ref 0, ref 0 - let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 + let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 let progCommitFails, progCommits = ref 0, ref 0 let badCats = CosmosIngester.CatStats() let dumpStats () = - if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then - Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed", - !rateLimited, !timedOut, !malformed) - rateLimited := 0; timedOut := 0; malformed := 0 + if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then + Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", + !rateLimited, !timedOut, !tooLarge, !malformed) + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn bytesPendedAgg <- bytesPendedAgg + bytesPended @@ -417,6 +417,7 @@ module EventStoreSource = match kind with | CosmosIngester.Ok -> res.WriteTo writerResultLog | CosmosIngester.RateLimited -> incr rateLimited + | CosmosIngester.TooLarge -> category stream |> badCats.Ingest; incr tooLarge | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed | CosmosIngester.TimedOut -> incr timedOut | CoordinationWork.ProgressResult (Choice1Of2 epoch) -> From 3c2670b1e59e3b81fd3c2bf24d09f711b2b6f9fb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 09:47:44 +0100 Subject: [PATCH 077/353] Add PurchaseOrder restriction --- equinox-sync/Sync/Program.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 8913d92d8..9c6ae2fd2 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -813,7 +813,8 @@ let main argv = || e.EventStreamId.StartsWith("$") || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId = "ReloadBatchId" + || e.EventStreamId = "ReloadBatchId" // does not start at 0 + || e.EventStreamId = "PurchaseOrder-5791" // Too large || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e From 7374c35911a0fd0d1502f3933c8003b251b41142 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 15:21:54 +0100 Subject: [PATCH 078/353] Blacklist InventoryLog --- equinox-sync/Sync/Program.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 9c6ae2fd2..90a7d2262 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -813,6 +813,7 @@ let main argv = || e.EventStreamId.StartsWith("$") || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused || e.EventStreamId = "ReloadBatchId" // does not start at 0 || e.EventStreamId = "PurchaseOrder-5791" // Too large || e.EventStreamId.EndsWith("_checkpoint") From aee674e6212fd6b6c706d0a39faf42ca1a2d9829 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 16:41:27 +0100 Subject: [PATCH 079/353] Update filters --- equinox-sync/Ingest/Program.fs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 0751e3d87..27b4f2973 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -311,6 +311,9 @@ let main argv = || e.EventStreamId.StartsWith("$") || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused + || e.EventStreamId = "ReloadBatchId" // does not start at 0 + || e.EventStreamId = "PurchaseOrder-5791" // Too large || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e From 0016675ff01086029699f43bf9ecc7f77a60f858 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 16:53:43 +0100 Subject: [PATCH 080/353] Catch --- equinox-sync/Ingest/Program.fs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 27b4f2973..950a314df 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -267,14 +267,15 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel states.Dump log static member Run log conn (spec : ReaderSpec, tryMapEvent) (ctx : Equinox.Cosmos.Core.CosmosContext) (writerCount, readerQueueLen) = async { - let! ct = Async.CancellationToken - let! max = establishMax conn - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount) - let readers = Readers(conn, spec, tryMapEvent, writers.Enqueue, max, ct) - let instance = Coordinator(log, writers, ct, readerQueueLen) - let! _ = Async.StartChild <| writers.Pump() - let! _ = Async.StartChild <| readers.Pump() - let! _ = Async.StartChild(async { instance.Pump() }) + try let! ct = Async.CancellationToken + let! max = establishMax conn + let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount) + let readers = Readers(conn, spec, tryMapEvent, writers.Enqueue, max, ct) + let instance = Coordinator(log, writers, ct, readerQueueLen) + let! _ = Async.StartChild <| writers.Pump() + let! _ = Async.StartChild <| readers.Pump() + let! _ = Async.StartChild(async { instance.Pump() }) in () + with e -> Log.Error(e,"Exiting") do! Async.AwaitKeyboardInterrupt() } // Illustrates how to emit direct to the Console using Serilog @@ -297,7 +298,7 @@ open Equinox.EventStore let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ConsoleMinLevel args.MaybeSeqEndpoint - let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) |> Async.RunSynchronously + let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.Random) |> Async.RunSynchronously let readerSpec = args.BuildFeedParams() let writerCount, readerQueueLen = 64,4096*10*10 let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu From 2ef2e58972a0ceed33bd28e79a9af5c15dd85d78 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 16:55:52 +0100 Subject: [PATCH 081/353] async --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 8cdfcaf56..2e7281d35 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -205,7 +205,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let! t,res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time + let! t,res = async { return! pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch } |> Stopwatch.Time match res with | PullResult.EndOfTranche -> Log.Warning("completed tranche in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) From 2a6a80147e0781fadb47cfe43f34b3fc6d54784b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 17:03:45 +0100 Subject: [PATCH 082/353] timing --- equinox-sync/Sync/EventStoreSource.fs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 2e7281d35..0d0031343 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -205,14 +205,17 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let! t,res = async { return! pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch } |> Stopwatch.Time + let sw = Stopwatch.StartNew() + let! res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch + let e = sw.Elapsed + //let! t,res = async { return! pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch } |> Stopwatch.Time match res with | PullResult.EndOfTranche -> - Log.Warning("completed tranche in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) + Log.Warning("completed tranche in {ms:n3}s", e.TotalSeconds) __.OverallStats.DumpIfIntervalExpired() return false | PullResult.Eof -> - Log.Warning("completed tranche AND REACHED THE END in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) + Log.Warning("completed tranche AND REACHED THE END in {ms:n3}s", e.TotalSeconds) __.OverallStats.DumpIfIntervalExpired(true) return true | PullResult.Exn e -> From 708c64fa04efeb9400336d97221e56be08bfab09 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 17:34:47 +0100 Subject: [PATCH 083/353] rework Ingester logging --- equinox-sync/Ingest/Program.fs | 28 ++++++++++++++++++---------- 1 file changed, 18 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 950a314df..80d83a78e 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -143,7 +143,7 @@ module CmdParser = and Arguments(args : ParseResults) = member val EventStore = EventStore.Arguments(args.GetResult Es) member __.Verbose = args.Contains Verbose - member __.ConsoleMinLevel = if args.Contains VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning + member __.VerboseConsole = args.Contains VerboseConsole member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None member __.StartingBatchSize = args.GetResult(BatchSize,4096) member __.MinBatchSize = args.GetResult(MinBatchSize,512) @@ -156,7 +156,7 @@ module CmdParser = | _, _, Some p -> Percentage p | None, None, None when args.GetResults Stream <> [] -> StreamList (args.GetResults Stream) | None, None, None -> Start - Log.Warning("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes covering from {startPos}", + Log.Information("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes covering from {startPos}", x.MinBatchSize, x.StartingBatchSize, x.Stripes, startPos) { start = startPos; batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } @@ -173,7 +173,7 @@ type Readers(conn, spec : ReaderSpec, tryMapEvent, postBatch, max : EventStore.C posFromChunk nextChunk let mutable remainder = let startAt (startPos : EventStore.ClientAPI.Position) = - Log.Warning("Start Position {pos} (chunk {chunk}, {pct:p1})", + Log.Information("Start Position {pos} (chunk {chunk}, {pct:p1})", startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition) let nextPos = posFromChunkAfter startPos work.AddTranche(startPos, nextPos, max) @@ -229,6 +229,7 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let badCats = CosmosIngester.CatStats() let progressTimer = Stopwatch.StartNew() + let writerResultLog = log.ForContext() while not cancellationToken.IsCancellationRequested do let mutable moreResults, rateLimited, timedOut = true, 0, 0 while moreResults do @@ -240,7 +241,7 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel | (stream, _), CosmosIngester.Malformed -> CosmosIngester.category stream |> badCats.Ingest | _, CosmosIngester.RateLimited -> rateLimited <- rateLimited + 1 | _, CosmosIngester.TimedOut -> timedOut <- timedOut + 1 - | _, CosmosIngester.Ok -> res.WriteTo log + | _, CosmosIngester.Ok -> res.WriteTo writerResultLog | false, _ -> moreResults <- false if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) let mutable t = Unchecked.defaultof<_> @@ -260,7 +261,7 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) if progressTimer.ElapsedMilliseconds > intervalMs then progressTimer.Restart() - Log.Warning("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", + Log.Information("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 @@ -281,23 +282,30 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog module Logging = - let initialize verbose consoleMinLevel maybeSeqEndpoint = + let initialize verbose verboseConsole maybeSeqEndpoint = Log.Logger <- + let ingesterLevel = if verboseConsole then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Information LoggerConfiguration() .Destructure.FSharpTypes() .Enrich.FromLogContext() |> fun c -> if verbose then c.MinimumLevel.Debug() else c + |> fun c -> c.MinimumLevel.Override(typeof.FullName, ingesterLevel) + |> fun c -> let generalLevel = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning + c.MinimumLevel.Override(typeof.FullName, generalLevel) + .MinimumLevel.Override(typeof.FullName, generalLevel) + .MinimumLevel.Override(typeof.FullName, generalLevel) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {NewLine}{Exception}" - c.WriteTo.Console(consoleMinLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) + c.WriteTo.Console(ingesterLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Literate, outputTemplate=t) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() + Log.ForContext() open Equinox.EventStore [] let main argv = try let args = CmdParser.parse argv - Logging.initialize args.Verbose args.ConsoleMinLevel args.MaybeSeqEndpoint + let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.Random) |> Async.RunSynchronously let readerSpec = args.BuildFeedParams() let writerCount, readerQueueLen = 64,4096*10*10 @@ -305,7 +313,7 @@ let main argv = let ctx = let destination = cosmos.Connect "SyncTemplate.Ingester" |> Async.RunSynchronously let colls = Equinox.Cosmos.CosmosCollections(cosmos.Database, cosmos.Collection) - Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.Logger) + Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.ForContext()) let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = match x.Event with | e when not e.IsJson @@ -318,7 +326,7 @@ let main argv = || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e - Coordinator.Run Log.Logger source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerCount, readerQueueLen) |> Async.RunSynchronously + Coordinator.Run log source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerCount, readerQueueLen) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 From f1334d803d3301f17b7c38bfc4b3d91aceff4fa6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 17:38:14 +0100 Subject: [PATCH 084/353] Fix tranche logging --- equinox-sync/Sync/EventStoreSource.fs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 0d0031343..0b85c10ad 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -205,17 +205,14 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Tranche (range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let sw = Stopwatch.StartNew() - let! res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch - let e = sw.Elapsed - //let! t,res = async { return! pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch } |> Stopwatch.Time + let! t, res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with | PullResult.EndOfTranche -> - Log.Warning("completed tranche in {ms:n3}s", e.TotalSeconds) + Log.Warning("completed tranche in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired() return false | PullResult.Eof -> - Log.Warning("completed tranche AND REACHED THE END in {ms:n3}s", e.TotalSeconds) + Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired(true) return true | PullResult.Exn e -> From 40d2e0e9da3039c5e62fe2263159811913913fde Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 17:41:31 +0100 Subject: [PATCH 085/353] Clean Ready logging --- equinox-sync/Sync/CosmosIngester.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 0591bd601..d8dfa4c87 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -234,8 +234,9 @@ type StreamStates() = waitingB <- waitingB + sz log.Information("Synced {synced} Dirty {dirty} Ready {ready}/{readyMb:n1}MB Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB) + if readyCats.Any then log.Information("Categories Ready {readyCats} (MB)", readyCats.StatsDescending) + if readyCats.Any then log.Information("Streams Ready {readyStreams} (MB)", Seq.truncate 5 readyStreams.StatsDescending) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) - if readyCats.Any then log.Information("Ready {readyCats}\nTop Streams (MB) {readyStreams}", readyCats.StatsDescending, Seq.truncate 10 readyStreams.StatsDescending) type RefCounted<'T> = { mutable refCount: int; value: 'T } From eaf1ad02b5a386bc469b5421d7bb1934de1f8afd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 17:42:14 +0100 Subject: [PATCH 086/353] Back to normal Code style --- equinox-sync/Ingest/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 80d83a78e..bc5cf9beb 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -295,7 +295,7 @@ module Logging = .MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, generalLevel) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {NewLine}{Exception}" - c.WriteTo.Console(ingesterLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Literate, outputTemplate=t) + c.WriteTo.Console(ingesterLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() Log.ForContext() From 327c73c5fa9a736c7e9a94aad63a3400c5d70141 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 17:49:15 +0100 Subject: [PATCH 087/353] Tail recursion --- equinox-sync/Sync/CosmosIngester.fs | 10 +++++----- equinox-sync/Sync/EventStoreSource.fs | 8 +++----- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index d8dfa4c87..b9591b37f 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -182,8 +182,8 @@ type StreamStates() = | Some stream -> match states.[stream].TryGap() with - | None -> aux () | Some (pos,count) -> Some (stream,pos,int count) + | None -> aux () aux () member __.TryReady(isBusy) = let blocked = ResizeArray() @@ -195,10 +195,7 @@ type StreamStates() = match states.[stream] with | s when not s.IsReady -> aux () | state -> - if isBusy stream then - blocked.Add(stream) |> ignore - aux () - else + if (not << isBusy) stream then let h = state.queue |> Array.head let mutable bytesBudget = cosmosPayloadLimit @@ -209,6 +206,9 @@ type StreamStates() = // Reduce the item count when we don't yet know the write position count <= (if Option.isNone state.write then 10 else 4096) && (bytesBudget >= 0 || count = 1) Some { stream = stream; span = { index = h.index; events = h.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } + else + blocked.Add(stream) |> ignore + aux () let res = aux () for x in blocked do markDirty x res diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 0b85c10ad..627b3c145 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -146,11 +146,9 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) - if range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream then - sw.Restart() // restart the clock as we hand off back to the Reader - return! aux () - else - return currentSlice.IsEndOfStream } + if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else + sw.Restart() // restart the clock as we hand off back to the Reader + return! aux () } async { try let! eof = aux () return if eof then Eof else EndOfTranche From 2d1c1b61d3082c01e8f73f28b46e908a5f93ec60 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 18:02:21 +0100 Subject: [PATCH 088/353] More logging cleanup --- equinox-sync/Ingest/Program.fs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index bc5cf9beb..3b336d8e8 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -225,7 +225,7 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel member __.Pump() = let _ = writers.Result.Subscribe __.HandleWriteResult // codependent, wont worry about unsubcribing let fiveMs = TimeSpan.FromMilliseconds 5. - let mutable bytesPended = 0L + let mutable bytesPended, bytesPendedAgg = 0L, 0L let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let badCats = CosmosIngester.CatStats() let progressTimer = Stopwatch.StartNew() @@ -261,10 +261,12 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) if progressTimer.ElapsedMilliseconds > intervalMs then progressTimer.Restart() - Log.Information("Ingested {ingestions}; Sent {queued} req {events} events; Completed {completed} reqs; Egress {gb:n3}GB", - !ingestionsHandled, !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) + Log.Information("Ingested {ingestions}", !ingestionsHandled) + bytesPendedAgg <- bytesPendedAgg + bytesPended + Log.Information("Writer Throughput {queued} reqs {events} events {mb:n}MB; Completed {completed} reqs; Egress {gb:n3}GB", + !workPended, !eventsPended, mb bytesPended, !resultsHandled, mb bytesPendedAgg / 1024.) if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() - ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0 + ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0; bytesPended <- 0L states.Dump log static member Run log conn (spec : ReaderSpec, tryMapEvent) (ctx : Equinox.Cosmos.Core.CosmosContext) (writerCount, readerQueueLen) = async { @@ -308,7 +310,7 @@ let main argv = let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.Random) |> Async.RunSynchronously let readerSpec = args.BuildFeedParams() - let writerCount, readerQueueLen = 64,4096*10*10 + let writerCount, readerQueueLen = 128,4096*10*10 let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu let ctx = let destination = cosmos.Connect "SyncTemplate.Ingester" |> Async.RunSynchronously From b6913f7e5ae28be7baeff167eee586a4f65631b3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 18:11:44 +0100 Subject: [PATCH 089/353] More tailrec --- equinox-sync/Ingest/Program.fs | 7 +++---- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 3b336d8e8..86213d44a 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -224,7 +224,6 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel member __.HandleWriteResult = results.Enqueue member __.Pump() = let _ = writers.Result.Subscribe __.HandleWriteResult // codependent, wont worry about unsubcribing - let fiveMs = TimeSpan.FromMilliseconds 5. let mutable bytesPended, bytesPendedAgg = 0L, 0L let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let badCats = CosmosIngester.CatStats() @@ -243,10 +242,10 @@ type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancel | _, CosmosIngester.TimedOut -> timedOut <- timedOut + 1 | _, CosmosIngester.Ok -> res.WriteTo writerResultLog | false, _ -> moreResults <- false - if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) + if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) let mutable t = Unchecked.defaultof<_> - let mutable toIngest = 4096 * 5 - while work.TryTake(&t,fiveMs) && toIngest > 0 do + let mutable toIngest = 4096 * 2 + while work.TryTake(&t) && toIngest > 0 do incr ingestionsHandled toIngest <- toIngest - 1 states.Add t |> ignore diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index b9591b37f..5d98bafdc 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -193,8 +193,7 @@ type StreamStates() = | Some stream -> match states.[stream] with - | s when not s.IsReady -> aux () - | state -> + | state when state.IsReady -> if (not << isBusy) stream then let h = state.queue |> Array.head @@ -209,6 +208,7 @@ type StreamStates() = else blocked.Add(stream) |> ignore aux () + | _ -> aux () let res = aux () for x in blocked do markDirty x res From 9fa85fdba0cd9883d5e839cfc477fa3d8c8880a0 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 17 Apr 2019 23:54:39 +0100 Subject: [PATCH 090/353] Log finessing --- equinox-sync/Sync/CosmosIngester.fs | 11 ++++------ equinox-sync/Sync/Program.fs | 33 +++++++++++++++++------------ 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 5d98bafdc..f39577fe1 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -149,10 +149,7 @@ type StreamStates() = let updated = StreamState.combine current state states.[stream] <- updated if updated.IsReady then - //if (not << dirty.Contains) stream then Log.Information("Dirty {s} {w} {sz}", (stream : string), updated.write, updated.Size) markDirty stream - //elif Option.isNone state.write then - // Log.Information("None {s} {w} {sz}", stream, updated.write, updated.Size) stream, updated let updateWritePos stream pos isMalformed span = update stream { write = pos; queue = span; isMalformed = isMalformed } @@ -232,10 +229,10 @@ type StreamStates() = waitCats.Ingest(category stream) waiting <- waiting + 1 waitingB <- waitingB + sz - log.Information("Synced {synced} Dirty {dirty} Ready {ready}/{readyMb:n1}MB Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", - synced, dirty.Count, ready, mb readyB, waiting, mb waitingB, malformed, mb malformedB) - if readyCats.Any then log.Information("Categories Ready {readyCats} (MB)", readyCats.StatsDescending) - if readyCats.Any then log.Information("Streams Ready {readyStreams} (MB)", Seq.truncate 5 readyStreams.StatsDescending) + log.Information("Ready {ready}/{readyMb:n1}MB Dirty {dirty} Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", + ready, mb readyB, dirty.Count, waiting, mb waitingB, malformed, mb malformedB, synced) + if readyCats.Any then log.Information("Ready Categories {readyCats} (MB)", readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams(MB) {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) type RefCounted<'T> = { mutable refCount: int; value: 'T } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 90a7d2262..189fdf246 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -365,36 +365,41 @@ module EventStoreSource = let progCommitFails, progCommits = ref 0, ref 0 let badCats = CosmosIngester.CatStats() let dumpStats () = - if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - Log.Warning("Writer exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", - !rateLimited, !timedOut, !tooLarge, !malformed) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 - if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Writer Throughput {cycles} cycles {queued} reqs {events} events {mb:n}MB; Completed {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", - !cycles, !workPended, !eventsPended, mb bytesPended, results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPendedAgg / 1024.) + Log.Information("Ingestion {cycles} cycles {queued} reqs {events} events {mb:n}MB; Pending {pendingBatches}/{maxPendingBatches}", + !cycles, !workPended, !eventsPended, mb bytesPended, pendingBatchCount, maxPendingBatches) cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L + + Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPendedAgg / 1024.) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then + Log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", + !rateLimited, !timedOut, !tooLarge, !malformed) + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 + if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - let throttle = shouldThrottle () - let level = if not throttle then Events.LogEventLevel.Debug else Events.LogEventLevel.Information - Log.Write(level, "Pending Batches {pb}", pendingBatchCount) + let pendingLevel = float pendingBatchCount / float maxPendingBatches |> function + | x when x > 0.8 -> Events.LogEventLevel.Warning + | x when x > 0.5 -> Events.LogEventLevel.Information + | _ -> Events.LogEventLevel.Debug + Log.Write(pendingLevel, "Pending Batches {pendingBatches} of {maxPendingBatches}", pendingBatchCount, maxPendingBatches) if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> - log.Error("Progress Epoch @ {validated}; writing failing: {failures} failures ({commits} successful commits)", + log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits)", Option.toNullable validatedEpoch, !progCommitFails, !progCommits) | Some committed when !progCommitFails <> 0 -> - log.Warning("Progress Epoch @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", + log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) | Some committed -> - log.Information("Progress Epoch @ {validated} (committed: {committed}, {commits} commits)", + log.Information("Progress @ {validated} (committed: {committed}, {commits} commits)", Option.toNullable validatedEpoch, committed, !progCommits) progCommits := 0; progCommitFails := 0 else - log.Information("Progress Epoch @ {validated} (committed: {committed})", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + log.Information("Progress @ {validated} (committed: {committed})", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) buffer.Dump log let tryDumpStats = every statsIntervalMs dumpStats let handle = function From a024ab1939b659b28a7888c46c9374a85e674611 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:05:49 +0100 Subject: [PATCH 091/353] Stats layout --- equinox-sync/Sync/Program.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 189fdf246..c817c822e 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -367,12 +367,12 @@ module EventStoreSource = let dumpStats () = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Ingestion {cycles} cycles {queued} reqs {events} events {mb:n}MB; Pending {pendingBatches}/{maxPendingBatches}", - !cycles, !workPended, !eventsPended, mb bytesPended, pendingBatchCount, maxPendingBatches) + Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB Uncomitted {pendingBatches}/{maxPendingBatches}", + !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024., pendingBatchCount, maxPendingBatches) cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L - Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns); Egress {gb:n3}GB", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn, mb bytesPendedAgg / 1024.) + Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then Log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", From 8b386e9e8c7a99f3dc7a38deb887f1c4e8c69c53 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:13:09 +0100 Subject: [PATCH 092/353] Reorder report --- equinox-sync/Sync/Program.fs | 28 ++++++++++++---------------- 1 file changed, 12 insertions(+), 16 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index c817c822e..5197ed0a3 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -367,10 +367,12 @@ module EventStoreSource = let dumpStats () = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB Uncomitted {pendingBatches}/{maxPendingBatches}", - !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024., pendingBatchCount, maxPendingBatches) + Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB", + !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024.) cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L + buffer.Dump log + Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; @@ -380,27 +382,21 @@ module EventStoreSource = rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - let pendingLevel = float pendingBatchCount / float maxPendingBatches |> function - | x when x > 0.8 -> Events.LogEventLevel.Warning - | x when x > 0.5 -> Events.LogEventLevel.Information - | _ -> Events.LogEventLevel.Debug - Log.Write(pendingLevel, "Pending Batches {pendingBatches} of {maxPendingBatches}", pendingBatchCount, maxPendingBatches) - if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> - log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits)", - Option.toNullable validatedEpoch, !progCommitFails, !progCommits) + log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) | Some committed when !progCommitFails <> 0 -> - log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", - Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) + log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) | Some committed -> - log.Information("Progress @ {validated} (committed: {committed}, {commits} commits)", - Option.toNullable validatedEpoch, committed, !progCommits) + log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) progCommits := 0; progCommitFails := 0 else - log.Information("Progress @ {validated} (committed: {committed})", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - buffer.Dump log + log.Information("Progress @ {validated} (committed: {committed}) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) let tryDumpStats = every statsIntervalMs dumpStats let handle = function | CoordinationWork.Unbatched item -> From 17a8e19ce912c2aa309450339ca5ac81937ab44c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:20:46 +0100 Subject: [PATCH 093/353] Tail Percent --- equinox-sync/Sync/EventStoreSource.fs | 10 +++++----- equinox-sync/Sync/Program.fs | 4 ++-- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 627b3c145..77f23d6e1 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -158,7 +158,7 @@ type [] Work = | Stream of name: string * batchSize: int | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int | Tranche of range: Range * batchSize : int - | Tail of pos: Position * interval: TimeSpan * batchSize : int + | Tail of pos: Position * max : Position * interval: TimeSpan * batchSize : int type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() @@ -173,8 +173,8 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) member __.AddTranche(pos, nextPos, max, ?batchSizeOverride) = __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) - member __.AddTail(pos, interval, ?batchSizeOverride) = - work.Enqueue <| Work.Tail (pos, interval, defaultArg batchSizeOverride batchSize) + member __.AddTail(pos, max, interval, ?batchSizeOverride) = + work.Enqueue <| Work.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() member __.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, work) = async { @@ -219,8 +219,8 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = __.OverallStats.DumpIfIntervalExpired() __.AddTranche(range, bs) return false - | Tail (pos, interval, batchSize) -> - let mutable count, pauses, batchSize, range = 0, 0, batchSize, Range(pos, None) + | Tail (pos, max, interval, batchSize) -> + let mutable count, pauses, batchSize, range = 0, 0, batchSize, Range(pos, None, max) let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds let tailSw = Stopwatch.StartNew() diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 5197ed0a3..bfe8970dd 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -312,7 +312,7 @@ module EventStoreSource = let dop = new SemaphoreSlim(maxDop) let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) member __.HasCapacity = work.QueueCount < dop.CurrentCount - member __.AddTail(startPos, interval) = work.AddTail(startPos, interval) + member __.AddTail(startPos, max, interval) = work.AddTail(startPos, max, interval) member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) member __.Pump(postItem, shouldTail, postBatch) = async { let! ct = Async.CancellationToken @@ -502,7 +502,7 @@ module EventStoreSource = float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) return startPos } let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) - readers.AddTail(startPos, spec.tailInterval) + readers.AddTail(startPos, max, spec.tailInterval) let progress = Checkpoint.ProgressWriter(checkpoints.Commit) let coordinator = Coordinator(log, readers, cosmosContext, maxWriters, progress, maxPendingBatches=maxPendingBatches) do! coordinator.Pump() } From 648eb82dac52e8d9cedb9eaf1b2baba6914ccb96 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:26:54 +0100 Subject: [PATCH 094/353] Finesse percentage --- equinox-sync/Sync/EventStoreSource.fs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 77f23d6e1..dab6c1759 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -89,7 +89,10 @@ type Range(start, sliceEnd : Position option, ?max : Position) = member __.PositionAsRangePercentage = match max with | None -> Double.NaN - | Some max -> float __.Current.CommitPosition/float max.CommitPosition + | Some max -> + match float __.Current.CommitPosition/float max.CommitPosition with + | p when p > 100. -> Double.NaN + | x -> x // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 From c49b75552654da027a522e4d7638e35893407e7e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:34:53 +0100 Subject: [PATCH 095/353] Fix Cats rendering --- equinox-sync/Sync/CosmosIngester.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index f39577fe1..3a8757017 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -222,7 +222,7 @@ type StreamStates() = malformedB <- malformedB + sz | sz when state.IsReady -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%A" stream state.write, mb sz |> int64) + readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) ready <- ready + 1 readyB <- readyB + sz | sz -> @@ -231,8 +231,8 @@ type StreamStates() = waitingB <- waitingB + sz log.Information("Ready {ready}/{readyMb:n1}MB Dirty {dirty} Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", ready, mb readyB, dirty.Count, waiting, mb waitingB, malformed, mb malformedB, synced) - if readyCats.Any then log.Information("Ready Categories {readyCats} (MB)", readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams(MB) {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + if readyCats.Any then log.Information("Ready Categories {readyCats} (Category, events)", readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams {readyStreams} (Name@Pos, MB)", Seq.truncate 5 readyStreams.StatsDescending) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) type RefCounted<'T> = { mutable refCount: int; value: 'T } From de3e735d7b62be14dce666542582fdf0725223a9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:43:10 +0100 Subject: [PATCH 096/353] Fix read layout --- equinox-sync/Sync/EventStoreSource.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index dab6c1759..5a65dd130 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -90,9 +90,9 @@ type Range(start, sliceEnd : Position option, ?max : Position) = match max with | None -> Double.NaN | Some max -> - match float __.Current.CommitPosition/float max.CommitPosition with - | p when p > 100. -> Double.NaN - | x -> x + match __.Current.CommitPosition, max.CommitPosition with + | p,m when p > m -> Double.NaN + | p,m -> float p / float m // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 @@ -146,7 +146,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:#0.000}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else From 2368eeb93bdd71b0a2b56c19f232ec3c33b600ba Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:48:24 +0100 Subject: [PATCH 097/353] mb --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 5a65dd130..5d0794781 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -146,7 +146,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:#0.000}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:,4#0.0}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else From cb0be31dc8cb5613e8f766729d5f3c87a6da6391 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:53:17 +0100 Subject: [PATCH 098/353] MB concession --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 5d0794781..f00dbf448 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -146,7 +146,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:,4#0.0}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n0}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else From 6aff4df8059252a1f9e98cdd85bc5d50e48c8c74 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 00:55:36 +0100 Subject: [PATCH 099/353] Typo --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index f00dbf448..e9f8bb83b 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -146,7 +146,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n0}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else From 20acd89211c48ec78e7bbe646174af69eaf3d5fe Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 01:57:51 +0100 Subject: [PATCH 100/353] Add stats printer --- equinox-sync/Sync/CosmosIngester.fs | 4 +- equinox-sync/Sync/Program.fs | 75 +++++++++++++++++++++++++++++ 2 files changed, 77 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 3a8757017..45fbf9f5e 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -231,8 +231,8 @@ type StreamStates() = waitingB <- waitingB + sz log.Information("Ready {ready}/{readyMb:n1}MB Dirty {dirty} Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", ready, mb readyB, dirty.Count, waiting, mb waitingB, malformed, mb malformedB, synced) - if readyCats.Any then log.Information("Ready Categories {readyCats} (Category, events)", readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams {readyStreams} (Name@Pos, MB)", Seq.truncate 5 readyStreams.StatsDescending) + if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) type RefCounted<'T> = { mutable refCount: int; value: 'T } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index bfe8970dd..8f586c6c0 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -752,6 +752,79 @@ module CosmosSource = // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog module Logging = + module RuCounters = + open Serilog.Events + open Equinox.Cosmos.Store + + let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds + + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function + | Log.Tip (Stats s) + | Log.TipNotFound (Stats s) + | Log.TipNotModified (Stats s) + | Log.Query (_,_, (Stats s)) -> CosmosReadRc s + // slices are rolled up into batches so be sure not to double-count + | Log.Response (_,(Stats s)) -> CosmosResponseRc s + | Log.SyncSuccess (Stats s) + | Log.SyncConflict (Stats s) -> CosmosWriteRc s + | Log.SyncResync (Stats s) -> CosmosResyncRc s + let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function + | (:? ScalarValue as x) -> Some x.Value + | _ -> None + let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = + match logEvent.Properties.TryGetValue("cosmosEvt") with + | true, SerilogScalar (:? Log.Event as e) -> Some e + | _ -> None + type RuCounter = + { mutable rux100: int64; mutable count: int64; mutable ms: int64 } + static member Create() = { rux100 = 0L; count = 0L; ms = 0L } + member __.Ingest (ru, ms) = + System.Threading.Interlocked.Increment(&__.count) |> ignore + System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore + System.Threading.Interlocked.Add(&__.ms, ms) |> ignore + type RuCounterSink() = + static member val Read = RuCounter.Create() with get, set + static member val Write = RuCounter.Create() with get, set + static member val Resync = RuCounter.Create() with get, set + static member Reset() = + RuCounterSink.Read <- RuCounter.Create() + RuCounterSink.Write <- RuCounter.Create() + RuCounterSink.Resync <- RuCounter.Create() + interface Serilog.Core.ILogEventSink with + member __.Emit logEvent = logEvent |> function + | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats + | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats + | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats + | _ -> () + + let dumpStats duration (log: Serilog.ILogger) = + let stats = + [ "Read", RuCounterSink.Read + "Write", RuCounterSink.Write + "Resync", RuCounterSink.Resync ] + let mutable totalCount, totalRc, totalMs = 0L, 0., 0L + let logActivity name count rc lat = + log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", + name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) + for name, stat in stats do + let ru = float stat.rux100 / 100. + totalCount <- totalCount + stat.count + totalRc <- totalRc + ru + totalMs <- totalMs + stat.ms + logActivity name stat.count ru stat.ms + logActivity "TOTAL" totalCount totalRc totalMs + let measures : (string * (TimeSpan -> float)) list = + [ "s", fun x -> x.TotalSeconds + "m", fun x -> x.TotalMinutes + "h", fun x -> x.TotalHours ] + let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) + let startTaskToDumpStatsEvery (freq : TimeSpan) = + let rec aux () = async { + dumpStats freq Log.Logger + do! Async.Sleep (int freq.TotalMilliseconds) + return! aux () } + Async.Start(aux ()) let initialize verbose changeLogVerbose maybeSeqEndpoint = Log.Logger <- LoggerConfiguration() @@ -770,7 +843,9 @@ module Logging = |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) + |> fun c -> c.WriteTo.Sink(RuCounters.RuCounterSink()) |> fun c -> c.CreateLogger() + Logging.RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) Log.ForContext() [] From b68d55a51486c71db65a761ce978cc760055b19a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 01:59:36 +0100 Subject: [PATCH 101/353] fix --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 8f586c6c0..461d6b5bf 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -845,7 +845,7 @@ module Logging = |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.WriteTo.Sink(RuCounters.RuCounterSink()) |> fun c -> c.CreateLogger() - Logging.RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) + RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) Log.ForContext() [] From 0447ed4ca8948d41b3fa7e435c010559e99dc24c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 08:53:22 +0100 Subject: [PATCH 102/353] Async logging --- equinox-sync/Sync/EventStoreSource.fs | 5 +++-- equinox-sync/Sync/Program.fs | 26 +++++++++++++++++++------- equinox-sync/Sync/Sync.fsproj | 1 + 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index e9f8bb83b..bdde06a4b 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -42,8 +42,9 @@ type OverallStats(?statsInterval) = member __.DumpIfIntervalExpired(?force) = if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then let totalMb = mb totalBytes - Log.Information("Reader Throughput {events} events {gb:n1}GB {mb:n2}MB/s", - totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) + if totalEvents <> 0L then + Log.Information("Reader Throughput {events} events {gb:n1}GB {mb:n2}MB/s", + totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) progressStart.Restart() type SliceStatsBuffer(?interval) = diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 461d6b5bf..11744bc73 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -752,9 +752,14 @@ module CosmosSource = // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog module Logging = + open Serilog.Configuration + open Serilog.Events + open Serilog.Filters module RuCounters = - open Serilog.Events open Equinox.Cosmos.Store + //open Serilog.Configuration + //open Serilog.Events + //open Serilog.Filters let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds @@ -821,12 +826,14 @@ module Logging = for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) let startTaskToDumpStatsEvery (freq : TimeSpan) = let rec aux () = async { - dumpStats freq Log.Logger do! Async.Sleep (int freq.TotalMilliseconds) + dumpStats freq Log.Logger + RuCounterSink.Reset() return! aux () } Async.Start(aux ()) let initialize verbose changeLogVerbose maybeSeqEndpoint = Log.Logger <- + let isEqx = Matching.FromSource() LoggerConfiguration() .Destructure.FSharpTypes() .Enrich.FromLogContext() @@ -839,12 +846,17 @@ module Logging = |> fun c -> let generalLevel = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, generalLevel) - .MinimumLevel.Override(typeof.FullName, generalLevel) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" - c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) - |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) - |> fun c -> c.WriteTo.Sink(RuCounters.RuCounterSink()) - |> fun c -> c.CreateLogger() + let configure (a : LoggerSinkConfiguration) : unit = + a.Logger(fun l -> + (if changeLogVerbose then l else l.Filter.ByExcluding isEqx) + .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) + |> ignore) + .WriteTo.Sink(RuCounters.RuCounterSink()) + |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) + |> ignore + c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) + .CreateLogger() RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) Log.ForContext() diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 787bc1463..30aba3fcf 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -22,6 +22,7 @@ + From ad51b93797beee3819925b9f465f9ae7aa645e08 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 09:03:41 +0100 Subject: [PATCH 103/353] Sink tweaks --- equinox-sync/Sync/Program.fs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 11744bc73..f80f322d9 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -214,8 +214,8 @@ module CmdParser = member __.CategoryFilterFunction : string -> bool = match a.GetResults CategoryBlacklist, a.GetResults CategoryWhitelist with | [], [] -> Log.Information("Not filtering by category"); fun _ -> true - | bad, [] -> let black = Set.ofList bad in Log.Information("Excluding categories: {cats}", black); fun x -> not (black.Contains x) - | [], good -> let white = Set.ofList good in Log.Information("Only copying categories: {cats}", white); fun x -> white.Contains x + | bad, [] -> let black = Set.ofList bad in Log.Warning("Excluding categories: {cats}", black); fun x -> not (black.Contains x) + | [], good -> let white = Set.ofList good in Log.Warning("Only copying categories: {cats}", white); fun x -> white.Contains x | _, _ -> raise (InvalidArguments "BlackList and Whitelist are mutually exclusive; inclusions and exclusions cannot be mixed") #if cosmos member __.Mode = a.GetResult(SourceConnectionMode, Equinox.Cosmos.ConnectionMode.DirectTcp) @@ -826,14 +826,13 @@ module Logging = for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) let startTaskToDumpStatsEvery (freq : TimeSpan) = let rec aux () = async { - do! Async.Sleep (int freq.TotalMilliseconds) + do! Async.Sleep freq dumpStats freq Log.Logger RuCounterSink.Reset() return! aux () } Async.Start(aux ()) let initialize verbose changeLogVerbose maybeSeqEndpoint = Log.Logger <- - let isEqx = Matching.FromSource() LoggerConfiguration() .Destructure.FSharpTypes() .Enrich.FromLogContext() @@ -847,14 +846,15 @@ module Logging = c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, generalLevel) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" + let isEqx = Matching.FromSource() let configure (a : LoggerSinkConfiguration) : unit = - a.Logger(fun l -> + a.Sink(RuCounters.RuCounterSink()) + .WriteTo.Logger(fun l -> (if changeLogVerbose then l else l.Filter.ByExcluding isEqx) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) - .WriteTo.Sink(RuCounters.RuCounterSink()) - |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) - |> ignore + |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) + |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) .CreateLogger() RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) From 82f3861d927e8049c316871b9abb7d80ad0954b5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 09:09:51 +0100 Subject: [PATCH 104/353] fix async config? --- equinox-sync/Sync/Program.fs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index f80f322d9..b7f9c621b 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -848,15 +848,17 @@ module Logging = |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let isEqx = Matching.FromSource() let configure (a : LoggerSinkConfiguration) : unit = - a.Sink(RuCounters.RuCounterSink()) - .WriteTo.Logger(fun l -> + a.Logger(fun l -> + l.WriteTo.Sink(RuCounters.RuCounterSink()) |> ignore) + |> ignore + a.Logger(fun l -> (if changeLogVerbose then l else l.Filter.ByExcluding isEqx) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) - |> ignore) - |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) + |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) - .CreateLogger() + |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) + |> fun c -> c.CreateLogger() RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) Log.ForContext() From 77bc2038ccb2f24ab84d84c8161c37e8c2d90ab8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 09:30:16 +0100 Subject: [PATCH 105/353] Fix log init --- equinox-sync/Sync/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index b7f9c621b..af29be6e6 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -877,7 +877,8 @@ let main argv = Equinox.Cosmos.CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve - let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.ForContext()) + let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint + let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, log.ForContext()) #if cosmos let log = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() @@ -893,7 +894,6 @@ let main argv = (leaseId, startFromHere, batchSize, lagFrequency) createSyncHandler #else - let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() From 446fd653a09c45e7c2dde297e1b1488249d433bd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 09:35:23 +0100 Subject: [PATCH 106/353] even earlier --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index af29be6e6..9f230ddae 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -865,6 +865,7 @@ module Logging = [] let main argv = try let args = CmdParser.parse argv + let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) let resolveCheckpointStream = @@ -877,7 +878,6 @@ let main argv = Equinox.Cosmos.CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve - let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, log.ForContext()) #if cosmos let log = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint From 4fe6433b5f942bb62a9959b7d0198163324e1e9b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 09:45:01 +0100 Subject: [PATCH 107/353] separate StoreLog --- equinox-sync/Sync/Program.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 9f230ddae..ec546a62f 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -299,7 +299,7 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -#if !cosmos +#if !cosmos module EventStoreSource = type [] CoordinationWork<'Pos> = | Result of CosmosIngester.Writer.Result @@ -846,12 +846,12 @@ module Logging = c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, generalLevel) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" - let isEqx = Matching.FromSource() let configure (a : LoggerSinkConfiguration) : unit = a.Logger(fun l -> l.WriteTo.Sink(RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> + let isEqx = Matching.FromSource() (if changeLogVerbose then l else l.Filter.ByExcluding isEqx) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) @@ -860,12 +860,12 @@ module Logging = |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) - Log.ForContext() + Log.ForContext(), Log.ForContext() [] let main argv = try let args = CmdParser.parse argv - let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint + let log, storeLog = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) let resolveCheckpointStream = @@ -878,7 +878,7 @@ let main argv = Equinox.Cosmos.CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve - let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, log.ForContext()) + let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, storeLog) #if cosmos let log = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() From a878fc7d505e61ede5fd8a4138af85349548d267 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 09:56:07 +0100 Subject: [PATCH 108/353] Another go --- equinox-sync/Sync/Program.fs | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index ec546a62f..1a951e44a 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -757,9 +757,6 @@ module Logging = open Serilog.Filters module RuCounters = open Equinox.Cosmos.Store - //open Serilog.Configuration - //open Serilog.Events - //open Serilog.Filters let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds @@ -831,28 +828,29 @@ module Logging = RuCounterSink.Reset() return! aux () } Async.Start(aux ()) - let initialize verbose changeLogVerbose maybeSeqEndpoint = + let initialize verbose verboseConsole maybeSeqEndpoint = + let rusSink = RuCounters.RuCounterSink() Log.Logger <- LoggerConfiguration() .Destructure.FSharpTypes() .Enrich.FromLogContext() |> fun c -> // LibLog writes to the global logger, so we need to control the emission if we don't want to pass loggers everywhere - let cfpLevel = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Warning + let cfpLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpLevel) - |> fun c -> let ingesterLevel = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Information + |> fun c -> let ingesterLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c - |> fun c -> let generalLevel = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning + |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning c.MinimumLevel.Override(typeof.FullName, generalLevel) - .MinimumLevel.Override(typeof.FullName, generalLevel) + .MinimumLevel.Override(typeof.FullName, LogEventLevel.Information) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let configure (a : LoggerSinkConfiguration) : unit = a.Logger(fun l -> - l.WriteTo.Sink(RuCounters.RuCounterSink()) |> ignore) + l.WriteTo.Sink(rusSink) |> ignore) |> ignore a.Logger(fun l -> let isEqx = Matching.FromSource() - (if changeLogVerbose then l else l.Filter.ByExcluding isEqx) + (if verboseConsole then l else l.Filter.ByExcluding isEqx) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) |> ignore From efff01e0e4906541f1979536359a9dba78366dab Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 10:09:17 +0100 Subject: [PATCH 109/353] Filter RU logging --- equinox-sync/Sync/Program.fs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 1a951e44a..524f23e99 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -845,12 +845,11 @@ module Logging = .MinimumLevel.Override(typeof.FullName, LogEventLevel.Information) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let configure (a : LoggerSinkConfiguration) : unit = + a.Logger(fun l -> l.WriteTo.Sink(rusSink) |> ignore) |> ignore a.Logger(fun l -> - l.WriteTo.Sink(rusSink) |> ignore) - |> ignore - a.Logger(fun l -> - let isEqx = Matching.FromSource() - (if verboseConsole then l else l.Filter.ByExcluding isEqx) + let isEqx = Matching.FromSource().Invoke + let isCheckpointing = Matching.FromSource().Invoke + (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) |> ignore From 0096d8911f4952cb9c773c580b370bab4cf28576 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 10:33:45 +0100 Subject: [PATCH 110/353] Clean RU Counters sink --- equinox-sync/Sync/Program.fs | 165 +++++++++++++++++------------------ 1 file changed, 79 insertions(+), 86 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 524f23e99..349f0cc40 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -1,7 +1,6 @@ module SyncTemplate.Program open Equinox.Cosmos -open Equinox.Cosmos.Core //#if !eventStore open Equinox.Cosmos.Projection //#endif @@ -299,6 +298,75 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments +module Metrics = + module RuCounters = + open Equinox.Cosmos.Store + open Serilog.Events + + let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds + + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function + | Log.Tip (Stats s) + | Log.TipNotFound (Stats s) + | Log.TipNotModified (Stats s) + | Log.Query (_,_, (Stats s)) -> CosmosReadRc s + // slices are rolled up into batches so be sure not to double-count + | Log.Response (_,(Stats s)) -> CosmosResponseRc s + | Log.SyncSuccess (Stats s) + | Log.SyncConflict (Stats s) -> CosmosWriteRc s + | Log.SyncResync (Stats s) -> CosmosResyncRc s + let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function + | (:? ScalarValue as x) -> Some x.Value + | _ -> None + let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = + match logEvent.Properties.TryGetValue("cosmosEvt") with + | true, SerilogScalar (:? Log.Event as e) -> Some e + | _ -> None + type RuCounter = + { mutable rux100: int64; mutable count: int64; mutable ms: int64 } + static member Create() = { rux100 = 0L; count = 0L; ms = 0L } + member __.Ingest (ru, ms) = + System.Threading.Interlocked.Increment(&__.count) |> ignore + System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore + System.Threading.Interlocked.Add(&__.ms, ms) |> ignore + type RuCounterSink() = + static member val Read = RuCounter.Create() with get, set + static member val Write = RuCounter.Create() with get, set + static member val Resync = RuCounter.Create() with get, set + static member Reset() = + RuCounterSink.Read <- RuCounter.Create() + RuCounterSink.Write <- RuCounter.Create() + RuCounterSink.Resync <- RuCounter.Create() + interface Serilog.Core.ILogEventSink with + member __.Emit logEvent = logEvent |> function + | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats + | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats + | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats + | _ -> () + + let dumpRuStats duration (log: Serilog.ILogger) = + let stats = + [ "Read", RuCounters.RuCounterSink.Read + "Write", RuCounters.RuCounterSink.Write + "Resync", RuCounters.RuCounterSink.Resync ] + let mutable totalCount, totalRc, totalMs = 0L, 0., 0L + let logActivity name count rc lat = + if count <> 0L then + log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", + name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) + for name, stat in stats do + let ru = float stat.rux100 / 100. + totalCount <- totalCount + stat.count + totalRc <- totalRc + ru + totalMs <- totalMs + stat.ms + logActivity name stat.count ru stat.ms + logActivity "TOTAL" totalCount totalRc totalMs + // Yes, there's a minor race here! + RuCounters.RuCounterSink.Reset() + let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] + let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) + #if !cosmos module EventStoreSource = type [] CoordinationWork<'Pos> = @@ -333,8 +401,9 @@ module EventStoreSource = type StartMode = Starting | Resuming | Overridding type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = - let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let sleepIntervalMs = 10 + let statsInterval = defaultArg interval (TimeSpan.FromMinutes 1.) + let statsIntervalMs = int64 statsInterval.TotalMilliseconds + let sleepIntervalMs = 1 let input = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), maxPendingBatches) let results = System.Collections.Concurrent.ConcurrentQueue() let buffer = CosmosIngester.StreamStates() @@ -381,6 +450,7 @@ module EventStoreSource = !rateLimited, !timedOut, !tooLarge, !malformed) rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + Metrics.dumpRuStats statsInterval log if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with @@ -752,84 +822,8 @@ module CosmosSource = // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog module Logging = - open Serilog.Configuration open Serilog.Events - open Serilog.Filters - module RuCounters = - open Equinox.Cosmos.Store - - let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds - - let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function - | Log.Tip (Stats s) - | Log.TipNotFound (Stats s) - | Log.TipNotModified (Stats s) - | Log.Query (_,_, (Stats s)) -> CosmosReadRc s - // slices are rolled up into batches so be sure not to double-count - | Log.Response (_,(Stats s)) -> CosmosResponseRc s - | Log.SyncSuccess (Stats s) - | Log.SyncConflict (Stats s) -> CosmosWriteRc s - | Log.SyncResync (Stats s) -> CosmosResyncRc s - let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function - | (:? ScalarValue as x) -> Some x.Value - | _ -> None - let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = - match logEvent.Properties.TryGetValue("cosmosEvt") with - | true, SerilogScalar (:? Log.Event as e) -> Some e - | _ -> None - type RuCounter = - { mutable rux100: int64; mutable count: int64; mutable ms: int64 } - static member Create() = { rux100 = 0L; count = 0L; ms = 0L } - member __.Ingest (ru, ms) = - System.Threading.Interlocked.Increment(&__.count) |> ignore - System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore - System.Threading.Interlocked.Add(&__.ms, ms) |> ignore - type RuCounterSink() = - static member val Read = RuCounter.Create() with get, set - static member val Write = RuCounter.Create() with get, set - static member val Resync = RuCounter.Create() with get, set - static member Reset() = - RuCounterSink.Read <- RuCounter.Create() - RuCounterSink.Write <- RuCounter.Create() - RuCounterSink.Resync <- RuCounter.Create() - interface Serilog.Core.ILogEventSink with - member __.Emit logEvent = logEvent |> function - | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats - | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats - | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats - | _ -> () - - let dumpStats duration (log: Serilog.ILogger) = - let stats = - [ "Read", RuCounterSink.Read - "Write", RuCounterSink.Write - "Resync", RuCounterSink.Resync ] - let mutable totalCount, totalRc, totalMs = 0L, 0., 0L - let logActivity name count rc lat = - log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", - name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) - for name, stat in stats do - let ru = float stat.rux100 / 100. - totalCount <- totalCount + stat.count - totalRc <- totalRc + ru - totalMs <- totalMs + stat.ms - logActivity name stat.count ru stat.ms - logActivity "TOTAL" totalCount totalRc totalMs - let measures : (string * (TimeSpan -> float)) list = - [ "s", fun x -> x.TotalSeconds - "m", fun x -> x.TotalMinutes - "h", fun x -> x.TotalHours ] - let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) - for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) - let startTaskToDumpStatsEvery (freq : TimeSpan) = - let rec aux () = async { - do! Async.Sleep freq - dumpStats freq Log.Logger - RuCounterSink.Reset() - return! aux () } - Async.Start(aux ()) let initialize verbose verboseConsole maybeSeqEndpoint = - let rusSink = RuCounters.RuCounterSink() Log.Logger <- LoggerConfiguration() .Destructure.FSharpTypes() @@ -844,19 +838,18 @@ module Logging = c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, LogEventLevel.Information) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" - let configure (a : LoggerSinkConfiguration) : unit = - a.Logger(fun l -> l.WriteTo.Sink(rusSink) |> ignore) |> ignore + let configure (a : Configuration.LoggerSinkConfiguration) : unit = + a.Logger(fun l -> + l.WriteTo.Sink(Metrics.RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> - let isEqx = Matching.FromSource().Invoke - let isCheckpointing = Matching.FromSource().Invoke + let isEqx = Filters.Matching.FromSource().Invoke + let isCheckpointing = Filters.Matching.FromSource().Invoke (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) - |> ignore) - |> ignore + |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() - RuCounters.startTaskToDumpStatsEvery (TimeSpan.FromMinutes 1.) Log.ForContext(), Log.ForContext() [] From 3f1cb1a56b7059710f3efa27839540c6023317b7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 10:44:32 +0100 Subject: [PATCH 111/353] Adjust fudge --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 45fbf9f5e..c724256e4 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -13,7 +13,7 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)2048 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 128 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 32 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } From 0abdcd11bb3243bab80d0dbff67195b584c4abd9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 10:52:04 +0100 Subject: [PATCH 112/353] Fix fudge --- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index c724256e4..8502fa82a 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -12,8 +12,8 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] -let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)2048 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 32 +let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 64 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } From 81ade3edb9006681d7d5b23e0d120e1b3a9354f9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 10:55:15 +0100 Subject: [PATCH 113/353] Another --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 8502fa82a..b4b9a66f9 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -13,7 +13,7 @@ let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 64 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] Batch = { stream: string; span: Span } From 172913aca34aca19f1578aefff0dc13d9436b6fa Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 13:44:08 +0100 Subject: [PATCH 114/353] Reorder progress --- equinox-sync/Sync/Program.fs | 23 ++++++++++------------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 349f0cc40..22a4c528f 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -439,19 +439,6 @@ module EventStoreSource = Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB", !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024.) cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L - - buffer.Dump log - - Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; - if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - Log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", - !rateLimited, !timedOut, !tooLarge, !malformed) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 - if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - Metrics.dumpRuStats statsInterval log - if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -467,6 +454,16 @@ module EventStoreSource = else log.Information("Progress @ {validated} (committed: {committed}) Uncomitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) + Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then + Log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", + !rateLimited, !timedOut, !tooLarge, !malformed) + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 + if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + Metrics.dumpRuStats statsInterval log + buffer.Dump log let tryDumpStats = every statsIntervalMs dumpStats let handle = function | CoordinationWork.Unbatched item -> From d2c4a67b56e180418823fe11fc70bc9d4982eb8c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 18 Apr 2019 13:48:35 +0100 Subject: [PATCH 115/353] Even more reorder --- equinox-sync/Sync/Program.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 22a4c528f..95fb29bfa 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -434,11 +434,6 @@ module EventStoreSource = let progCommitFails, progCommits = ref 0, ref 0 let badCats = CosmosIngester.CatStats() let dumpStats () = - let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB", - !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024.) - cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -454,6 +449,11 @@ module EventStoreSource = else log.Information("Progress @ {validated} (committed: {committed}) Uncomitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn + bytesPendedAgg <- bytesPendedAgg + bytesPended + Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB", + !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024.) + cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; From 3138ed957a7a0e38fd8f7581b2a7ee70e6b27145 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 23 Apr 2019 10:17:13 +0100 Subject: [PATCH 116/353] Naming consistency for args --- equinox-projector/Projector/Program.fs | 2 +- equinox-projector/README.md | 4 ++-- equinox-sync/Ingest/Program.fs | 4 ++-- equinox-sync/README.md | 2 +- equinox-sync/Sync/Program.fs | 8 ++++---- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index add139bf7..ededdcfd1 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -73,7 +73,7 @@ module CmdParser = | [] ConsumerGroupName of string | [] LeaseCollectionSuffix of string | [] ForceStartFromHere - | [] BatchSize of int + | [] BatchSize of int | [] LagFreqS of float | [] Verbose | [] ChangeFeedVerbose diff --git a/equinox-projector/README.md b/equinox-projector/README.md index e91b254f8..6e665c1de 100644 --- a/equinox-projector/README.md +++ b/equinox-projector/README.md @@ -46,10 +46,10 @@ This project was generated using: $env:EQUINOX_KAFKA_BROKER="instance.kafka.mysite.com:9092" # or use -b # `default` defines the Projector Group identity - each id has separated state in the aux collection (aka LeaseId) - # `-m 1000` sets the max batch size to 1000 + # `-mi 1000` sets the change feed item count limit to 1000 # `-t topic0` identifies the Kafka topic to which the Projector should write # cosmos specifies the source (if you have specified 3x EQUINOX_COSMOS_* environment vars, no arguments are needed) - dotnet run -p Projector -- default -m 1000 -t topic0 cosmos + dotnet run -p Projector -- default -mi 1000 -t topic0 cosmos # (assuming you've scaled up enough to have >1 range, you can run a second instance in a second console with the same arguments) diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 86213d44a..79bff3715 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -115,8 +115,8 @@ module CmdParser = [] type Parameters = - | [] BatchSize of int - | [] MinBatchSize of int + | [] BatchSize of int + | [] MinBatchSize of int | [] Verbose | [] VerboseConsole | [] LocalSeq diff --git a/equinox-sync/README.md b/equinox-sync/README.md index 57ad3eb50..5a0505f2f 100644 --- a/equinox-sync/README.md +++ b/equinox-sync/README.md @@ -41,7 +41,7 @@ This project was generated using: # (either add environment variables as per step 0 or use -s/-d/-c to specify them) # `defaultSync` defines the Projector Group identity ('LeaseId') - each id has separated state in the aux collection - # `-m 1000` sets the max batch size to 1000 + # `-mi 1000` sets the change feed item count limit to 1000 # `cosmos` specifies the destination (if you have specified 3x EQUINOX_COSMOS_* environment vars, no arguments are needed) # `source -s connection -d database -c collection` specifies the input datasource diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 95fb29bfa..cf3970891 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -67,8 +67,8 @@ module CmdParser = | [] LagFreqS of float | [] ChangeFeedVerbose #else - | [] MinBatchSize of int - | [] MaxPending of int + | [] MinBatchSize of int + | [] MaxPendingBatches of int | [] MaxWriters of int | [] Position of int64 | [] Chunk of int @@ -98,7 +98,7 @@ module CmdParser = | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | MaxPending _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 64" | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" @@ -122,7 +122,7 @@ module CmdParser = member __.VerboseConsole = a.Contains VerboseConsole member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) - member __.MaxPendingBatches = a.GetResult(MaxPending,64) + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,64) member __.MaxWriters = a.GetResult(MaxWriters,64) member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.StreamReaders = a.GetResult(StreamReaders,8) From b4232515d071a088dac306aeefe4689c363d263c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 23 Apr 2019 10:36:35 +0100 Subject: [PATCH 117/353] Fix: remove temp logging --- equinox-sync/Sync/ProgressBatcher.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/equinox-sync/Sync/ProgressBatcher.fs b/equinox-sync/Sync/ProgressBatcher.fs index 78f796a3a..4a5b6799b 100644 --- a/equinox-sync/Sync/ProgressBatcher.fs +++ b/equinox-sync/Sync/ProgressBatcher.fs @@ -28,7 +28,6 @@ type State<'Pos>(?currentPos : 'Pos) = | false, _ -> () | true, batch -> for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do - //Log.Warning("VI {s} {ri}", stream, requiredIndex) match tryGetStreamWritePos stream with | Some index when requiredIndex <= index -> batch.streamToRequiredIndex.Remove stream |> ignore | _ -> () From 275a85a86fb28e6cfd320a00ae88e133d1f19099 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 23 Apr 2019 14:29:26 +0100 Subject: [PATCH 118/353] Fix comment --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index cf3970891..886771fd6 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -713,7 +713,7 @@ module CosmosSource = let mutable coordinator = Unchecked.defaultof<_> let init rangeLog = coordinator <- Coordinator.Start(rangeLog, ctx) - let ingest docs checkpoint : (*streams*)int * (*events*)int = + let ingest docs checkpoint : (*events*)int * (*streams*)int = let events = docs |> Seq.collect transform |> Array.ofSeq coordinator.Submit(checkpoint,events) events.Length, HashSet(seq { for x in events -> x.stream }).Count From 88cbeb023d68be5c5cd76bb0009a81067636273f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 12:15:32 +0100 Subject: [PATCH 119/353] Add Projector State --- equinox-projector/Consumer/Infrastructure.fs | 2 +- .../Projector.Tests/ProgressTests.fs | 38 ++ .../Projector.Tests/Projector.Tests.fsproj | 26 ++ .../Projector.Tests/StreamStateTests.fs | 78 ++++ equinox-projector/Projector/Infrastructure.fs | 30 +- equinox-projector/Projector/Program.fs | 62 ++- equinox-projector/Projector/Projector.fsproj | 2 +- equinox-projector/Projector/State.fs | 439 ++++++++++++++++++ .../equinox-projector-consumer.sln | 10 +- equinox-sync/Sync/CosmosIngester.fs | 2 +- 10 files changed, 663 insertions(+), 26 deletions(-) create mode 100644 equinox-projector/Projector.Tests/ProgressTests.fs create mode 100644 equinox-projector/Projector.Tests/Projector.Tests.fsproj create mode 100644 equinox-projector/Projector.Tests/StreamStateTests.fs create mode 100644 equinox-projector/Projector/State.fs diff --git a/equinox-projector/Consumer/Infrastructure.fs b/equinox-projector/Consumer/Infrastructure.fs index 7ef7ded63..a1524f12f 100644 --- a/equinox-projector/Consumer/Infrastructure.fs +++ b/equinox-projector/Consumer/Infrastructure.fs @@ -44,4 +44,4 @@ type SemaphoreSlim with let! _ = semaphore.Await() try return! workflow finally semaphore.Release() |> ignore - } + } \ No newline at end of file diff --git a/equinox-projector/Projector.Tests/ProgressTests.fs b/equinox-projector/Projector.Tests/ProgressTests.fs new file mode 100644 index 000000000..affd0639d --- /dev/null +++ b/equinox-projector/Projector.Tests/ProgressTests.fs @@ -0,0 +1,38 @@ +module ProgressTests + +open ProjectorTemplate.Projector.State + +open Swensen.Unquote +open Xunit + +let [] ``Empty has zero streams pending or progress to write`` () = + let sut = ProgressState<_>() + let validatedPos, batches = sut.Validate(fun _ -> None) + None =! validatedPos + 0 =! batches + +let [] ``Can add multiple batches`` () = + let sut = ProgressState<_>() + sut.AppendBatch(0,["a",1L; "b",2L]) + sut.AppendBatch(1,["b",2L; "c",3L]) + let validatedPos, batches = sut.Validate(fun _ -> None) + None =! validatedPos + 2 =! batches + +let [] ``Marking Progress Removes batches and updates progress`` () = + let sut = ProgressState<_>() + sut.AppendBatch(0,["a",1L; "b",2L]) + sut.MarkStreamProgress("a",1L) + sut.MarkStreamProgress("b",1L) + let validatedPos, batches = sut.Validate(fun _ -> None) + None =! validatedPos + 1 =! batches + +let [] ``Marking progress is not persistent`` () = + let sut = ProgressState<_>() + sut.AppendBatch(0,["a",1L]) + sut.MarkStreamProgress("a",2L) + sut.AppendBatch(1,["a",1L; "b",2L]) + let validatedPos, batches = sut.Validate(fun _ -> None) + Some 0 =! validatedPos + 1 =! batches \ No newline at end of file diff --git a/equinox-projector/Projector.Tests/Projector.Tests.fsproj b/equinox-projector/Projector.Tests/Projector.Tests.fsproj new file mode 100644 index 000000000..954aa11db --- /dev/null +++ b/equinox-projector/Projector.Tests/Projector.Tests.fsproj @@ -0,0 +1,26 @@ + + + + netcoreapp2.2 + + false + false + + + + + + + + + + + + + + + + + + + diff --git a/equinox-projector/Projector.Tests/StreamStateTests.fs b/equinox-projector/Projector.Tests/StreamStateTests.fs new file mode 100644 index 000000000..8d35d267f --- /dev/null +++ b/equinox-projector/Projector.Tests/StreamStateTests.fs @@ -0,0 +1,78 @@ +module CosmosIngesterTests + +open ProjectorTemplate.Projector.State +open Swensen.Unquote +open Xunit + +let canonicalTime = System.DateTimeOffset.UtcNow +let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } +let mergeSpans = StreamState.Span.merge +let trimSpans = StreamState.Span.trim + +let [] ``nothing`` () = + let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] + r =! null + +let [] ``synced`` () = + let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] + r =! null + +let [] ``no overlap`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] + r =! [| mk 0L 1; mk 2L 2 |] + +let [] ``overlap`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] + r =! [| mk 0L 2 |] + +let [] ``remove nulls`` () = + let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] + r =! [| mk 1L 1 |] + +let [] ``adjacent`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] + r =! [| mk 0L 3 |] + +let [] ``adjacent to min`` () = + let r = List.map (trimSpans 2L) [ mk 0L 1; mk 1L 2 ] + r =! [ mk 2L 0; mk 2L 1 ] + +let [] ``adjacent to min merge`` () = + let r = mergeSpans 2L [ mk 0L 1; mk 1L 2 ] + r =! [| mk 2L 1 |] + +let [] ``adjacent to min no overlap`` () = + let r = mergeSpans 2L [ mk 0L 1; mk 2L 1 ] + r =! [| mk 2L 1|] + +let [] ``adjacent trim`` () = + let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2 ] + r =! [ mk 1L 1; mk 2L 2 ] + +let [] ``adjacent trim merge`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] + r =! [| mk 1L 3 |] + +let [] ``adjacent trim append`` () = + let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2; mk 5L 1] + r =! [ mk 1L 1; mk 2L 2; mk 5L 1 ] + +let [] ``adjacent trim append merge`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] + r =! [| mk 1L 3; mk 5L 1 |] + +let [] ``mixed adjacent trim append`` () = + let r = List.map (trimSpans 1L) [ mk 0L 2; mk 5L 1; mk 2L 2] + r =! [ mk 1L 1; mk 5L 1; mk 2L 2 ] + +let [] ``mixed adjacent trim append merge`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2] + r =! [| mk 1L 3; mk 5L 1 |] + +let [] ``fail`` () = + let r = mergeSpans 11614L [ {index=11614L; events=null}; mk 11614L 1 ] + r =! [| mk 11614L 1 |] + +let [] ``fail 2`` () = + let r = mergeSpans 11613L [ mk 11614L 1; {index=11614L; events=null} ] + r =! [| mk 11614L 1 |] diff --git a/equinox-projector/Projector/Infrastructure.fs b/equinox-projector/Projector/Infrastructure.fs index 1fcebdd18..970315bf7 100644 --- a/equinox-projector/Projector/Infrastructure.fs +++ b/equinox-projector/Projector/Infrastructure.fs @@ -16,4 +16,32 @@ type Async with let isDisposed = ref 0 let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) \ No newline at end of file + in ()) + static member AwaitTaskCorrect (task : Task<'T>) : Async<'T> = + Async.FromContinuations <| fun (k,ek,_) -> + task.ContinueWith (fun (t:Task<'T>) -> + if t.IsFaulted then + let e = t.Exception + if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] + else ek e + elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) + elif t.IsCompleted then k t.Result + else ek(Exception "invalid Task state!")) + |> ignore + + +type SemaphoreSlim with + /// F# friendly semaphore await function + member semaphore.Await(?timeout : TimeSpan) = async { + let! ct = Async.CancellationToken + let timeout = defaultArg timeout Timeout.InfiniteTimeSpan + let task = semaphore.WaitAsync(timeout, ct) + return! Async.AwaitTaskCorrect task + } + + /// Throttling wrapper which waits asynchronously until the semaphore has available capacity + member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { + let! _ = semaphore.Await() + try return! workflow + finally semaphore.Release() |> ignore + } \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index ededdcfd1..d1df7367e 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -8,6 +8,8 @@ open Equinox.Cosmos.Projection //#if kafka open Equinox.Projection.Codec open Jet.ConfluentKafka.FSharp +//#else +open ProjectorTemplate.Projector.State //#endif open Equinox.Store open Microsoft.Azure.Documents.ChangeFeedProcessor @@ -72,8 +74,10 @@ module CmdParser = (* ChangeFeed Args*) | [] ConsumerGroupName of string | [] LeaseCollectionSuffix of string - | [] ForceStartFromHere + | [] ForceStartFromHere | [] BatchSize of int + | [] MaxPendingBatches of int + | [] ProcessorDop of int | [] LagFreqS of float | [] Verbose | [] ChangeFeedVerbose @@ -91,6 +95,8 @@ module CmdParser = | LeaseCollectionSuffix _ -> "specify Collection Name suffix for Leases collection (default: `-aux`)." | ForceStartFromHere _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | BatchSize _ -> "maximum item count to request from feed. Default: 1000" + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" + | ProcessorDop _ -> "Maximum number of streams to process concurrently. Default: 64" | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | Verbose -> "request Verbose Logging. Default: off" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" @@ -109,6 +115,8 @@ module CmdParser = member __.Verbose = args.Contains Verbose member __.ChangeFeedVerbose = args.Contains ChangeFeedVerbose member __.BatchSize = args.GetResult(BatchSize,1000) + member __.MaxPendingBatches = args.GetResult(MaxPendingBatches,64) + member __.ProcessorDop = args.GetResult(ProcessorDop,64) member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.AuxCollectionName = __.Cosmos.Collection + __.Suffix member __.StartFromHere = args.Contains ForceStartFromHere @@ -116,7 +124,7 @@ module CmdParser = Log.Information("Processing {leaseId} in {auxCollName} in batches of {batchSize}", x.LeaseId, x.AuxCollectionName, x.BatchSize) if x.StartFromHere then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) - { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, x.StartFromHere, x.BatchSize, x.LagFrequency + { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, x.StartFromHere, x.BatchSize, x.MaxPendingBatches, x.ProcessorDop, x.LagFrequency //#if kafka and TargetInfo(args : ParseResults) = member __.Broker = Uri(match args.TryGetResult Broker with Some x -> x | None -> envBackstop "Broker" "EQUINOX_KAFKA_BROKER") @@ -130,21 +138,21 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -let run discovery connectionPolicy source +let run (log : ILogger) discovery connectionPolicy source (aux, leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - Log.Information("Lags by Range {@rangeLags}", remainingWork) + log.Information("Lags by Range {@rangeLags}", remainingWork) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = ChangeFeedProcessor.Start - ( Log.Logger, discovery, connectionPolicy, source, aux, leasePrefix = leaseId, forceSkipExistingEvents = forceSkip, + ( log, discovery, connectionPolicy, source, aux, leasePrefix = leaseId, forceSkipExistingEvents = forceSkip, cfBatchSize = batchSize, createObserver = createRangeProjector, ?reportLagAndAwaitNextEstimation = maybeLogLag) do! Async.AwaitKeyboardInterrupt() } //#if kafka -let mkRangeProjector (broker, topic) = +let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broker, topic) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = CompressionType.Lz4) let producer = KafkaProducer.Create(Log.Logger, cfg, topic) @@ -163,20 +171,33 @@ let mkRangeProjector (broker, topic) = float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - ChangeFeedObserver.Create(Log.Logger, projectBatch, dispose = disposeProducer) + ChangeFeedObserver.Create(log, projectBatch, dispose = disposeProducer) //#else -let createRangeHandler () = +let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, busyPause : TimeSpan, project) () = + let mutable coordinator = Unchecked.defaultof let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let pt,events = (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> Seq.length) |> Stopwatch.Time - let! ct,() = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } |> Stopwatch.Time - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Parse {events,5} events {p:n3}s Checkpoint {c:n3}s", - ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), - float sw.ElapsedMilliseconds / 1000., events, (let e = pt.Elapsed in e.TotalSeconds), (let e = ct.Elapsed in e.TotalSeconds)) + // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved + let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 + let ingest docs : (*events*)int * (*streams*)int = + let events = docs |> Seq.collect DocumentParser.enumEvents |> Array.ofSeq + coordinator.Submit(epoch,checkpoint,[| for x in events -> { stream = x.Stream; span = { index = x.Index; events = [| x |] } }|]) + events.Length, HashSet(seq { for x in events -> x.Stream }).Count + let pt, (events,streams) = Stopwatch.Time (fun () -> ingest docs) + log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Ingested {streams,5}s {events,5}e s {p:n3}ms", + epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), + float sw.ElapsedMilliseconds / 1000., streams, events, (let e = pt.Elapsed in e.TotalMilliseconds)) + let mutable first = true + while coordinator.IsFullyLoaded do // Only hand back control to the CFP iff backlog is under control + if first then first <- false; log.Information("Pausing due to backlog of incomplete batches...") + do! Async.Sleep busyPause sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - ChangeFeedObserver.Create(Log.Logger, processBatch) + let init rangeLog = coordinator <- Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project) + let dispose () = coordinator.Stop() + ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) //#endif // Illustrates how to emit direct to the Console using Serilog @@ -186,26 +207,27 @@ module Logging = Log.Logger <- LoggerConfiguration().Destructure.FSharpTypes().Enrich.FromLogContext() |> fun c -> if verbose then c.MinimumLevel.Debug() else c - // LibLog writes to the global logger, so we need to control the emission if we don't want to pass loggers everywhere + // LibLog writes to the global logger, so we need to control the emission |> fun c -> let cfpl = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpl) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Message:lj} {NewLine}{Exception}" c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) - .CreateLogger() + |> fun c -> c.CreateLogger() [] let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ChangeFeedVerbose let discovery, connector, source = args.Cosmos.BuildConnectionDetails() - let aux, leaseId, startFromHere, batchSize, lagFrequency = args.BuildChangeFeedParams() + let aux, leaseId, startFromHere, batchSize, maxPendingBatches, processorDop, lagFrequency = args.BuildChangeFeedParams() //#if kafka let targetParams = args.Target.BuildTargetParams() - let createRangeHandler () = mkRangeProjector targetParams + let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams //#endif - run discovery connector.ConnectionPolicy source + let project stream span = async { do () } + run Log.Logger discovery connector.ConnectionPolicy source (aux, leaseId, startFromHere, batchSize, lagFrequency) - createRangeHandler + (createRangeHandler Log.Logger (maxPendingBatches, processorDop, TimeSpan.FromMilliseconds 100., project)) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 4a74114c9..abdf59346 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -7,8 +7,8 @@ - + diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs new file mode 100644 index 000000000..a1f693e0a --- /dev/null +++ b/equinox-projector/Projector/State.fs @@ -0,0 +1,439 @@ +module ProjectorTemplate.Projector.State + +open Serilog +open System +open System.Collections.Concurrent +open System.Collections.Generic +open System.Diagnostics +open System.Threading + +module Metrics = + module RuCounters = + open Equinox.Cosmos.Store + open Serilog.Events + + let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds + + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function + | Log.Tip (Stats s) + | Log.TipNotFound (Stats s) + | Log.TipNotModified (Stats s) + | Log.Query (_,_, (Stats s)) -> CosmosReadRc s + // slices are rolled up into batches so be sure not to double-count + | Log.Response (_,(Stats s)) -> CosmosResponseRc s + | Log.SyncSuccess (Stats s) + | Log.SyncConflict (Stats s) -> CosmosWriteRc s + | Log.SyncResync (Stats s) -> CosmosResyncRc s + let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function + | (:? ScalarValue as x) -> Some x.Value + | _ -> None + let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = + match logEvent.Properties.TryGetValue("cosmosEvt") with + | true, SerilogScalar (:? Log.Event as e) -> Some e + | _ -> None + type RuCounter = + { mutable rux100: int64; mutable count: int64; mutable ms: int64 } + static member Create() = { rux100 = 0L; count = 0L; ms = 0L } + member __.Ingest (ru, ms) = + System.Threading.Interlocked.Increment(&__.count) |> ignore + System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore + System.Threading.Interlocked.Add(&__.ms, ms) |> ignore + type RuCounterSink() = + static member val Read = RuCounter.Create() with get, set + static member val Write = RuCounter.Create() with get, set + static member val Resync = RuCounter.Create() with get, set + static member Reset() = + RuCounterSink.Read <- RuCounter.Create() + RuCounterSink.Write <- RuCounter.Create() + RuCounterSink.Resync <- RuCounter.Create() + interface Serilog.Core.ILogEventSink with + member __.Emit logEvent = logEvent |> function + | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats + | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats + | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats + | _ -> () + + let dumpRuStats duration (log: Serilog.ILogger) = + let stats = + [ "Read", RuCounters.RuCounterSink.Read + "Write", RuCounters.RuCounterSink.Write + "Resync", RuCounters.RuCounterSink.Resync ] + let mutable totalCount, totalRc, totalMs = 0L, 0., 0L + let logActivity name count rc lat = + if count <> 0L then + log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", + name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) + for name, stat in stats do + let ru = float stat.rux100 / 100. + totalCount <- totalCount + stat.count + totalRc <- totalRc + ru + totalMs <- totalMs + stat.ms + logActivity name stat.count ru stat.ms + logActivity "TOTAL" totalCount totalRc totalMs + // Yes, there's a minor race here! + RuCounters.RuCounterSink.Reset() + let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] + let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) + +module ConcurrentQueue = + let drain handle (xs : ConcurrentQueue<_>) = + let rec aux () = + match xs.TryDequeue() with + | true, x -> handle x; aux () + | false, _ -> () + aux () + +let every ms f = + let timer = Stopwatch.StartNew() + fun () -> + if timer.ElapsedMilliseconds > ms then + f () + timer.Restart() +let expired ms = + let timer = Stopwatch.StartNew() + fun () -> + let due = timer.ElapsedMilliseconds > ms + if due then timer.Restart() + due + +let arrayBytes (x:byte[]) = if x = null then 0 else x.Length +let private mb x = float x / 1024. / 1024. +let category (streamName : string) = streamName.Split([|'-'|],2).[0] + +type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } +type [] StreamBatch = { stream: string; span: Span } +type [] StreamState = { write: int64 option; queue: Span[] } with + member __.Size = + if __.queue = null then 0 + else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) + +module StreamState = + let (|NNA|) xs = if xs = null then Array.empty else xs + module Span = + let (|End|) x = x.index + if x.events = null then 0L else x.events.LongLength + let trim min = function + | x when x.index >= min -> x // don't adjust if min not within + | End n when n < min -> { index = min; events = [||] } // throw away if before min + | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice + let merge min (xs : Span seq) = + let xs = + seq { for x in xs -> { x with events = (|NNA|) x.events } } + |> Seq.map (trim min) + |> Seq.filter (fun x -> x.events.Length <> 0) + |> Seq.sortBy (fun x -> x.index) + let buffer = ResizeArray() + let mutable curr = None + for x in xs do + match curr, x with + // Not overlapping, no data buffered -> buffer + | None, _ -> + curr <- Some x + // Gap + | Some (End nextIndex as c), x when x.index > nextIndex -> + buffer.Add c + curr <- Some x + // Overlapping, join + | Some (End nextIndex as c), x -> + curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } + curr |> Option.iter buffer.Add + if buffer.Count = 0 then null else buffer.ToArray() + + let inline optionCombine f (r1: int64 option) (r2: int64 option) = + match r1, r2 with + | Some x, Some y -> f x y |> Some + | None, None -> None + | None, x | x, None -> x + let combine (s1: StreamState) (s2: StreamState) : StreamState = + let writePos = optionCombine max s1.write s2.write + let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 + { write = writePos; queue = Span.merge (defaultArg writePos 0L) items } + +/// Gathers stats relating to how many items of a given category have been observed +type CatStats() = + let cats = Dictionary() + member __.Ingest(cat,?weight) = + let weight = defaultArg weight 1L + match cats.TryGetValue cat with + | true, catCount -> cats.[cat] <- catCount + weight + | false, _ -> cats.[cat] <- weight + member __.Any = cats.Count <> 0 + member __.Clear() = cats.Clear() + member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd + +type StreamStates() = + let states = Dictionary() + let update stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + stream, state + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + stream, updated + let updateWritePos stream pos span = update stream { write = pos; queue = span } + let markCompleted stream index = updateWritePos stream (Some index) null |> ignore + let enqueue item = updateWritePos item.stream None [|item.span|] + + let busy = HashSet() + let schedule (requestedOrder : string seq) (capacity: int) = + let toSchedule = ResizeArray<_>(capacity) + let xs = requestedOrder.GetEnumerator() + while xs.MoveNext() && toSchedule.Capacity <> 0 do + let x = xs.Current + if busy.Add x then + let q = states.[x].queue + if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", x) + toSchedule.Add(x,q.[0]) + toSchedule.ToArray() + let markNotBusy stream = + busy.Remove stream |> ignore + + member __.Add(item: StreamBatch) = enqueue item + member __.TryGetStreamWritePos stream = + match states.TryGetValue stream with + | true, value -> value.write + | false, _ -> None + member __.QueueLength(stream) = + let q = states.[stream].queue + if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", stream) + q.[0].events.Length + member __.MarkCompleted(stream, index) = + markNotBusy stream + markCompleted stream index + member __.MarkFailed stream = + markNotBusy stream + member __.Schedule(requestedOrder : string seq, capacity: int) : (string*Span)[] = + schedule requestedOrder capacity + member __.Dump(log : ILogger) = + let mutable busyCount, busyB, ready, readyB, synced = 0, 0L, 0, 0L, 0 + let busyCats, readyCats, readyStreams = CatStats(), CatStats(), CatStats() + for KeyValue (stream,state) in states do + match int64 state.Size with + | 0L -> + synced <- synced + 1 + | sz when busy.Contains stream -> + busyCats.Ingest(category stream) + busyCount <- busyCount + 1 + busyB <- busyB + sz + | sz -> + readyCats.Ingest(category stream) + readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) + ready <- ready + 1 + readyB <- readyB + sz + log.Information("Busy {busy}/{busyMb:n1}MB Ready {ready}/{readyMb:n1}MB Synced {synced}", busyCount, mb busyB, ready, mb readyB, synced) + if busyCats.Any then log.Information("Busy Categories, events {busyCats}", busyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + +type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } + +type ProgressState<'Pos>(?currentPos : 'Pos) = + let pending = Queue<_>() + let mutable validatedPos = currentPos + member __.AppendBatch(pos, streamWithRequiredIndices : (string * int64) seq) = + let byStream = streamWithRequiredIndices |> Seq.groupBy fst |> Seq.map (fun (s,xs) -> KeyValuePair(s,xs |> Seq.map snd |> Seq.max)) + pending.Enqueue { pos = pos; streamToRequiredIndex = Dictionary byStream } + member __.MarkStreamProgress(stream, index) = + for x in pending do + match x.streamToRequiredIndex.TryGetValue stream with + | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore + | _, _ -> () + let headIsComplete () = + match pending.TryPeek() with + | true, batch -> batch.streamToRequiredIndex.Count = 0 + | _ -> false + while headIsComplete () do + let headBatch = pending.Dequeue() + validatedPos <- Some headBatch.pos + member __.ScheduledOrder getStreamQueueLength = + let raw = seq { + let mutable batch = 0 + let streams = HashSet() + for x in pending do + batch <- batch + 1 + for s in x.streamToRequiredIndex.Keys do + if streams.Add s then + yield s,struct (batch,getStreamQueueLength s) + } + raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst + member __.Validate tryGetStreamWritePos : 'Pos option * int = + let rec aux () = + match pending.TryPeek() with + | false, _ -> () + | true, batch -> + for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do + match tryGetStreamWritePos stream with + | Some index when requiredIndex <= index -> + Log.Warning("Validation had to remove {stream}", stream) + batch.streamToRequiredIndex.Remove stream |> ignore + | _ -> () + if batch.streamToRequiredIndex.Count = 0 then + let headBatch = pending.Dequeue() + validatedPos <- Some headBatch.pos + aux () + aux () + validatedPos, pending.Count + +/// Manages writing of progress +/// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) +/// - retries until success or a new item is posted +type ProgressWriter() = + let pumpSleepMs = 100 + let mutable committedEpoch = None + let mutable validatedPos = None + let result = Event<_>() + [] member __.Result = result.Publish + member __.Post(version,f) = + Volatile.Write(&validatedPos,Some (version,f)) + member __.CommittedEpoch = Volatile.Read(&committedEpoch) + member __.Pump() = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + match Volatile.Read &validatedPos with + | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v -> + try do! f + Volatile.Write(&committedEpoch, Some v) + result.Trigger (Choice1Of2 v) + with e -> result.Trigger (Choice2Of2 e) + | _ -> do! Async.Sleep pumpSleepMs } + +[] +type CoordinatorWork = + /// Enqueue a batch of items with supplied tag and progress marking function + | Add of epoch: int64 * markCompleted: Async * items: StreamBatch[] + /// Result of processing on stream - processed up to nominated `index` or threw `exn` + | Result of stream: string * outcome: Choice + /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` + | ProgressResult of Choice + +type CoordinatorStats(log : ILogger, maxPendingBatches, ?statsInterval) = + let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.) + let statsIntervalMs = int64 statsInterval.TotalMilliseconds + let mutable pendingBatchCount = 0 + let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None + let workPended, eventsPended, cycles = ref 0, ref 0, ref 0 + let progCommitFails, progCommits = ref 0, ref 0 + let resultOk, resultExn = ref 0, ref 0 + let statsDue = expired statsIntervalMs + let dumpStats (streams : StreamStates) = + if !progCommitFails <> 0 || !progCommits <> 0 then + match comittedEpoch with + | None -> + log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) + | Some committed when !progCommitFails <> 0 -> + log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) + | Some committed -> + log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) + progCommits := 0; progCommitFails := 0 + else + log.Information("Progress @ {validated} (committed: {committed}) Uncomitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) + let results = !resultOk + !resultExn + Log.Information("Cycles {cycles} Queued {queued} reqs {events} events", + !cycles, !workPended, !eventsPended) + cycles := 0; workPended := 0; eventsPended := 0 + Log.Information("Completed {completed} ({ok} ok {exns} Exns)", + results, !resultOk, !resultExn) + resultOk := 0; resultExn := 0 + Metrics.dumpRuStats statsInterval log + streams.Dump log + member __.Handle = function + | ProgressResult (Choice1Of2 epoch) -> + incr progCommits + comittedEpoch <- Some epoch + | ProgressResult (Choice2Of2 (_exn : exn)) -> + incr progCommitFails + | Add (_epoch, _markCompleted,items) -> + incr workPended + eventsPended := !eventsPended + (items |> Array.sumBy (fun x -> x.span.events.Length)) + | Result _ -> () + member __.HandleValidated(epoch, pendingBatches) = + incr cycles + pendingBatchCount <- pendingBatches + validatedEpoch <- epoch + member __.HandleCommitted epoch = + comittedEpoch <- epoch + member __.TryDump(streams) = if statsDue () then dumpStats streams + +/// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint +type Dispatcher(maxDop) = + let cancellationCheckInterval = TimeSpan.FromMilliseconds 5. + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let result = Event<_>() + let dop = new SemaphoreSlim(maxDop) + let dispatch work = async { let! res = work in result.Trigger res } |> dop.Throttle + let capacity = new SemaphoreSlim(maxDop) + [] member __.Result = result.Publish + member __.Capacity = capacity.CurrentCount + member __.Enqueue item = work.Add item + member __.Pump () = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + let mutable item = Unchecked.defaultof> + if work.TryTake(&item, cancellationCheckInterval) then + let! _ = Async.StartChild(dispatch item) in () + } + +/// Single instance per ChangeFeedObserver, spun up as leases are won and allocated by the ChangeFeedProcessor hosting framework +/// Coordinates a) ingestion of events b) execution of projection work c) writing of progress d) reporting of state +type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) = + let sleepIntervalMs = 5 + let cts = new CancellationTokenSource() + let stats = CoordinatorStats(log, maxPendingBatches, ?statsInterval=statsInterval) + let mutable pendingBatches = 0 + let progressWriter = ProgressWriter() + let progressState = ProgressState() + let streams = StreamStates() + let work = ConcurrentQueue<_>() + member private __.Pump(project : string -> Span -> Async>) = async { + let dispatcher = Dispatcher(processorDop) + use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) + use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) + Async.Start(progressWriter.Pump(), cts.Token) + Async.Start(dispatcher.Pump(), cts.Token) + let handle = function + | ProgressResult (Choice1Of2 epoch) -> () + | ProgressResult (Choice2Of2 (_exn : exn)) -> () + | Add (epoch, checkpoint,items) -> + for item in items do + streams.Add item |> ignore + progressState.AppendBatch((epoch,checkpoint), [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) + | Result (stream, Choice1Of2 index) -> + streams.MarkCompleted(stream,index) + | Result (stream, Choice2Of2 _) -> + streams.MarkFailed stream + while not cts.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + work |> ConcurrentQueue.drain handle + // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) + let validatedPos, batches = progressState.Validate(streams.TryGetStreamWritePos) + stats.HandleValidated(Option.map fst validatedPos, batches) + validatedPos |> Option.iter progressWriter.Post + stats.HandleCommitted progressWriter.CommittedEpoch + pendingBatches <- batches + // 3. After that, [over] provision writers queue + let capacity = dispatcher.Capacity + if capacity <> 0 then + let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) + for stream,span in work do + dispatcher.Enqueue <| async { + let! res = project stream span + return stream,res } + do! Async.Sleep sleepIntervalMs + // 4. Periodically emit status info + stats.TryDump streams } + static member Start(rangeLog, maxPendingBatches, processorDop, project) = + let instance = new Coordinator(rangeLog, maxPendingBatches, processorDop) + Async.Start <| instance.Pump(project) + instance + member __.Submit(epoch, markBatchCompleted, events) = + Add (epoch, markBatchCompleted, events) |> work.Enqueue + member __.IsFullyLoaded = + Volatile.Read(&pendingBatches) >= maxPendingBatches + member __.Stop() = + cts.Cancel() \ No newline at end of file diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index aa427035b..5e7967ea3 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -1,7 +1,7 @@  Microsoft Visual Studio Solution File, Format Version 12.00 -# Visual Studio 15 -VisualStudioVersion = 15.0.26124.0 +# Visual Studio Version 16 +VisualStudioVersion = 16.0.28729.10 MinimumVisualStudioVersion = 15.0.26124.0 Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Projector", "Projector\Projector.fsproj", "{6C72C937-ECFC-4DD4-9BA0-7355B237F974}" EndProject @@ -12,6 +12,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consumer.fsproj", "{7ED94D2B-1744-48A0-9B20-94E4777617E9}" EndProject +Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Projector.Tests", "Projector.Tests\Projector.Tests.fsproj", "{964E1EA5-9A40-422D-9673-DE169E6D49EE}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -26,6 +28,10 @@ Global {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.Build.0 = Debug|Any CPU {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.Build.0 = Release|Any CPU + {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index b4b9a66f9..3cf041f99 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -75,7 +75,7 @@ type [] StreamState = { isMalformed : bool; write: int64 option; q | _ -> false member __.Size = if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length*2 + 16) + else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) module StreamState = let (|NNA|) xs = if xs = null then Array.empty else xs From d3298a9b2d6f2da7eb922cdc4e463a28e7a1dee7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 17:16:50 +0100 Subject: [PATCH 120/353] Fix bugs --- equinox-projector/Projector/Program.fs | 27 +++++---- equinox-projector/Projector/State.fs | 81 ++++++++++++++------------ 2 files changed, 59 insertions(+), 49 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index d1df7367e..88093958d 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -94,7 +94,7 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LeaseCollectionSuffix _ -> "specify Collection Name suffix for Leases collection (default: `-aux`)." | ForceStartFromHere _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." - | BatchSize _ -> "maximum item count to request from feed. Default: 1000" + | BatchSize _ -> "maximum item count to request from feed. Default: 1024" | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" | ProcessorDop _ -> "Maximum number of streams to process concurrently. Default: 64" | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" @@ -114,14 +114,15 @@ module CmdParser = member __.Suffix = args.GetResult(LeaseCollectionSuffix,"-aux") member __.Verbose = args.Contains Verbose member __.ChangeFeedVerbose = args.Contains ChangeFeedVerbose - member __.BatchSize = args.GetResult(BatchSize,1000) + member __.BatchSize = args.GetResult(BatchSize,1024) member __.MaxPendingBatches = args.GetResult(MaxPendingBatches,64) member __.ProcessorDop = args.GetResult(ProcessorDop,64) member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.AuxCollectionName = __.Cosmos.Collection + __.Suffix member __.StartFromHere = args.Contains ForceStartFromHere member x.BuildChangeFeedParams() = - Log.Information("Processing {leaseId} in {auxCollName} in batches of {batchSize}", x.LeaseId, x.AuxCollectionName, x.BatchSize) + Log.Information("Processing {leaseId} in {auxCollName} in batches of {batchSize} (<= {maxPending} pending) using {dop} processors", + x.LeaseId, x.AuxCollectionName, x.BatchSize, x.MaxPendingBatches, x.ProcessorDop) if x.StartFromHere then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, x.StartFromHere, x.BatchSize, x.MaxPendingBatches, x.ProcessorDop, x.LagFrequency @@ -166,8 +167,8 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broke let! _ = producer.ProduceBatch es do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } |> Stopwatch.Time - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", - ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), + log.Information("Read {token,8} {count,4} docs {requestCharge,6}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", + ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n0")), float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } @@ -186,9 +187,9 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, busyPause coordinator.Submit(epoch,checkpoint,[| for x in events -> { stream = x.Stream; span = { index = x.Index; events = [| x |] } }|]) events.Length, HashSet(seq { for x in events -> x.Stream }).Count let pt, (events,streams) = Stopwatch.Time (fun () -> ingest docs) - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Ingested {streams,5}s {events,5}e s {p:n3}ms", - epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), - float sw.ElapsedMilliseconds / 1000., streams, events, (let e = pt.Elapsed in e.TotalMilliseconds)) + log.Information("Read {token,8} {count,4} docs {requestCharge,6}RU {l:n1}s Ingested {streams,5}s {events,5}e {p,2}ms", + epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n0")), + float sw.ElapsedMilliseconds / 1000., streams, events, (let e = pt.Elapsed in int e.TotalMilliseconds)) let mutable first = true while coordinator.IsFullyLoaded do // Only hand back control to the CFP iff backlog is under control if first then first <- false; log.Information("Pausing due to backlog of incomplete batches...") @@ -221,10 +222,14 @@ let main argv = let discovery, connector, source = args.Cosmos.BuildConnectionDetails() let aux, leaseId, startFromHere, batchSize, maxPendingBatches, processorDop, lagFrequency = args.BuildChangeFeedParams() //#if kafka - let targetParams = args.Target.BuildTargetParams() - let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams + //let targetParams = args.Target.BuildTargetParams() + //let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams //#endif - let project stream span = async { do () } + let project (batch : StreamBatch) = async { + let r = Random() + let ms = r.Next(1,batch.span.events.Length) + do! Async.Sleep ms + return batch.span.events.Length } run Log.Logger discovery connector.ConnectionPolicy source (aux, leaseId, startFromHere, batchSize, lagFrequency) (createRangeHandler Log.Logger (maxPendingBatches, processorDop, TimeSpan.FromMilliseconds 100., project)) diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index a1f693e0a..9694e0d7e 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -90,7 +90,7 @@ let every ms f = if timer.ElapsedMilliseconds > ms then f () timer.Restart() -let expired ms = +let expiredMs ms = let timer = Stopwatch.StartNew() fun () -> let due = timer.ElapsedMilliseconds > ms @@ -185,7 +185,7 @@ type StreamStates() = if busy.Add x then let q = states.[x].queue if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", x) - toSchedule.Add(x,q.[0]) + toSchedule.Add { stream = x; span = q.[0] } toSchedule.ToArray() let markNotBusy stream = busy.Remove stream |> ignore @@ -204,7 +204,7 @@ type StreamStates() = markCompleted stream index member __.MarkFailed stream = markNotBusy stream - member __.Schedule(requestedOrder : string seq, capacity: int) : (string*Span)[] = + member __.Schedule(requestedOrder : string seq, capacity: int) : StreamBatch[] = schedule requestedOrder capacity member __.Dump(log : ILogger) = let mutable busyCount, busyB, ready, readyB, synced = 0, 0L, 0, 0L, 0 @@ -255,8 +255,7 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = batch <- batch + 1 for s in x.streamToRequiredIndex.Keys do if streams.Add s then - yield s,struct (batch,getStreamQueueLength s) - } + yield s,struct (batch,getStreamQueueLength s) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst member __.Validate tryGetStreamWritePos : 'Pos option * int = let rec aux () = @@ -281,6 +280,7 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = /// - retries until success or a new item is posted type ProgressWriter() = let pumpSleepMs = 100 + let due = expiredMs 5000L let mutable committedEpoch = None let mutable validatedPos = None let result = Event<_>() @@ -292,7 +292,7 @@ type ProgressWriter() = let! ct = Async.CancellationToken while not ct.IsCancellationRequested do match Volatile.Read &validatedPos with - | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v -> + | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> try do! f Volatile.Write(&committedEpoch, Some v) result.Trigger (Choice1Of2 v) @@ -311,36 +311,31 @@ type CoordinatorWork = type CoordinatorStats(log : ILogger, maxPendingBatches, ?statsInterval) = let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.) let statsIntervalMs = int64 statsInterval.TotalMilliseconds - let mutable pendingBatchCount = 0 - let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None - let workPended, eventsPended, cycles = ref 0, ref 0, ref 0 - let progCommitFails, progCommits = ref 0, ref 0 - let resultOk, resultExn = ref 0, ref 0 - let statsDue = expired statsIntervalMs - let dumpStats (streams : StreamStates) = + let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None + let progCommitFails, progCommits = ref 0, ref 0 + let cycles, workPended, eventsPended, resultOk, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs statsIntervalMs + let dumpStats (busy,capacity) (streams : StreamStates) = if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> - log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncomitted {pendingBatches}/{maxPendingBatches}", + log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) | Some committed when !progCommitFails <> 0 -> - log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncomitted {pendingBatches}/{maxPendingBatches}", + log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) | Some committed -> - log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncomitted {pendingBatches}/{maxPendingBatches}", + log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) progCommits := 0; progCommitFails := 0 else - log.Information("Progress @ {validated} (committed: {committed}) Uncomitted {pendingBatches}/{maxPendingBatches}", + log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) let results = !resultOk + !resultExn - Log.Information("Cycles {cycles} Queued {queued} reqs {events} events", - !cycles, !workPended, !eventsPended) - cycles := 0; workPended := 0; eventsPended := 0 - Log.Information("Completed {completed} ({ok} ok {exns} Exns)", - results, !resultOk, !resultExn) - resultOk := 0; resultExn := 0 - Metrics.dumpRuStats statsInterval log + log.Information("Cycles {cycles} Ingested {batches} ({events} events) Active {busy}/{processors} Completed {completed} ({ok} ok {exns} exn)", + !cycles, !workPended, !eventsPended, busy, capacity, results, !resultOk, !resultExn) + cycles := 0; workPended := 0; eventsPended := 0; resultOk := 0; resultExn:= 0 + //Metrics.dumpRuStats statsInterval log streams.Dump log member __.Handle = function | ProgressResult (Choice1Of2 epoch) -> @@ -351,14 +346,17 @@ type CoordinatorStats(log : ILogger, maxPendingBatches, ?statsInterval) = | Add (_epoch, _markCompleted,items) -> incr workPended eventsPended := !eventsPended + (items |> Array.sumBy (fun x -> x.span.events.Length)) - | Result _ -> () + | Result (_stream, Choice1Of2 _) -> + incr resultOk + | Result (_stream, Choice2Of2 _) -> + incr resultExn member __.HandleValidated(epoch, pendingBatches) = incr cycles pendingBatchCount <- pendingBatches validatedEpoch <- epoch member __.HandleCommitted epoch = comittedEpoch <- epoch - member __.TryDump(streams) = if statsDue () then dumpStats streams + member __.TryDump(busy,capacity,streams) = if statsDue () then dumpStats (busy,capacity) streams /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher(maxDop) = @@ -366,17 +364,21 @@ type Dispatcher(maxDop) = let work = new BlockingCollection<_>(ConcurrentQueue<_>()) let result = Event<_>() let dop = new SemaphoreSlim(maxDop) - let dispatch work = async { let! res = work in result.Trigger res } |> dop.Throttle - let capacity = new SemaphoreSlim(maxDop) + let dispatch work = async { + let! res = work + result.Trigger res + dop.Release() |> ignore } [] member __.Result = result.Publish - member __.Capacity = capacity.CurrentCount + member __.Capacity = dop.CurrentCount member __.Enqueue item = work.Add item member __.Pump () = async { let! ct = Async.CancellationToken while not ct.IsCancellationRequested do - let mutable item = Unchecked.defaultof> - if work.TryTake(&item, cancellationCheckInterval) then - let! _ = Async.StartChild(dispatch item) in () + let! got = dop.Await(cancellationCheckInterval) + if got then + let mutable item = Unchecked.defaultof> + if work.TryTake(&item, cancellationCheckInterval) then let! _ = Async.StartChild(dispatch item) in () + else dop.Release() |> ignore } /// Single instance per ChangeFeedObserver, spun up as leases are won and allocated by the ChangeFeedProcessor hosting framework @@ -390,7 +392,7 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) let progressState = ProgressState() let streams = StreamStates() let work = ConcurrentQueue<_>() - member private __.Pump(project : string -> Span -> Async>) = async { + member private __.Pump(project : StreamBatch -> Async) = async { let dispatcher = Dispatcher(processorDop) use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) @@ -404,12 +406,13 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) streams.Add item |> ignore progressState.AppendBatch((epoch,checkpoint), [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) | Result (stream, Choice1Of2 index) -> + progressState.MarkStreamProgress(stream,index) streams.MarkCompleted(stream,index) | Result (stream, Choice2Of2 _) -> streams.MarkFailed stream while not cts.IsCancellationRequested do // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - work |> ConcurrentQueue.drain handle + work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) let validatedPos, batches = progressState.Validate(streams.TryGetStreamWritePos) stats.HandleValidated(Option.map fst validatedPos, batches) @@ -420,13 +423,15 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) let capacity = dispatcher.Capacity if capacity <> 0 then let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) - for stream,span in work do + for batch in work do dispatcher.Enqueue <| async { - let! res = project stream span - return stream,res } + try let! count = project batch + return batch.stream, Choice1Of2 (batch.span.index + int64 count) + with e -> return batch.stream, Choice2Of2 e } do! Async.Sleep sleepIntervalMs // 4. Periodically emit status info - stats.TryDump streams } + let busy = processorDop - dispatcher.Capacity + stats.TryDump(busy,processorDop,streams) } static member Start(rangeLog, maxPendingBatches, processorDop, project) = let instance = new Coordinator(rangeLog, maxPendingBatches, processorDop) Async.Start <| instance.Pump(project) From b7701870b2176ef46b05976b8da2013a9e45b067 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 17:20:38 +0100 Subject: [PATCH 121/353] No struct tuples --- equinox-projector/Projector/State.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 9694e0d7e..5f2ba2bf3 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -255,7 +255,7 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = batch <- batch + 1 for s in x.streamToRequiredIndex.Keys do if streams.Add s then - yield s,struct (batch,getStreamQueueLength s) } + yield s,(batch,getStreamQueueLength s) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst member __.Validate tryGetStreamWritePos : 'Pos option * int = let rec aux () = From aa0d3d12150133a849a59f3991b20a2a573ec9a3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 19:44:07 +0100 Subject: [PATCH 122/353] Tidy ingest effiency --- equinox-projector/Projector/Program.fs | 14 +++++++++----- equinox-projector/Projector/State.fs | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 88093958d..7e405603f 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -182,11 +182,15 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, busyPause // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - let ingest docs : (*events*)int * (*streams*)int = - let events = docs |> Seq.collect DocumentParser.enumEvents |> Array.ofSeq - coordinator.Submit(epoch,checkpoint,[| for x in events -> { stream = x.Stream; span = { index = x.Index; events = [| x |] } }|]) - events.Length, HashSet(seq { for x in events -> x.Stream }).Count - let pt, (events,streams) = Stopwatch.Time (fun () -> ingest docs) + let ingest (inputs : DocumentParser.IEvent seq) : (*events*)int * (*streams*)int = + let streams, events = HashSet(), ResizeArray() + for x in inputs do + streams.Add x.Stream |> ignore + events.Add { stream = x.Stream; span = { index = x.Index; events = Array.singleton (upcast x) } } + let events = events.ToArray() + coordinator.Submit(epoch,checkpoint,events) + events.Length, streams.Count + let pt, (events,streams) = Stopwatch.Time (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> ingest) log.Information("Read {token,8} {count,4} docs {requestCharge,6}RU {l:n1}s Ingested {streams,5}s {events,5}e {p,2}ms", epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n0")), float sw.ElapsedMilliseconds / 1000., streams, events, (let e = pt.Elapsed in int e.TotalMilliseconds)) diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 5f2ba2bf3..8d7ef2d9d 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -249,8 +249,8 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = validatedPos <- Some headBatch.pos member __.ScheduledOrder getStreamQueueLength = let raw = seq { - let mutable batch = 0 let streams = HashSet() + let mutable batch = 0 for x in pending do batch <- batch + 1 for s in x.streamToRequiredIndex.Keys do From 73b0b4222d9d734082c13f9f77facb1b2ee565c5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 23:02:51 +0100 Subject: [PATCH 123/353] Partition alignment --- equinox-projector/Projector/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 7e405603f..03f09f281 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -215,7 +215,7 @@ module Logging = // LibLog writes to the global logger, so we need to control the emission |> fun c -> let cfpl = if changeLogVerbose then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpl) - |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Message:lj} {NewLine}{Exception}" + |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId,2} {Message:lj} {NewLine}{Exception}" c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> c.CreateLogger() From c9bb1eb5c60939da9823600ca91003b14c9f6b6b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 23:06:22 +0100 Subject: [PATCH 124/353] Tidy RC --- equinox-projector/Projector/Program.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 03f09f281..d379b155e 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -167,8 +167,8 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broke let! _ = producer.ProduceBatch es do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } |> Stopwatch.Time - log.Information("Read {token,8} {count,4} docs {requestCharge,6}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", - ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n0")), + log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", + ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } @@ -191,8 +191,8 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, busyPause coordinator.Submit(epoch,checkpoint,events) events.Length, streams.Count let pt, (events,streams) = Stopwatch.Time (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> ingest) - log.Information("Read {token,8} {count,4} docs {requestCharge,6}RU {l:n1}s Ingested {streams,5}s {events,5}e {p,2}ms", - epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n0")), + log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingested {streams,5}s {events,5}e {p,2}ms", + epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., streams, events, (let e = pt.Elapsed in int e.TotalMilliseconds)) let mutable first = true while coordinator.IsFullyLoaded do // Only hand back control to the CFP iff backlog is under control From 5c4c88c7733c9b93c473ed7934da3a72327ef5ad Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 23:31:35 +0100 Subject: [PATCH 125/353] Lag order --- equinox-projector/Projector/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index d379b155e..4ce5ef3d6 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -143,7 +143,7 @@ let run (log : ILogger) discovery connectionPolicy source (aux, leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - log.Information("Lags by Range {@rangeLags}", remainingWork) + log.Information("Lags by Range {@rangeLags}", remainingWork |> Seq.sortByDescending snd) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = From 9624c7f7e69512af5c48d982d7c3818faae58493 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 24 Apr 2019 23:51:03 +0100 Subject: [PATCH 126/353] Fix lag display --- equinox-projector/Projector/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 4ce5ef3d6..011a2af37 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -143,7 +143,7 @@ let run (log : ILogger) discovery connectionPolicy source (aux, leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - log.Information("Lags by Range {@rangeLags}", remainingWork |> Seq.sortByDescending snd) + log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = From 6d4f157dc00a479bf4d2704cd8e7db09ab99b007 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 02:16:07 +0100 Subject: [PATCH 127/353] Batch blocking by semaphore --- .../Projector.Tests/ProgressTests.fs | 13 ++- equinox-projector/Projector/Program.fs | 26 ++--- equinox-projector/Projector/State.fs | 101 ++++++++++-------- 3 files changed, 71 insertions(+), 69 deletions(-) diff --git a/equinox-projector/Projector.Tests/ProgressTests.fs b/equinox-projector/Projector.Tests/ProgressTests.fs index affd0639d..6b5ae8fd7 100644 --- a/equinox-projector/Projector.Tests/ProgressTests.fs +++ b/equinox-projector/Projector.Tests/ProgressTests.fs @@ -4,6 +4,9 @@ open ProjectorTemplate.Projector.State open Swensen.Unquote open Xunit +open System.Collections.Generic + +let mkDictionary xs = Dictionary(dict xs) let [] ``Empty has zero streams pending or progress to write`` () = let sut = ProgressState<_>() @@ -13,15 +16,15 @@ let [] ``Empty has zero streams pending or progress to write`` () = let [] ``Can add multiple batches`` () = let sut = ProgressState<_>() - sut.AppendBatch(0,["a",1L; "b",2L]) - sut.AppendBatch(1,["b",2L; "c",3L]) + sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) + sut.AppendBatch(1,mkDictionary["b",2L; "c",3L]) let validatedPos, batches = sut.Validate(fun _ -> None) None =! validatedPos 2 =! batches let [] ``Marking Progress Removes batches and updates progress`` () = let sut = ProgressState<_>() - sut.AppendBatch(0,["a",1L; "b",2L]) + sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) sut.MarkStreamProgress("a",1L) sut.MarkStreamProgress("b",1L) let validatedPos, batches = sut.Validate(fun _ -> None) @@ -30,9 +33,9 @@ let [] ``Marking Progress Removes batches and updates progress`` () = let [] ``Marking progress is not persistent`` () = let sut = ProgressState<_>() - sut.AppendBatch(0,["a",1L]) + sut.AppendBatch(0, mkDictionary ["a",1L]) sut.MarkStreamProgress("a",2L) - sut.AppendBatch(1,["a",1L; "b",2L]) + sut.AppendBatch(1, mkDictionary ["a",1L; "b",2L]) let validatedPos, batches = sut.Validate(fun _ -> None) Some 0 =! validatedPos 1 =! batches \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 011a2af37..e4116b01d 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -174,30 +174,18 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broke } ChangeFeedObserver.Create(log, projectBatch, dispose = disposeProducer) //#else -let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, busyPause : TimeSpan, project) () = +let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) () = let mutable coordinator = Unchecked.defaultof let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let pt = Stopwatch.StartNew() // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - let ingest (inputs : DocumentParser.IEvent seq) : (*events*)int * (*streams*)int = - let streams, events = HashSet(), ResizeArray() - for x in inputs do - streams.Add x.Stream |> ignore - events.Add { stream = x.Stream; span = { index = x.Index; events = Array.singleton (upcast x) } } - let events = events.ToArray() - coordinator.Submit(epoch,checkpoint,events) - events.Length, streams.Count - let pt, (events,streams) = Stopwatch.Time (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> ingest) - log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingested {streams,5}s {events,5}e {p,2}ms", - epoch, docs.Count, int ctx.FeedResponse.RequestCharge, - float sw.ElapsedMilliseconds / 1000., streams, events, (let e = pt.Elapsed in int e.TotalMilliseconds)) - let mutable first = true - while coordinator.IsFullyLoaded do // Only hand back control to the CFP iff backlog is under control - if first then first <- false; log.Information("Pausing due to backlog of incomplete batches...") - do! Async.Sleep busyPause + do! coordinator.Submit(epoch,checkpoint,seq { for x in Seq.collect DocumentParser.enumEvents docs -> { stream = x.Stream; index = x.Index; event = x } }) + log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingest {p:n1}s", + epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., (let e = pt.Elapsed in int e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } let init rangeLog = coordinator <- Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project) @@ -229,14 +217,14 @@ let main argv = //let targetParams = args.Target.BuildTargetParams() //let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams //#endif - let project (batch : StreamBatch) = async { + let project (batch : StreamBatch) = async { let r = Random() let ms = r.Next(1,batch.span.events.Length) do! Async.Sleep ms return batch.span.events.Length } run Log.Logger discovery connector.ConnectionPolicy source (aux, leaseId, startFromHere, batchSize, lagFrequency) - (createRangeHandler Log.Logger (maxPendingBatches, processorDop, TimeSpan.FromMilliseconds 100., project)) + (createRangeHandler Log.Logger (maxPendingBatches, processorDop, project)) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 8d7ef2d9d..0609510b0 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -101,6 +101,7 @@ let arrayBytes (x:byte[]) = if x = null then 0 else x.Length let private mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] +type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] StreamBatch = { stream: string; span: Span } type [] StreamState = { write: int64 option; queue: Span[] } with @@ -111,7 +112,7 @@ type [] StreamState = { write: int64 option; queue: Span[] } with module StreamState = let (|NNA|) xs = if xs = null then Array.empty else xs module Span = - let (|End|) x = x.index + if x.events = null then 0L else x.events.LongLength + let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength let trim min = function | x when x.index >= min -> x // don't adjust if min not within | End n when n < min -> { index = min; events = [||] } // throw away if before min @@ -174,7 +175,7 @@ type StreamStates() = stream, updated let updateWritePos stream pos span = update stream { write = pos; queue = span } let markCompleted stream index = updateWritePos stream (Some index) null |> ignore - let enqueue item = updateWritePos item.stream None [|item.span|] + let enqueue (item : StreamItem) = updateWritePos item.stream None [| { index = item.index; events = [| item.event |]}|] let busy = HashSet() let schedule (requestedOrder : string seq) (capacity: int) = @@ -190,7 +191,8 @@ type StreamStates() = let markNotBusy stream = busy.Remove stream |> ignore - member __.Add(item: StreamBatch) = enqueue item + //member __.Add(item: StreamBatch) = enqueue item + member __.Add(item: StreamItem) = enqueue item |> ignore member __.TryGetStreamWritePos stream = match states.TryGetValue stream with | true, value -> value.write @@ -232,9 +234,8 @@ type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex type ProgressState<'Pos>(?currentPos : 'Pos) = let pending = Queue<_>() let mutable validatedPos = currentPos - member __.AppendBatch(pos, streamWithRequiredIndices : (string * int64) seq) = - let byStream = streamWithRequiredIndices |> Seq.groupBy fst |> Seq.map (fun (s,xs) -> KeyValuePair(s,xs |> Seq.map snd |> Seq.max)) - pending.Enqueue { pos = pos; streamToRequiredIndex = Dictionary byStream } + member __.AppendBatch(pos, reqs : Dictionary) = + pending.Enqueue { pos = pos; streamToRequiredIndex = reqs } member __.MarkStreamProgress(stream, index) = for x in pending do match x.streamToRequiredIndex.TryGetValue stream with @@ -244,9 +245,12 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = match pending.TryPeek() with | true, batch -> batch.streamToRequiredIndex.Count = 0 | _ -> false + let mutable completed = 0 while headIsComplete () do + completed <- completed + 1 let headBatch = pending.Dequeue() validatedPos <- Some headBatch.pos + completed member __.ScheduledOrder getStreamQueueLength = let raw = seq { let streams = HashSet() @@ -302,7 +306,9 @@ type ProgressWriter() = [] type CoordinatorWork = /// Enqueue a batch of items with supplied tag and progress marking function - | Add of epoch: int64 * markCompleted: Async * items: StreamBatch[] + | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq + /// Log stats about an ingested batch + | Added of streams: int * events: int /// Result of processing on stream - processed up to nominated `index` or threw `exn` | Result of stream: string * outcome: Choice /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` @@ -310,11 +316,10 @@ type CoordinatorWork = type CoordinatorStats(log : ILogger, maxPendingBatches, ?statsInterval) = let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.) - let statsIntervalMs = int64 statsInterval.TotalMilliseconds let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None let progCommitFails, progCommits = ref 0, ref 0 - let cycles, workPended, eventsPended, resultOk, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs statsIntervalMs + let cycles, batchesPended, streamsPended, eventsPended, resultOk, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (busy,capacity) (streams : StreamStates) = if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with @@ -331,32 +336,35 @@ type CoordinatorStats(log : ILogger, maxPendingBatches, ?statsInterval) = else log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - let results = !resultOk + !resultExn - log.Information("Cycles {cycles} Ingested {batches} ({events} events) Active {busy}/{processors} Completed {completed} ({ok} ok {exns} exn)", - !cycles, !workPended, !eventsPended, busy, capacity, results, !resultOk, !resultExn) - cycles := 0; workPended := 0; eventsPended := 0; resultOk := 0; resultExn:= 0 + log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}e) Busy {busy}/{processors} Completed {completed} ({ok} ok {exns} exn)", + !cycles, !batchesPended, !streamsPended, !eventsPended, busy, capacity, !resultOk + !resultExn, !resultOk, !resultExn) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; resultOk := 0; resultExn:= 0 //Metrics.dumpRuStats statsInterval log streams.Dump log member __.Handle = function + | Add _ -> () + | Added (streams, events) -> + incr batchesPended + eventsPended := !eventsPended + events + streamsPended := !streamsPended + streams + | Result (_stream, Choice1Of2 _) -> + incr resultOk + | Result (_stream, Choice2Of2 _) -> + incr resultExn | ProgressResult (Choice1Of2 epoch) -> incr progCommits comittedEpoch <- Some epoch | ProgressResult (Choice2Of2 (_exn : exn)) -> incr progCommitFails - | Add (_epoch, _markCompleted,items) -> - incr workPended - eventsPended := !eventsPended + (items |> Array.sumBy (fun x -> x.span.events.Length)) - | Result (_stream, Choice1Of2 _) -> - incr resultOk - | Result (_stream, Choice2Of2 _) -> - incr resultExn member __.HandleValidated(epoch, pendingBatches) = incr cycles pendingBatchCount <- pendingBatches validatedEpoch <- epoch member __.HandleCommitted epoch = comittedEpoch <- epoch - member __.TryDump(busy,capacity,streams) = if statsDue () then dumpStats (busy,capacity) streams + member __.TryDump(busy,capacity,streams) = + if statsDue () then + dumpStats (busy,capacity) streams /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher(maxDop) = @@ -378,38 +386,43 @@ type Dispatcher(maxDop) = if got then let mutable item = Unchecked.defaultof> if work.TryTake(&item, cancellationCheckInterval) then let! _ = Async.StartChild(dispatch item) in () - else dop.Release() |> ignore - } + else dop.Release() |> ignore } /// Single instance per ChangeFeedObserver, spun up as leases are won and allocated by the ChangeFeedProcessor hosting framework /// Coordinates a) ingestion of events b) execution of projection work c) writing of progress d) reporting of state type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) = let sleepIntervalMs = 5 let cts = new CancellationTokenSource() + let batches = new SemaphoreSlim(maxPendingBatches) let stats = CoordinatorStats(log, maxPendingBatches, ?statsInterval=statsInterval) - let mutable pendingBatches = 0 let progressWriter = ProgressWriter() let progressState = ProgressState() let streams = StreamStates() let work = ConcurrentQueue<_>() + let handle = function + | Add (epoch, checkpoint,items) -> + let reqs = Dictionary() + let mutable count = 0 + for item in items do + streams.Add item + count <- count + 1 + reqs.[item.stream] <- item.index + 1L + progressState.AppendBatch((epoch,checkpoint),reqs) + work.Enqueue(Added (reqs.Count,count)) + | Added _ -> () + | Result (stream, Choice1Of2 index) -> + batches.Release(progressState.MarkStreamProgress(stream,index)) |> ignore + streams.MarkCompleted(stream,index) + | Result (stream, Choice2Of2 _) -> + streams.MarkFailed stream + | ProgressResult _ -> () + member private __.Pump(project : StreamBatch -> Async) = async { let dispatcher = Dispatcher(processorDop) use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) Async.Start(dispatcher.Pump(), cts.Token) - let handle = function - | ProgressResult (Choice1Of2 epoch) -> () - | ProgressResult (Choice2Of2 (_exn : exn)) -> () - | Add (epoch, checkpoint,items) -> - for item in items do - streams.Add item |> ignore - progressState.AppendBatch((epoch,checkpoint), [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) - | Result (stream, Choice1Of2 index) -> - progressState.MarkStreamProgress(stream,index) - streams.MarkCompleted(stream,index) - | Result (stream, Choice2Of2 _) -> - streams.MarkFailed stream while not cts.IsCancellationRequested do // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) @@ -418,8 +431,7 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) stats.HandleValidated(Option.map fst validatedPos, batches) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch - pendingBatches <- batches - // 3. After that, [over] provision writers queue + // 3. After that, provision writers queue let capacity = dispatcher.Capacity if capacity <> 0 then let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) @@ -428,17 +440,16 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) try let! count = project batch return batch.stream, Choice1Of2 (batch.span.index + int64 count) with e -> return batch.stream, Choice2Of2 e } - do! Async.Sleep sleepIntervalMs // 4. Periodically emit status info let busy = processorDop - dispatcher.Capacity - stats.TryDump(busy,processorDop,streams) } + stats.TryDump(busy,processorDop,streams) + do! Async.Sleep sleepIntervalMs } static member Start(rangeLog, maxPendingBatches, processorDop, project) = let instance = new Coordinator(rangeLog, maxPendingBatches, processorDop) Async.Start <| instance.Pump(project) instance - member __.Submit(epoch, markBatchCompleted, events) = - Add (epoch, markBatchCompleted, events) |> work.Enqueue - member __.IsFullyLoaded = - Volatile.Read(&pendingBatches) >= maxPendingBatches + member __.Submit(epoch, markBatchCompleted, events) = async { + let! _ = batches.Await() + Add (epoch, markBatchCompleted, events) |> work.Enqueue } member __.Stop() = cts.Cancel() \ No newline at end of file From da79921a3413fef88196ce7ddf7ac9f1e2d3de5a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 02:18:36 +0100 Subject: [PATCH 128/353] Fixes --- equinox-projector/Projector/Program.fs | 2 +- equinox-projector/Projector/State.fs | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index e4116b01d..fe0d4d7aa 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -219,7 +219,7 @@ let main argv = //#endif let project (batch : StreamBatch) = async { let r = Random() - let ms = r.Next(1,batch.span.events.Length) + let ms = r.Next(1,batch.span.events.Length * 10) do! Async.Sleep ms return batch.span.events.Length } run Log.Logger discovery connector.ConnectionPolicy source diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 0609510b0..1b61bad63 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -411,7 +411,8 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) work.Enqueue(Added (reqs.Count,count)) | Added _ -> () | Result (stream, Choice1Of2 index) -> - batches.Release(progressState.MarkStreamProgress(stream,index)) |> ignore + let batchesCompleted = progressState.MarkStreamProgress(stream,index) + if batchesCompleted <> 0 then batches.Release(batchesCompleted) |> ignore streams.MarkCompleted(stream,index) | Result (stream, Choice2Of2 _) -> streams.MarkFailed stream From e2277c7929289224b0aae1ea35ded29ac26708e5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 02:27:14 +0100 Subject: [PATCH 129/353] Fix delay layout --- equinox-projector/Projector/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index fe0d4d7aa..b389e1d1d 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -185,7 +185,7 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 do! coordinator.Submit(epoch,checkpoint,seq { for x in Seq.collect DocumentParser.enumEvents docs -> { stream = x.Stream; index = x.Index; event = x } }) log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingest {p:n1}s", - epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., (let e = pt.Elapsed in int e.TotalSeconds)) + epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., (let e = pt.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } let init rangeLog = coordinator <- Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project) @@ -219,7 +219,7 @@ let main argv = //#endif let project (batch : StreamBatch) = async { let r = Random() - let ms = r.Next(1,batch.span.events.Length * 10) + let ms = r.Next(1,batch.span.events.Length * 100) do! Async.Sleep ms return batch.span.events.Length } run Log.Logger discovery connector.ConnectionPolicy source From 3f1b147b52157f179dcc3580018451655095ff0d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 02:58:53 +0100 Subject: [PATCH 130/353] Async.Start instead of StartChild --- equinox-projector/Projector/State.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 1b61bad63..6150f4cc7 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -385,7 +385,7 @@ type Dispatcher(maxDop) = let! got = dop.Await(cancellationCheckInterval) if got then let mutable item = Unchecked.defaultof> - if work.TryTake(&item, cancellationCheckInterval) then let! _ = Async.StartChild(dispatch item) in () + if work.TryTake(&item, cancellationCheckInterval) then Async.Start(dispatch item) else dop.Release() |> ignore } /// Single instance per ChangeFeedObserver, spun up as leases are won and allocated by the ChangeFeedProcessor hosting framework From 3caea3d1832c8ff78e8230bde3057e5da7223de9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 03:06:54 +0100 Subject: [PATCH 131/353] Log in flight batches --- equinox-projector/Projector.Tests/ProgressTests.fs | 8 ++++---- equinox-projector/Projector/Program.fs | 6 +++--- equinox-projector/Projector/State.fs | 4 +++- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/equinox-projector/Projector.Tests/ProgressTests.fs b/equinox-projector/Projector.Tests/ProgressTests.fs index 6b5ae8fd7..67677101b 100644 --- a/equinox-projector/Projector.Tests/ProgressTests.fs +++ b/equinox-projector/Projector.Tests/ProgressTests.fs @@ -17,7 +17,7 @@ let [] ``Empty has zero streams pending or progress to write`` () = let [] ``Can add multiple batches`` () = let sut = ProgressState<_>() sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) - sut.AppendBatch(1,mkDictionary["b",2L; "c",3L]) + sut.AppendBatch(1,mkDictionary ["b",2L; "c",3L]) let validatedPos, batches = sut.Validate(fun _ -> None) None =! validatedPos 2 =! batches @@ -25,8 +25,8 @@ let [] ``Can add multiple batches`` () = let [] ``Marking Progress Removes batches and updates progress`` () = let sut = ProgressState<_>() sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) - sut.MarkStreamProgress("a",1L) - sut.MarkStreamProgress("b",1L) + sut.MarkStreamProgress("a",1L) |> ignore + sut.MarkStreamProgress("b",1L) |> ignore let validatedPos, batches = sut.Validate(fun _ -> None) None =! validatedPos 1 =! batches @@ -34,7 +34,7 @@ let [] ``Marking Progress Removes batches and updates progress`` () = let [] ``Marking progress is not persistent`` () = let sut = ProgressState<_>() sut.AppendBatch(0, mkDictionary ["a",1L]) - sut.MarkStreamProgress("a",2L) + sut.MarkStreamProgress("a",2L) |> ignore sut.AppendBatch(1, mkDictionary ["a",1L; "b",2L]) let validatedPos, batches = sut.Validate(fun _ -> None) Some 0 =! validatedPos diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index b389e1d1d..149d2e077 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -183,9 +183,9 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - do! coordinator.Submit(epoch,checkpoint,seq { for x in Seq.collect DocumentParser.enumEvents docs -> { stream = x.Stream; index = x.Index; event = x } }) - log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingest {p:n1}s", - epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., (let e = pt.Elapsed in e.TotalSeconds)) + let! index,max = coordinator.Submit(epoch,checkpoint,seq { for x in Seq.collect DocumentParser.enumEvents docs -> { stream = x.Stream; index = x.Index; event = x } }) + log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingest {index}/{max} {p:n1}s", + epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., index, max, (let e = pt.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } let init rangeLog = coordinator <- Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project) diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 6150f4cc7..73bbc2e04 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -451,6 +451,8 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) instance member __.Submit(epoch, markBatchCompleted, events) = async { let! _ = batches.Await() - Add (epoch, markBatchCompleted, events) |> work.Enqueue } + Add (epoch, markBatchCompleted, events) |> work.Enqueue + return maxPendingBatches-batches.CurrentCount,maxPendingBatches } + member __.Stop() = cts.Cancel() \ No newline at end of file From c0190c772d51e830d7db06e2bb80efacc92bf57a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 05:07:24 +0100 Subject: [PATCH 132/353] MaxThreads --- equinox-projector/Projector/Program.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 149d2e077..1b5eec7dc 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -211,6 +211,7 @@ module Logging = let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ChangeFeedVerbose + if not (System.Threading.ThreadPool.SetMaxThreads(1024,256)) then raise <| CmdParser.MissingArg "Cannot set MaxThreads to 1024" let discovery, connector, source = args.Cosmos.BuildConnectionDetails() let aux, leaseId, startFromHere, batchSize, maxPendingBatches, processorDop, lagFrequency = args.BuildChangeFeedParams() //#if kafka From 6c03a3decad3796adec52de05ed73478497790f0 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 05:29:56 +0100 Subject: [PATCH 133/353] Move work back to main thread --- equinox-projector/Projector/State.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs index 73bbc2e04..6fd13a389 100644 --- a/equinox-projector/Projector/State.fs +++ b/equinox-projector/Projector/State.fs @@ -451,7 +451,7 @@ type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) instance member __.Submit(epoch, markBatchCompleted, events) = async { let! _ = batches.Await() - Add (epoch, markBatchCompleted, events) |> work.Enqueue + Add (epoch, markBatchCompleted, Array.ofSeq events) |> work.Enqueue return maxPendingBatches-batches.CurrentCount,maxPendingBatches } member __.Stop() = From 8cbb31c8fe2a3ac7a3e415e7643e347c1d14c62d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 05:40:22 +0100 Subject: [PATCH 134/353] Drop back to 10x delay --- equinox-projector/Projector/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 1b5eec7dc..8dbdc7eb3 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -220,7 +220,7 @@ let main argv = //#endif let project (batch : StreamBatch) = async { let r = Random() - let ms = r.Next(1,batch.span.events.Length * 100) + let ms = r.Next(1,batch.span.events.Length * 10) do! Async.Sleep ms return batch.span.events.Length } run Log.Logger discovery connector.ConnectionPolicy source From 6baf343d6e617f6158854783c44502e2122d6769 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 25 Apr 2019 05:47:36 +0100 Subject: [PATCH 135/353] Fix MinBatch naming --- equinox-sync/Sync/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 886771fd6..4a10e987b 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -67,7 +67,7 @@ module CmdParser = | [] LagFreqS of float | [] ChangeFeedVerbose #else - | [] MinBatchSize of int + | [] MinBatchSize of int | [] MaxPendingBatches of int | [] MaxWriters of int | [] Position of int64 @@ -78,7 +78,7 @@ module CmdParser = | [] VerboseConsole #endif | [] ForceRestart - | [] BatchSize of int + | [] BatchSize of int | [] Verbose | [] Source of ParseResults interface IArgParserTemplate with From 223257f90fcd863b14f679fb5eec921717b1b143 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 11:21:13 +0100 Subject: [PATCH 136/353] Generalize Projector and Sync --- .../CosmosInternalJson.fs | 57 +++ .../Equinox.Projection.Codec.fsproj | 28 ++ .../Equinox.Projection.Codec/RenderedEvent.fs | 28 ++ .../CosmosIngester.fs | 135 ++++++ .../Equinox.Projection.Cosmos.fsproj | 33 ++ .../Equinox.Projection.Cosmos/Metrics.fs | 71 +++ .../Equinox.Projection.Tests.fsproj | 30 ++ .../FeedValidatorTests.fs | 70 +++ .../Equinox.Projection.Tests/ProgressTests.fs | 41 ++ .../StreamStateTests.fs | 78 +++ .../Equinox.Projection/Coordination.fs | 163 +++++++ .../Equinox.Projection.fsproj | 29 ++ .../Equinox.Projection/FeedValidator.fs | 110 +++++ .../Equinox.Projection/Infrastructure.fs | 45 ++ equinox-projector/Equinox.Projection/State.fs | 269 ++++++++++ equinox-projector/Projector/Program.fs | 11 +- equinox-projector/Projector/Projector.fsproj | 9 +- equinox-projector/Projector/State.fs | 458 ------------------ .../equinox-projector-consumer.sln | 40 +- equinox-sync/Ingest/Program.fs | 2 +- equinox-sync/Sync/Checkpoint.fs | 22 +- equinox-sync/Sync/CosmosIngester.fs | 292 ----------- equinox-sync/Sync/EventStoreSource.fs | 42 +- equinox-sync/Sync/Program.fs | 285 +++-------- equinox-sync/Sync/ProgressBatcher.fs | 39 -- equinox-sync/Sync/Sync.fsproj | 11 +- 26 files changed, 1328 insertions(+), 1070 deletions(-) create mode 100644 equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs create mode 100644 equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj create mode 100644 equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs create mode 100644 equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs create mode 100644 equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj create mode 100644 equinox-projector/Equinox.Projection.Cosmos/Metrics.fs create mode 100644 equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj create mode 100644 equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs create mode 100644 equinox-projector/Equinox.Projection.Tests/ProgressTests.fs create mode 100644 equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs create mode 100644 equinox-projector/Equinox.Projection/Coordination.fs create mode 100644 equinox-projector/Equinox.Projection/Equinox.Projection.fsproj create mode 100644 equinox-projector/Equinox.Projection/FeedValidator.fs create mode 100644 equinox-projector/Equinox.Projection/Infrastructure.fs create mode 100644 equinox-projector/Equinox.Projection/State.fs delete mode 100644 equinox-projector/Projector/State.fs delete mode 100644 equinox-sync/Sync/CosmosIngester.fs delete mode 100644 equinox-sync/Sync/ProgressBatcher.fs diff --git a/equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs b/equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs new file mode 100644 index 000000000..1ed994ce6 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs @@ -0,0 +1,57 @@ +namespace Equinox.Cosmos.Internal.Json + +open Newtonsoft.Json.Linq +open Newtonsoft.Json + +/// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be +type VerbatimUtf8JsonConverter() = + inherit JsonConverter() + + static let enc = System.Text.Encoding.UTF8 + + override __.ReadJson(reader, _, _, _) = + let token = JToken.Load reader + if token.Type = JTokenType.Null then null + else token |> string |> enc.GetBytes |> box + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + + override __.WriteJson(writer, value, serializer) = + let array = value :?> byte[] + if array = null || array.Length = 0 then serializer.Serialize(writer, null) + else writer.WriteRawValue(enc.GetString(array)) + +open System.IO +open System.IO.Compression + +/// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc +/// Only applied to snapshots in the Index +type Base64ZipUtf8JsonConverter() = + inherit JsonConverter() + let pickle (input : byte[]) : string = + if input = null then null else + + use output = new MemoryStream() + use compressor = new DeflateStream(output, CompressionLevel.Optimal) + compressor.Write(input,0,input.Length) + compressor.Close() + System.Convert.ToBase64String(output.ToArray()) + let unpickle str : byte[] = + if str = null then null else + + let compressedBytes = System.Convert.FromBase64String str + use input = new MemoryStream(compressedBytes) + use decompressor = new DeflateStream(input, CompressionMode.Decompress) + use output = new MemoryStream() + decompressor.CopyTo(output) + output.ToArray() + + override __.CanConvert(objectType) = + typeof.Equals(objectType) + override __.ReadJson(reader, _, _, serializer) = + //( if reader.TokenType = JsonToken.Null then null else + serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box + override __.WriteJson(writer, value, serializer) = + let pickled = value |> unbox |> pickle + serializer.Serialize(writer, pickled) \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj b/equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj new file mode 100644 index 000000000..bd30eac24 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj @@ -0,0 +1,28 @@ + + + + netstandard2.0 + 5 + false + true + true + true + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs b/equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs new file mode 100644 index 000000000..0629dd3f4 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs @@ -0,0 +1,28 @@ +namespace Equinox.Projection + +open Newtonsoft.Json +open System + +module Codec = + /// Default rendition of an event when being projected to Kafka + type [] RenderedEvent = + { /// Stream Name + s: string + + /// Index within stream + i: int64 + + /// Event Type associated with event data in `d` + c: string + + /// Timestamp of original write + t: DateTimeOffset // ISO 8601 + + /// Event body, as UTF-8 encoded json ready to be injected directly into the Json being rendered + [)>] + d: byte[] // required + + /// Optional metadata, as UTF-8 encoded json, ready to emit directly (entire field is not written if value is null) + [)>] + [] + m: byte[] } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs new file mode 100644 index 000000000..20afd625d --- /dev/null +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -0,0 +1,135 @@ +module Equinox.Projection.Cosmos.CosmosIngester + +open Equinox.Cosmos.Core +open Equinox.Cosmos.Store +open Equinox.Projection.Coordination +open Equinox.Projection.State +open Serilog + +let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 +let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 + +type [] ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Other | Ok + +module Writer = + type [] Result = + | Ok of updatedPos: int64 + | Duplicate of updatedPos: int64 + | PartialDuplicate of overage: Span + | PrefixMissing of batch: Span * writePos: int64 + let logTo (log: ILogger) (res : string * Choice) = + match res with + | stream, (Choice1Of2 (Ok pos)) -> + log.Information("Wrote {stream} up to {pos}", stream, pos) + | stream, (Choice1Of2 (Duplicate updatedPos)) -> + log.Information("Ignored {stream} (synced up to {pos})", stream, updatedPos) + | stream, (Choice1Of2 (PartialDuplicate overage)) -> + log.Information("Requeing {stream} {pos} ({count} events)", stream, overage.index, overage.events.Length) + | stream, (Choice1Of2 (PrefixMissing (batch,pos))) -> + log.Information("Waiting {stream} missing {gap} events ({count} events @ {pos})", stream, batch.index-pos, batch.events.Length, batch.index) + | stream, Choice2Of2 exn -> + log.Warning(exn,"Writing {stream} failed, retrying", stream) + + let write (log : ILogger) (ctx : CosmosContext) ({ stream = s; span = { index = i; events = e}} as batch) = async { + let stream = ctx.CreateStream s + log.Debug("Writing {s}@{i}x{n}",s,i,e.Length) + let! res = ctx.Sync(stream, { index = i; etag = None }, e) + let ress = + match res with + | AppendResult.Ok pos -> Ok pos.index + | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> + match pos.index with + | actual when actual < i -> PrefixMissing (batch.span, actual) + | actual when actual >= i + e.LongLength -> Duplicate actual + | actual -> PartialDuplicate { index = actual; events = e |> Array.skip (actual-i |> int) } + log.Debug("Result: {res}",ress) + return ress } + let (|TimedOutMessage|RateLimitedMessage|TooLargeMessage|MalformedMessage|Other|) (e: exn) = + match string e with + | m when m.Contains "Microsoft.Azure.Documents.RequestTimeoutException" -> TimedOutMessage + | m when m.Contains "Microsoft.Azure.Documents.RequestRateTooLargeException" -> RateLimitedMessage + | m when m.Contains "Microsoft.Azure.Documents.RequestEntityTooLargeException" -> TooLargeMessage + | m when m.Contains "SyntaxError: JSON.parse Error: Unexpected input at position" + || m.Contains "SyntaxError: JSON.parse Error: Invalid character at position" -> MalformedMessage + | _ -> Other + + let classify = function + | RateLimitedMessage -> ResultKind.RateLimited, false + | TimedOutMessage -> ResultKind.TimedOut, false + | TooLargeMessage -> ResultKind.TooLarge, true + | MalformedMessage -> ResultKind.Malformed, true + | Other -> ResultKind.Other, false + + //member __.TryGap() : (string*int64*int) option = + // let rec aux () = + // match gap |> Queue.tryDequeue with + // | None -> None + // | Some stream -> + + // match states.[stream].TryGap() with + // | Some (pos,count) -> Some (stream,pos,int count) + // | None -> aux () + // aux () + //member __.TryReady(isBusy) = + // let blocked = ResizeArray() + // let rec aux () = + // match dirty |> Queue.tryDequeue with + // | None -> None + // | Some stream -> + + // match states.[stream] with + // | state when state.IsReady -> + // if (not << isBusy) stream then + // let h = state.queue |> Array.head + + // let mutable bytesBudget = cosmosPayloadLimit + // let mutable count = 0 + // let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = + // bytesBudget <- bytesBudget - cosmosPayloadBytes y + // count <- count + 1 + // // Reduce the item count when we don't yet know the write position + // count <= (if Option.isNone state.write then 10 else 4096) && (bytesBudget >= 0 || count = 1) + // Some { stream = stream; span = { index = h.index; events = h.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } + // else + // blocked.Add(stream) |> ignore + // aux () + // | _ -> aux () + // let res = aux () + // for x in blocked do markDirty x + // res + +type CosmosStats private (log : ILogger, writerResultLog, maxPendingBatches, statsInterval) = + inherit Stats(log, maxPendingBatches, statsInterval) + let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 + let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 + let badCats = CatStats() + + override __.DumpExtraStats() = + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn + log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then + log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", + !rateLimited, !timedOut, !tooLarge, !malformed) + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 + if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + Metrics.dumpRuStats statsInterval log + + override __.Handle message = + base.Handle message + match message with + | Message.Result (_stream, Choice1Of2 r) -> + match r with + | Writer.Result.Ok _ -> incr resultOk + | Writer.Result.Duplicate _ -> incr resultDup + | Writer.Result.PartialDuplicate _ -> incr resultPartialDup + | Writer.Result.PrefixMissing _ -> incr resultPrefix + | Result (stream, Choice2Of2 exn) -> + match Writer.classify exn with + | ResultKind.Ok, _ | ResultKind.Other, _ -> () + | ResultKind.RateLimited, _ -> incr rateLimited + | ResultKind.TooLarge, _ -> category stream |> badCats.Ingest; incr tooLarge + | ResultKind.Malformed, _ -> category stream |> badCats.Ingest; incr malformed + | ResultKind.TimedOut, _ -> incr timedOut + | Add _ | AddStream _ | Added _ | ProgressResult _ -> () \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj new file mode 100644 index 000000000..4242a0a46 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj @@ -0,0 +1,33 @@ + + + + netstandard2.0 + 5 + false + true + true + true + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs b/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs new file mode 100644 index 000000000..8e554d856 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs @@ -0,0 +1,71 @@ +module Equinox.Projection.Cosmos.Metrics + +open System + +module RuCounters = + open Equinox.Cosmos.Store + open Serilog.Events + + let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds + + let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function + | Log.Tip (Stats s) + | Log.TipNotFound (Stats s) + | Log.TipNotModified (Stats s) + | Log.Query (_,_, (Stats s)) -> CosmosReadRc s + // slices are rolled up into batches so be sure not to double-count + | Log.Response (_,(Stats s)) -> CosmosResponseRc s + | Log.SyncSuccess (Stats s) + | Log.SyncConflict (Stats s) -> CosmosWriteRc s + | Log.SyncResync (Stats s) -> CosmosResyncRc s + let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function + | (:? ScalarValue as x) -> Some x.Value + | _ -> None + let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = + match logEvent.Properties.TryGetValue("cosmosEvt") with + | true, SerilogScalar (:? Log.Event as e) -> Some e + | _ -> None + type RuCounter = + { mutable rux100: int64; mutable count: int64; mutable ms: int64 } + static member Create() = { rux100 = 0L; count = 0L; ms = 0L } + member __.Ingest (ru, ms) = + System.Threading.Interlocked.Increment(&__.count) |> ignore + System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore + System.Threading.Interlocked.Add(&__.ms, ms) |> ignore + type RuCounterSink() = + static member val Read = RuCounter.Create() with get, set + static member val Write = RuCounter.Create() with get, set + static member val Resync = RuCounter.Create() with get, set + static member Reset() = + RuCounterSink.Read <- RuCounter.Create() + RuCounterSink.Write <- RuCounter.Create() + RuCounterSink.Resync <- RuCounter.Create() + interface Serilog.Core.ILogEventSink with + member __.Emit logEvent = logEvent |> function + | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats + | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats + | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats + | _ -> () + +let dumpRuStats duration (log: Serilog.ILogger) = + let stats = + [ "Read", RuCounters.RuCounterSink.Read + "Write", RuCounters.RuCounterSink.Write + "Resync", RuCounters.RuCounterSink.Resync ] + let mutable totalCount, totalRc, totalMs = 0L, 0., 0L + let logActivity name count rc lat = + if count <> 0L then + log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", + name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) + for name, stat in stats do + let ru = float stat.rux100 / 100. + totalCount <- totalCount + stat.count + totalRc <- totalRc + ru + totalMs <- totalMs + stat.ms + logActivity name stat.count ru stat.ms + logActivity "TOTAL" totalCount totalRc totalMs + // Yes, there's a minor race here! + RuCounters.RuCounterSink.Reset() + let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] + let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) + for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj b/equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj new file mode 100644 index 000000000..002dc015f --- /dev/null +++ b/equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj @@ -0,0 +1,30 @@ + + + + netcoreapp2.1;net461 + 5 + false + + + + + + + + + + + + + + + + + + + all + runtime; build; native; contentfiles; analyzers + + + + diff --git a/equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs b/equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs new file mode 100644 index 000000000..7df7c8112 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs @@ -0,0 +1,70 @@ +module Equinox.Projection.FeedValidator.Tests + +open Equinox.Projection.Validation +open FsCheck.Xunit +open Swensen.Unquote +open Xunit + +let [] ``Properties`` state index = + let result,state' = StreamState.combine state index + match state with + | Some (Partial (_, pos)) when pos <= 1 -> () + | Some (Partial (min, pos)) when min > pos || min = 0 -> () + | None -> + match state' with + | All _ -> result =! New + | Partial _ -> result =! Ok + | Some (All x) -> + match state',result with + | All x' , Duplicate -> x' =! x + | All x', New -> x' =! x+1 + | All x', Gap -> x' =! x + | x -> failwithf "Unexpected %A" x + | Some (Partial (min, pos)) -> + match state',result with + | All 0,Duplicate when min=0 && index = 0 -> () + | All x,Duplicate when min=1 && pos = x && index = 0 -> () + | Partial (min', pos'), Duplicate -> min' =! max min' index; pos' =! pos + | Partial (min', pos'), Ok + | Partial (min', pos'), New -> min' =! min; pos' =! index + | x -> failwithf "Unexpected %A" x + +let [] ``Zero on unknown stream is New`` () = + let result,state = StreamState.combine None 0 + New =! result + All 0 =! state + +let [] ``Non-zero on unknown stream is Ok`` () = + let result,state = StreamState.combine None 1 + Ok =! result + Partial (1,1) =! state + +let [] ``valid successor is New`` () = + let result,state = StreamState.combine (All 0 |> Some) 1 + New =! result + All 1 =! state + +let [] ``single immediate repeat is flagged`` () = + let result,state = StreamState.combine (All 0 |> Some) 0 + Duplicate =! result + All 0 =! state + +let [] ``non-immediate repeat is flagged`` () = + let result,state = StreamState.combine (All 1 |> Some) 0 + Duplicate =! result + All 1 =! state + +let [] ``Gap is flagged`` () = + let result,state = StreamState.combine (All 1 |> Some) 3 + Gap =! result + All 1 =! state + +let [] ``Potential gaps are not flagged as such when we're processing a Partial`` () = + let result,state = StreamState.combine (Some (Partial (1,1))) 3 + New =! result + Partial (1,3) =! state + +let [] ``Earlier values widen the min when we're processing a Partial`` () = + let result,state = StreamState.combine (Some (Partial (2,3))) 1 + Duplicate =! result + Partial (1,3) =! state \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs new file mode 100644 index 000000000..ab1bdde2f --- /dev/null +++ b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs @@ -0,0 +1,41 @@ +module ProgressTests + +open Equinox.Projection.State + +open Swensen.Unquote +open Xunit +open System.Collections.Generic + +let mkDictionary xs = Dictionary(dict xs) + +let [] ``Empty has zero streams pending or progress to write`` () = + let sut = ProgressState<_>() + let validatedPos, batches = sut.Validate(fun _ -> None) + None =! validatedPos + 0 =! batches + +let [] ``Can add multiple batches`` () = + let sut = ProgressState<_>() + sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) + sut.AppendBatch(1,mkDictionary ["b",2L; "c",3L]) + let validatedPos, batches = sut.Validate(fun _ -> None) + None =! validatedPos + 2 =! batches + +let [] ``Marking Progress Removes batches and updates progress`` () = + let sut = ProgressState<_>() + sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) + sut.MarkStreamProgress("a",1L) |> ignore + sut.MarkStreamProgress("b",1L) |> ignore + let validatedPos, batches = sut.Validate(fun _ -> None) + None =! validatedPos + 1 =! batches + +let [] ``Marking progress is not persistent`` () = + let sut = ProgressState<_>() + sut.AppendBatch(0, mkDictionary ["a",1L]) + sut.MarkStreamProgress("a",2L) |> ignore + sut.AppendBatch(1, mkDictionary ["a",1L; "b",2L]) + let validatedPos, batches = sut.Validate(fun _ -> None) + Some 0 =! validatedPos + 1 =! batches \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs b/equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs new file mode 100644 index 000000000..5431c7df7 --- /dev/null +++ b/equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs @@ -0,0 +1,78 @@ +module CosmosIngesterTests + +open Equinox.Projection.State +open Swensen.Unquote +open Xunit + +let canonicalTime = System.DateTimeOffset.UtcNow +let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } +let mergeSpans = StreamState.Span.merge +let trimSpans = StreamState.Span.trim + +let [] ``nothing`` () = + let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] + r =! null + +let [] ``synced`` () = + let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] + r =! null + +let [] ``no overlap`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] + r =! [| mk 0L 1; mk 2L 2 |] + +let [] ``overlap`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] + r =! [| mk 0L 2 |] + +let [] ``remove nulls`` () = + let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] + r =! [| mk 1L 1 |] + +let [] ``adjacent`` () = + let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] + r =! [| mk 0L 3 |] + +let [] ``adjacent to min`` () = + let r = List.map (trimSpans 2L) [ mk 0L 1; mk 1L 2 ] + r =! [ mk 2L 0; mk 2L 1 ] + +let [] ``adjacent to min merge`` () = + let r = mergeSpans 2L [ mk 0L 1; mk 1L 2 ] + r =! [| mk 2L 1 |] + +let [] ``adjacent to min no overlap`` () = + let r = mergeSpans 2L [ mk 0L 1; mk 2L 1 ] + r =! [| mk 2L 1|] + +let [] ``adjacent trim`` () = + let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2 ] + r =! [ mk 1L 1; mk 2L 2 ] + +let [] ``adjacent trim merge`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] + r =! [| mk 1L 3 |] + +let [] ``adjacent trim append`` () = + let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2; mk 5L 1] + r =! [ mk 1L 1; mk 2L 2; mk 5L 1 ] + +let [] ``adjacent trim append merge`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] + r =! [| mk 1L 3; mk 5L 1 |] + +let [] ``mixed adjacent trim append`` () = + let r = List.map (trimSpans 1L) [ mk 0L 2; mk 5L 1; mk 2L 2] + r =! [ mk 1L 1; mk 5L 1; mk 2L 2 ] + +let [] ``mixed adjacent trim append merge`` () = + let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2] + r =! [| mk 1L 3; mk 5L 1 |] + +let [] ``fail`` () = + let r = mergeSpans 11614L [ {index=11614L; events=null}; mk 11614L 1 ] + r =! [| mk 11614L 1 |] + +let [] ``fail 2`` () = + let r = mergeSpans 11613L [ mk 11614L 1; {index=11614L; events=null} ] + r =! [| mk 11614L 1 |] diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs new file mode 100644 index 000000000..a4509d2dd --- /dev/null +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -0,0 +1,163 @@ +module Equinox.Projection.Coordination + +open Equinox.Projection.State +open Serilog +open System +open System.Collections.Concurrent +open System.Collections.Generic +open System.Threading + +[] +type Message<'R> = + /// Enqueue a batch of items with supplied tag and progress marking function + | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq + | AddStream of StreamSpan + /// Log stats about an ingested batch + | Added of streams: int * events: int + /// Result of processing on stream - specified number of items or threw `exn` + | Result of stream: string * outcome: Choice<'R,exn> + /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` + | ProgressResult of Choice + +type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = + let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None + let progCommitFails, progCommits = ref 0, ref 0 + let cycles, batchesPended, streamsPended, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + let dumpStats (busy,capacity) (streams : StreamStates) = + if !progCommitFails <> 0 || !progCommits <> 0 then + match comittedEpoch with + | None -> + log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncommitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) + | Some committed when !progCommitFails <> 0 -> + log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncommitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) + | Some committed -> + log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncommitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) + progCommits := 0; progCommitFails := 0 + else + log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", + Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) + log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}e) Busy {busy}/{processors} Completed {completed} ({passed} passed {exns} exn)", + !cycles, !batchesPended, !streamsPended, !eventsPended, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + streams.Dump log + abstract member Handle : Message<'R> -> unit + default __.Handle res = + match res with + | Add _ | AddStream _ -> () + | Added (streams, events) -> + incr batchesPended + eventsPended := !eventsPended + events + streamsPended := !streamsPended + streams + | Result (_stream, Choice1Of2 _) -> + incr resultCompleted + | Result (_stream, Choice2Of2 _) -> + incr resultExn + | ProgressResult (Choice1Of2 epoch) -> + incr progCommits + comittedEpoch <- Some epoch + | ProgressResult (Choice2Of2 (_exn : exn)) -> + incr progCommitFails + member __.HandleValidated(epoch, pendingBatches) = + incr cycles + pendingBatchCount <- pendingBatches + validatedEpoch <- epoch + member __.HandleCommitted epoch = + comittedEpoch <- epoch + member __.TryDump(busy,capacity,streams) = + if statsDue () then + dumpStats (busy,capacity) streams + __.DumpExtraStats() + abstract DumpExtraStats : unit -> unit + default __.DumpExtraStats () = () + +/// Single instance per Source; Coordinates +/// a) ingestion of events +/// b) execution of projection/ingestion work +/// c) writing of progress +/// d) reporting of state +/// The key bit that's managed externally is the reading/accepting of incoming data +type Coordinator<'R>(log : ILogger, maxPendingBatches, processorDop, project : int64 option * StreamSpan -> Async>, handleResult, statsInterval) = + let sleepIntervalMs = 5 + let cts = new CancellationTokenSource() + let batches = new SemaphoreSlim(maxPendingBatches) + let work = ConcurrentQueue>() + let streams = StreamStates() + let dispatcher = Dispatcher(processorDop) + let progressState = ProgressState() + let progressWriter = ProgressWriter<_>() + let stats = Stats(log, maxPendingBatches, statsInterval) + let handle = function + | Add (epoch, checkpoint,items) -> + let reqs = Dictionary() + let mutable count = 0 + for item in items do + streams.Add item + count <- count + 1 + reqs.[item.stream] <- item.index + 1L + progressState.AppendBatch((epoch,checkpoint),reqs) + work.Enqueue(Added (reqs.Count,count)) + | AddStream streamSpan -> + streams.Add(streamSpan,false) |> ignore + work.Enqueue(Added (1,streamSpan.span.events.Length)) + | Added _ | ProgressResult _ | Result _ -> + () + + member private __.Pump() = async { + use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) + use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) + Async.Start(progressWriter.Pump(), cts.Token) + Async.Start(dispatcher.Pump(), cts.Token) + let handle x = + handle x + match x with Result _ as r -> handleResult (streams, progressState, batches) r | _ -> () + stats.Handle x + while not cts.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + work |> ConcurrentQueue.drain handle + // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) + let validatedPos, batches = progressState.Validate(streams.TryGetStreamWritePos) + stats.HandleValidated(Option.map fst validatedPos, batches) + validatedPos |> Option.iter progressWriter.Post + stats.HandleCommitted progressWriter.CommittedEpoch + // 3. After that, provision writers queue + let capacity = dispatcher.Capacity + if capacity <> 0 then + let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) + for batch in work do + dispatcher.Enqueue(project batch) + // 4. Periodically emit status info + let busy = processorDop - dispatcher.Capacity + stats.TryDump(busy,processorDop,streams) + do! Async.Sleep sleepIntervalMs } + static member Start<'R>(rangeLog, maxPendingBatches, processorDop, project, handleResult, statsInterval) = + let instance = new Coordinator<'R>(rangeLog, maxPendingBatches, processorDop, project, handleResult, statsInterval) + Async.Start <| instance.Pump() + instance + static member Start(rangeLog, maxPendingBatches, processorDop, project : StreamSpan -> Async, statsInterval) = + let project (_maybeWritePos, batch) = async { + try let! count = project batch + return batch.stream, Choice1Of2 (batch.span.index + int64 count) + with e -> return batch.stream, Choice2Of2 e } + let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) = function + | Result (stream, Choice1Of2 index) -> + match progressState.MarkStreamProgress(stream,index) with 0 -> () | batchesCompleted -> batches.Release(batchesCompleted) |> ignore + streams.MarkCompleted(stream,index) + | Result (stream, Choice2Of2 _) -> + streams.MarkFailed stream + | _ -> () + Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project, handleResult, statsInterval) + + member __.Submit(epoch, markBatchCompleted, events) = async { + let! _ = batches.Await() + Add (epoch, markBatchCompleted, Array.ofSeq events) |> work.Enqueue + return maxPendingBatches-batches.CurrentCount,maxPendingBatches } + + member __.Submit(streamSpan) = + AddStream streamSpan |> work.Enqueue + + member __.Stop() = + cts.Cancel() \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj new file mode 100644 index 000000000..c26c3d2c8 --- /dev/null +++ b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj @@ -0,0 +1,29 @@ + + + + netstandard2.0 + 5 + false + true + true + true + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/FeedValidator.fs b/equinox-projector/Equinox.Projection/FeedValidator.fs new file mode 100644 index 000000000..bdb176125 --- /dev/null +++ b/equinox-projector/Equinox.Projection/FeedValidator.fs @@ -0,0 +1,110 @@ +module Equinox.Projection.Validation + +open System.Collections.Generic + +module private Option = + let defaultValue def option = defaultArg option def + +/// Represents the categorisation of an item being ingested +type IngestResult = + /// The item is a correct item at the tail of the known sequence + | New + /// Consistent as far as we know (but this Validator has not seen the head) + | Ok + /// The item represents a duplicate of an earlier item + | Duplicate + /// The item is beyond the tail of the stream and likely represets a gap + | Gap + +/// Represents the present state of a given stream +type StreamState = + /// We've observed the stream from the start + | All of pos: int + /// We've not observed the stream from the start + | Partial of min: int * pos: int + +module StreamState = + let combine (state : StreamState option) index : IngestResult*StreamState = + match state, index with + | None, 0 -> New, All 0 + | None, x -> Ok, Partial (x,x) + | Some (All x), i when i <= x -> Duplicate, All x + | Some (All x), i when i = x + 1 -> New, All i + | Some (All x), _ -> Gap, All x + | Some (Partial (min=1; pos=pos)), 0 -> Duplicate, All pos + | Some (Partial (min=min; pos=x)), i when i <= min -> Duplicate, Partial (i, x) + | Some (Partial (min=min; pos=x)), i when i = x + 1 -> Ok, Partial (min, i) + | Some (Partial (min=min)), i -> New, Partial (min, i) + + +type FeedStats = { complete: int; partial: int } + +/// Maintains the state of a set of streams being ingested into a processor for consistency checking purposes +/// - to determine whether an incoming event on a stream should be considered a duplicate and hence not processed +/// - to allow inconsistencies to be logged +type FeedValidator() = + let streamStates = System.Collections.Generic.Dictionary() + + /// Thread safe operation to a) classify b) track change implied by a new message as encountered + member __.Ingest(stream, index) : IngestResult * StreamState = + lock streamStates <| fun () -> + let state = + match streamStates.TryGetValue stream with + | true, state -> Some state + | false, _ -> None + let (res, state') = StreamState.combine state index + streamStates.[stream] <- state' + res, state' + + /// Determine count of streams being tracked + member __.Stats = + lock streamStates <| fun () -> + let raw = streamStates |> Seq.countBy (fun x -> match x.Value with All _ -> true | Partial _ -> false) |> Seq.toList + { complete = raw |> List.tryPick (function (true,c) -> Some c | (false,_) -> None) |> Option.defaultValue 0 + partial = raw |> List.tryPick (function (false,c) -> Some c | (true,_) -> None) |> Option.defaultValue 0 } + +type [] StreamSummary = { mutable fresh : int; mutable ok : int; mutable dup : int; mutable gap : int; mutable complete: bool } + +type BatchStats = { fresh : int; ok : int; dup : int; gap : int; categories : int; streams : BatchStreamStats } with + member s.TotalStreams = let s = s.streams in s.complete + s.incomplete + +and BatchStreamStats = { complete: int; incomplete: int } + +/// Used to establish aggregate stats for a batch of inputs for logging purposes +/// The Ingested inputs are passed to the supplied validator in order to classify them +type BatchValidator(validator : FeedValidator) = + let streams = System.Collections.Generic.Dictionary() + let streamSummary (streamName : string) = + match streams.TryGetValue streamName with + | true, acc -> acc + | false, _ -> let t = { fresh = 0; ok = 0; dup = 0; gap = 0; complete = false } in streams.[streamName] <- t; t + + /// Collate into Feed Validator and Batch stats + member __.TryIngest(stream, index) : IngestResult = + let res, state = validator.Ingest(stream, index) + let streamStats = streamSummary stream + match state with + | All _ -> streamStats.complete <- true + | Partial _ -> streamStats.complete <- false + match res with + | New -> streamStats.fresh <- streamStats.fresh + 1 + | Ok -> streamStats.ok <- streamStats.ok + 1 + | Duplicate -> streamStats.dup <- streamStats.dup + 1 + | Gap -> streamStats.gap <- streamStats.gap + 1 + res + + member __.Enum() : IEnumerable> = upcast streams + + member __.Stats : BatchStats = + let mutable fresh, ok, dup, gap, complete, incomplete = 0, 0, 0, 0, 0, 0 + let cats = HashSet() + for KeyValue(k,v) in streams do + fresh <- fresh + v.fresh + ok <- ok + v.ok + dup <- dup + v.dup + gap <- gap + v.gap + match k.IndexOf('-') with + | -1 -> () + | i -> cats.Add(k.Substring(0, i)) |> ignore + if v.complete then complete <- complete + 1 else incomplete <- incomplete + 1 + { fresh = fresh; ok = ok; dup = dup; gap = gap; categories = cats.Count; streams = { complete = complete; incomplete = incomplete } } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Infrastructure.fs b/equinox-projector/Equinox.Projection/Infrastructure.fs new file mode 100644 index 000000000..ea928dc24 --- /dev/null +++ b/equinox-projector/Equinox.Projection/Infrastructure.fs @@ -0,0 +1,45 @@ +[] +module Equinox.Projection.Infrastructure + +open System +open System.Collections.Concurrent +open System.Threading +open System.Threading.Tasks + +module ConcurrentQueue = + let drain handle (xs : ConcurrentQueue<_>) = + let rec aux () = + match xs.TryDequeue() with + | true, x -> handle x; aux () + | false, _ -> () + aux () + +type Async with + static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) + /// Asynchronously awaits the next keyboard interrupt event + static member AwaitKeyboardInterrupt () : Async = + Async.FromContinuations(fun (sc,_,_) -> + let isDisposed = ref 0 + let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore + and d : IDisposable = Console.CancelKeyPress.Subscribe callback + in ()) + static member AwaitTaskCorrect (task : Task<'T>) : Async<'T> = + Async.FromContinuations <| fun (k,ek,_) -> + task.ContinueWith (fun (t:Task<'T>) -> + if t.IsFaulted then + let e = t.Exception + if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] + else ek e + elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) + elif t.IsCompleted then k t.Result + else ek(Exception "invalid Task state!")) + |> ignore + +type SemaphoreSlim with + /// F# friendly semaphore await function + member semaphore.Await(?timeout : TimeSpan) = async { + let! ct = Async.CancellationToken + let timeout = defaultArg timeout Timeout.InfiniteTimeSpan + let task = semaphore.WaitAsync(timeout, ct) + return! Async.AwaitTaskCorrect task + } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs new file mode 100644 index 000000000..5abbc5f2c --- /dev/null +++ b/equinox-projector/Equinox.Projection/State.fs @@ -0,0 +1,269 @@ +module Equinox.Projection.State + +open Serilog +open System.Collections.Generic +open System.Diagnostics +open System.Threading +open System +open System.Collections.Concurrent + +let every ms f = + let timer = Stopwatch.StartNew() + fun () -> + if timer.ElapsedMilliseconds > ms then + f () + timer.Restart() +let expiredMs ms = + let timer = Stopwatch.StartNew() + fun () -> + let due = timer.ElapsedMilliseconds > ms + if due then timer.Restart() + due + +let arrayBytes (x:byte[]) = if x = null then 0 else x.Length +let private mb x = float x / 1024. / 1024. +let category (streamName : string) = streamName.Split([|'-'|],2).[0] + +type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } +type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } +type [] StreamSpan = { stream: string; span: Span } +type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with + member __.Size = + if __.queue = null then 0 + else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) + member __.TryGap() = + if __.queue = null then None + else + match __.write, Array.tryHead __.queue with + | Some w, Some { index = i } when i > w -> Some (w, i-w) + | _ -> None + member __.IsReady = + if __.queue = null || __.isMalformed then false + else + match __.write, Array.tryHead __.queue with + | Some w, Some { index = i } -> i = w + | None, _ -> true + | _ -> false + +module StreamState = + let (|NNA|) xs = if xs = null then Array.empty else xs + module Span = + let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength + let trim min = function + | x when x.index >= min -> x // don't adjust if min not within + | End n when n < min -> { index = min; events = [||] } // throw away if before min + | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice + let merge min (xs : Span seq) = + let xs = + seq { for x in xs -> { x with events = (|NNA|) x.events } } + |> Seq.map (trim min) + |> Seq.filter (fun x -> x.events.Length <> 0) + |> Seq.sortBy (fun x -> x.index) + let buffer = ResizeArray() + let mutable curr = None + for x in xs do + match curr, x with + // Not overlapping, no data buffered -> buffer + | None, _ -> + curr <- Some x + // Gap + | Some (End nextIndex as c), x when x.index > nextIndex -> + buffer.Add c + curr <- Some x + // Overlapping, join + | Some (End nextIndex as c), x -> + curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } + curr |> Option.iter buffer.Add + if buffer.Count = 0 then null else buffer.ToArray() + + let inline optionCombine f (r1: int64 option) (r2: int64 option) = + match r1, r2 with + | Some x, Some y -> f x y |> Some + | None, None -> None + | None, x | x, None -> x + let combine (s1: StreamState) (s2: StreamState) : StreamState = + let writePos = optionCombine max s1.write s2.write + let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 + { write = writePos; queue = Span.merge (defaultArg writePos 0L) items; isMalformed = s1.isMalformed || s2.isMalformed } + +/// Gathers stats relating to how many items of a given category have been observed +type CatStats() = + let cats = Dictionary() + member __.Ingest(cat,?weight) = + let weight = defaultArg weight 1L + match cats.TryGetValue cat with + | true, catCount -> cats.[cat] <- catCount + weight + | false, _ -> cats.[cat] <- weight + member __.Any = cats.Count <> 0 + member __.Clear() = cats.Clear() + member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd + +type StreamStates() = + let states = Dictionary() + let update stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + stream, state + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + stream, updated + let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } + let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore + let enqueue isMalformed (item : StreamItem) = updateWritePos item.stream isMalformed None [| { index = item.index; events = [| item.event |] } |] |> ignore + + let busy = HashSet() + let schedule (requestedOrder : string seq) (capacity: int) = + let toSchedule = ResizeArray<_>(capacity) + let xs = requestedOrder.GetEnumerator() + while xs.MoveNext() && toSchedule.Capacity <> 0 do + let x = xs.Current + let state = states.[x] + if not state.isMalformed && busy.Add x then + let q = state.queue + if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", x) + toSchedule.Add(state.write, { stream = x; span = q.[0] }) + toSchedule.ToArray() + let markNotBusy stream = + busy.Remove stream |> ignore + + member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } + member __.Add(item: StreamItem, ?isMalformed) = + enqueue (defaultArg isMalformed false) item + member __.Add(batch: StreamSpan, isMalformed) = + updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] + member __.SetMalformed(stream,isMalformed) = + updateWritePos stream isMalformed None [| { index = 0L; events = null } |] + // DEPRECATED - will be removed + member __.TryGetStreamWritePos stream = + match states.TryGetValue stream with + | true, value -> value.write + | false, _ -> None + member __.QueueLength(stream) = + let q = states.[stream].queue + if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", stream) + q.[0].events.Length + member __.MarkCompleted(stream, index) = + markNotBusy stream + markCompleted stream index + member __.MarkFailed stream = + markNotBusy stream + member __.Schedule(requestedOrder : string seq, capacity: int) : (int64 option * StreamSpan)[] = + schedule requestedOrder capacity + member __.Dump(log : ILogger) = + let mutable busyCount, busyB, ready, readyB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0 + let busyCats, readyCats, readyStreams = CatStats(), CatStats(), CatStats() + for KeyValue (stream,state) in states do + match int64 state.Size with + | 0L -> + synced <- synced + 1 + | sz when state.isMalformed -> + malformed <- malformed + 1 + malformedB <- malformedB + sz + | sz when busy.Contains stream -> + busyCats.Ingest(category stream) + busyCount <- busyCount + 1 + busyB <- busyB + sz + | sz -> + readyCats.Ingest(category stream) + readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) + ready <- ready + 1 + readyB <- readyB + sz + log.Information("Busy {busy}/{busyMb:n1}MB Ready {ready}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", + busyCount, mb busyB, ready, mb readyB, malformed, mb, malformedB, synced) + if busyCats.Any then log.Information("Busy Categories, events {busyCats}", busyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + +type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } + +type ProgressState<'Pos>(?currentPos : 'Pos) = + let pending = Queue<_>() + let mutable validatedPos = currentPos + member __.AppendBatch(pos, reqs : Dictionary) = + pending.Enqueue { pos = pos; streamToRequiredIndex = reqs } + member __.MarkStreamProgress(stream, index) = + for x in pending do + match x.streamToRequiredIndex.TryGetValue stream with + | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore + | _, _ -> () + let headIsComplete () = pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 + let mutable completed = 0 + while headIsComplete () do + completed <- completed + 1 + let headBatch = pending.Dequeue() + validatedPos <- Some headBatch.pos + completed + member __.ScheduledOrder getStreamQueueLength = + let raw = seq { + let streams = HashSet() + let mutable batch = 0 + for x in pending do + batch <- batch + 1 + for s in x.streamToRequiredIndex.Keys do + if streams.Add s then + yield s,(batch,getStreamQueueLength s) } + raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst + member __.Validate tryGetStreamWritePos : 'Pos option * int = + let rec aux () = + if pending.Count = 0 then () else + let batch = pending.Peek() + for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do + match tryGetStreamWritePos stream with + | Some index when requiredIndex <= index -> + Log.Warning("Validation had to remove {stream}", stream) + batch.streamToRequiredIndex.Remove stream |> ignore + | _ -> () + if batch.streamToRequiredIndex.Count = 0 then + let headBatch = pending.Dequeue() + validatedPos <- Some headBatch.pos + aux () + aux () + validatedPos, pending.Count + +/// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint +type Dispatcher<'R>(maxDop) = + let cancellationCheckInterval = TimeSpan.FromMilliseconds 5. + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let result = Event<'R>() + let dop = new SemaphoreSlim(maxDop) + let dispatch work = async { + let! res = work + result.Trigger res + dop.Release() |> ignore } + [] member __.Result = result.Publish + member __.Capacity = dop.CurrentCount + member __.Enqueue item = work.Add item + member __.Pump () = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + let! got = dop.Await(cancellationCheckInterval) + if got then + let mutable item = Unchecked.defaultof> + if work.TryTake(&item, cancellationCheckInterval) then Async.Start(dispatch item) + else dop.Release() |> ignore } + +/// Manages writing of progress +/// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) +/// - retries until success or a new item is posted +type ProgressWriter<'Res when 'Res: equality>() = + let pumpSleepMs = 100 + let due = expiredMs 5000L + let mutable committedEpoch = None + let mutable validatedPos = None + let result = Event>() + [] member __.Result = result.Publish + member __.Post(version,f) = + Volatile.Write(&validatedPos,Some (version,f)) + member __.CommittedEpoch = Volatile.Read(&committedEpoch) + member __.Pump() = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + match Volatile.Read &validatedPos with + | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> + try do! f + Volatile.Write(&committedEpoch, Some v) + result.Trigger (Choice1Of2 v) + with e -> result.Trigger (Choice2Of2 e) + | _ -> do! Async.Sleep pumpSleepMs } \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 8dbdc7eb3..c71686724 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -5,13 +5,14 @@ open Confluent.Kafka //#endif open Equinox.Cosmos open Equinox.Cosmos.Projection +open Equinox.Projection.Coordination //#if kafka open Equinox.Projection.Codec +open Equinox.Store open Jet.ConfluentKafka.FSharp //#else -open ProjectorTemplate.Projector.State +open Equinox.Projection.State //#endif -open Equinox.Store open Microsoft.Azure.Documents.ChangeFeedProcessor open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing //#if kafka @@ -175,7 +176,7 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broke ChangeFeedObserver.Create(log, projectBatch, dispose = disposeProducer) //#else let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) () = - let mutable coordinator = Unchecked.defaultof + let mutable coordinator = Unchecked.defaultof> let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us @@ -188,7 +189,7 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., index, max, (let e = pt.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - let init rangeLog = coordinator <- Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project) + let init rangeLog = coordinator <- Coordinator<_>.Start(rangeLog, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) let dispose () = coordinator.Stop() ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) //#endif @@ -218,7 +219,7 @@ let main argv = //let targetParams = args.Target.BuildTargetParams() //let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams //#endif - let project (batch : StreamBatch) = async { + let project (batch : StreamSpan) = async { let r = Random() let ms = r.Next(1,batch.span.events.Length * 10) do! Async.Sleep ms diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index abdf59346..0461755b0 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -8,19 +8,22 @@ - - - + + + + + + \ No newline at end of file diff --git a/equinox-projector/Projector/State.fs b/equinox-projector/Projector/State.fs deleted file mode 100644 index 6fd13a389..000000000 --- a/equinox-projector/Projector/State.fs +++ /dev/null @@ -1,458 +0,0 @@ -module ProjectorTemplate.Projector.State - -open Serilog -open System -open System.Collections.Concurrent -open System.Collections.Generic -open System.Diagnostics -open System.Threading - -module Metrics = - module RuCounters = - open Equinox.Cosmos.Store - open Serilog.Events - - let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds - - let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function - | Log.Tip (Stats s) - | Log.TipNotFound (Stats s) - | Log.TipNotModified (Stats s) - | Log.Query (_,_, (Stats s)) -> CosmosReadRc s - // slices are rolled up into batches so be sure not to double-count - | Log.Response (_,(Stats s)) -> CosmosResponseRc s - | Log.SyncSuccess (Stats s) - | Log.SyncConflict (Stats s) -> CosmosWriteRc s - | Log.SyncResync (Stats s) -> CosmosResyncRc s - let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function - | (:? ScalarValue as x) -> Some x.Value - | _ -> None - let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = - match logEvent.Properties.TryGetValue("cosmosEvt") with - | true, SerilogScalar (:? Log.Event as e) -> Some e - | _ -> None - type RuCounter = - { mutable rux100: int64; mutable count: int64; mutable ms: int64 } - static member Create() = { rux100 = 0L; count = 0L; ms = 0L } - member __.Ingest (ru, ms) = - System.Threading.Interlocked.Increment(&__.count) |> ignore - System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore - System.Threading.Interlocked.Add(&__.ms, ms) |> ignore - type RuCounterSink() = - static member val Read = RuCounter.Create() with get, set - static member val Write = RuCounter.Create() with get, set - static member val Resync = RuCounter.Create() with get, set - static member Reset() = - RuCounterSink.Read <- RuCounter.Create() - RuCounterSink.Write <- RuCounter.Create() - RuCounterSink.Resync <- RuCounter.Create() - interface Serilog.Core.ILogEventSink with - member __.Emit logEvent = logEvent |> function - | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats - | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats - | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats - | _ -> () - - let dumpRuStats duration (log: Serilog.ILogger) = - let stats = - [ "Read", RuCounters.RuCounterSink.Read - "Write", RuCounters.RuCounterSink.Write - "Resync", RuCounters.RuCounterSink.Resync ] - let mutable totalCount, totalRc, totalMs = 0L, 0., 0L - let logActivity name count rc lat = - if count <> 0L then - log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", - name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) - for name, stat in stats do - let ru = float stat.rux100 / 100. - totalCount <- totalCount + stat.count - totalRc <- totalRc + ru - totalMs <- totalMs + stat.ms - logActivity name stat.count ru stat.ms - logActivity "TOTAL" totalCount totalRc totalMs - // Yes, there's a minor race here! - RuCounters.RuCounterSink.Reset() - let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] - let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) - for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) - -module ConcurrentQueue = - let drain handle (xs : ConcurrentQueue<_>) = - let rec aux () = - match xs.TryDequeue() with - | true, x -> handle x; aux () - | false, _ -> () - aux () - -let every ms f = - let timer = Stopwatch.StartNew() - fun () -> - if timer.ElapsedMilliseconds > ms then - f () - timer.Restart() -let expiredMs ms = - let timer = Stopwatch.StartNew() - fun () -> - let due = timer.ElapsedMilliseconds > ms - if due then timer.Restart() - due - -let arrayBytes (x:byte[]) = if x = null then 0 else x.Length -let private mb x = float x / 1024. / 1024. -let category (streamName : string) = streamName.Split([|'-'|],2).[0] - -type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } -type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } -type [] StreamBatch = { stream: string; span: Span } -type [] StreamState = { write: int64 option; queue: Span[] } with - member __.Size = - if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) - -module StreamState = - let (|NNA|) xs = if xs = null then Array.empty else xs - module Span = - let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength - let trim min = function - | x when x.index >= min -> x // don't adjust if min not within - | End n when n < min -> { index = min; events = [||] } // throw away if before min - | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice - let merge min (xs : Span seq) = - let xs = - seq { for x in xs -> { x with events = (|NNA|) x.events } } - |> Seq.map (trim min) - |> Seq.filter (fun x -> x.events.Length <> 0) - |> Seq.sortBy (fun x -> x.index) - let buffer = ResizeArray() - let mutable curr = None - for x in xs do - match curr, x with - // Not overlapping, no data buffered -> buffer - | None, _ -> - curr <- Some x - // Gap - | Some (End nextIndex as c), x when x.index > nextIndex -> - buffer.Add c - curr <- Some x - // Overlapping, join - | Some (End nextIndex as c), x -> - curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } - curr |> Option.iter buffer.Add - if buffer.Count = 0 then null else buffer.ToArray() - - let inline optionCombine f (r1: int64 option) (r2: int64 option) = - match r1, r2 with - | Some x, Some y -> f x y |> Some - | None, None -> None - | None, x | x, None -> x - let combine (s1: StreamState) (s2: StreamState) : StreamState = - let writePos = optionCombine max s1.write s2.write - let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 - { write = writePos; queue = Span.merge (defaultArg writePos 0L) items } - -/// Gathers stats relating to how many items of a given category have been observed -type CatStats() = - let cats = Dictionary() - member __.Ingest(cat,?weight) = - let weight = defaultArg weight 1L - match cats.TryGetValue cat with - | true, catCount -> cats.[cat] <- catCount + weight - | false, _ -> cats.[cat] <- weight - member __.Any = cats.Count <> 0 - member __.Clear() = cats.Clear() - member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd - -type StreamStates() = - let states = Dictionary() - let update stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - stream, state - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - stream, updated - let updateWritePos stream pos span = update stream { write = pos; queue = span } - let markCompleted stream index = updateWritePos stream (Some index) null |> ignore - let enqueue (item : StreamItem) = updateWritePos item.stream None [| { index = item.index; events = [| item.event |]}|] - - let busy = HashSet() - let schedule (requestedOrder : string seq) (capacity: int) = - let toSchedule = ResizeArray<_>(capacity) - let xs = requestedOrder.GetEnumerator() - while xs.MoveNext() && toSchedule.Capacity <> 0 do - let x = xs.Current - if busy.Add x then - let q = states.[x].queue - if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", x) - toSchedule.Add { stream = x; span = q.[0] } - toSchedule.ToArray() - let markNotBusy stream = - busy.Remove stream |> ignore - - //member __.Add(item: StreamBatch) = enqueue item - member __.Add(item: StreamItem) = enqueue item |> ignore - member __.TryGetStreamWritePos stream = - match states.TryGetValue stream with - | true, value -> value.write - | false, _ -> None - member __.QueueLength(stream) = - let q = states.[stream].queue - if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", stream) - q.[0].events.Length - member __.MarkCompleted(stream, index) = - markNotBusy stream - markCompleted stream index - member __.MarkFailed stream = - markNotBusy stream - member __.Schedule(requestedOrder : string seq, capacity: int) : StreamBatch[] = - schedule requestedOrder capacity - member __.Dump(log : ILogger) = - let mutable busyCount, busyB, ready, readyB, synced = 0, 0L, 0, 0L, 0 - let busyCats, readyCats, readyStreams = CatStats(), CatStats(), CatStats() - for KeyValue (stream,state) in states do - match int64 state.Size with - | 0L -> - synced <- synced + 1 - | sz when busy.Contains stream -> - busyCats.Ingest(category stream) - busyCount <- busyCount + 1 - busyB <- busyB + sz - | sz -> - readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) - ready <- ready + 1 - readyB <- readyB + sz - log.Information("Busy {busy}/{busyMb:n1}MB Ready {ready}/{readyMb:n1}MB Synced {synced}", busyCount, mb busyB, ready, mb readyB, synced) - if busyCats.Any then log.Information("Busy Categories, events {busyCats}", busyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - -type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } - -type ProgressState<'Pos>(?currentPos : 'Pos) = - let pending = Queue<_>() - let mutable validatedPos = currentPos - member __.AppendBatch(pos, reqs : Dictionary) = - pending.Enqueue { pos = pos; streamToRequiredIndex = reqs } - member __.MarkStreamProgress(stream, index) = - for x in pending do - match x.streamToRequiredIndex.TryGetValue stream with - | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore - | _, _ -> () - let headIsComplete () = - match pending.TryPeek() with - | true, batch -> batch.streamToRequiredIndex.Count = 0 - | _ -> false - let mutable completed = 0 - while headIsComplete () do - completed <- completed + 1 - let headBatch = pending.Dequeue() - validatedPos <- Some headBatch.pos - completed - member __.ScheduledOrder getStreamQueueLength = - let raw = seq { - let streams = HashSet() - let mutable batch = 0 - for x in pending do - batch <- batch + 1 - for s in x.streamToRequiredIndex.Keys do - if streams.Add s then - yield s,(batch,getStreamQueueLength s) } - raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst - member __.Validate tryGetStreamWritePos : 'Pos option * int = - let rec aux () = - match pending.TryPeek() with - | false, _ -> () - | true, batch -> - for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do - match tryGetStreamWritePos stream with - | Some index when requiredIndex <= index -> - Log.Warning("Validation had to remove {stream}", stream) - batch.streamToRequiredIndex.Remove stream |> ignore - | _ -> () - if batch.streamToRequiredIndex.Count = 0 then - let headBatch = pending.Dequeue() - validatedPos <- Some headBatch.pos - aux () - aux () - validatedPos, pending.Count - -/// Manages writing of progress -/// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) -/// - retries until success or a new item is posted -type ProgressWriter() = - let pumpSleepMs = 100 - let due = expiredMs 5000L - let mutable committedEpoch = None - let mutable validatedPos = None - let result = Event<_>() - [] member __.Result = result.Publish - member __.Post(version,f) = - Volatile.Write(&validatedPos,Some (version,f)) - member __.CommittedEpoch = Volatile.Read(&committedEpoch) - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match Volatile.Read &validatedPos with - | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> - try do! f - Volatile.Write(&committedEpoch, Some v) - result.Trigger (Choice1Of2 v) - with e -> result.Trigger (Choice2Of2 e) - | _ -> do! Async.Sleep pumpSleepMs } - -[] -type CoordinatorWork = - /// Enqueue a batch of items with supplied tag and progress marking function - | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq - /// Log stats about an ingested batch - | Added of streams: int * events: int - /// Result of processing on stream - processed up to nominated `index` or threw `exn` - | Result of stream: string * outcome: Choice - /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` - | ProgressResult of Choice - -type CoordinatorStats(log : ILogger, maxPendingBatches, ?statsInterval) = - let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.) - let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None - let progCommitFails, progCommits = ref 0, ref 0 - let cycles, batchesPended, streamsPended, eventsPended, resultOk, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (busy,capacity) (streams : StreamStates) = - if !progCommitFails <> 0 || !progCommits <> 0 then - match comittedEpoch with - | None -> - log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) - | Some committed when !progCommitFails <> 0 -> - log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) - | Some committed -> - log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) - progCommits := 0; progCommitFails := 0 - else - log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}e) Busy {busy}/{processors} Completed {completed} ({ok} ok {exns} exn)", - !cycles, !batchesPended, !streamsPended, !eventsPended, busy, capacity, !resultOk + !resultExn, !resultOk, !resultExn) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; resultOk := 0; resultExn:= 0 - //Metrics.dumpRuStats statsInterval log - streams.Dump log - member __.Handle = function - | Add _ -> () - | Added (streams, events) -> - incr batchesPended - eventsPended := !eventsPended + events - streamsPended := !streamsPended + streams - | Result (_stream, Choice1Of2 _) -> - incr resultOk - | Result (_stream, Choice2Of2 _) -> - incr resultExn - | ProgressResult (Choice1Of2 epoch) -> - incr progCommits - comittedEpoch <- Some epoch - | ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progCommitFails - member __.HandleValidated(epoch, pendingBatches) = - incr cycles - pendingBatchCount <- pendingBatches - validatedEpoch <- epoch - member __.HandleCommitted epoch = - comittedEpoch <- epoch - member __.TryDump(busy,capacity,streams) = - if statsDue () then - dumpStats (busy,capacity) streams - -/// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint -type Dispatcher(maxDop) = - let cancellationCheckInterval = TimeSpan.FromMilliseconds 5. - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) - let result = Event<_>() - let dop = new SemaphoreSlim(maxDop) - let dispatch work = async { - let! res = work - result.Trigger res - dop.Release() |> ignore } - [] member __.Result = result.Publish - member __.Capacity = dop.CurrentCount - member __.Enqueue item = work.Add item - member __.Pump () = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - let! got = dop.Await(cancellationCheckInterval) - if got then - let mutable item = Unchecked.defaultof> - if work.TryTake(&item, cancellationCheckInterval) then Async.Start(dispatch item) - else dop.Release() |> ignore } - -/// Single instance per ChangeFeedObserver, spun up as leases are won and allocated by the ChangeFeedProcessor hosting framework -/// Coordinates a) ingestion of events b) execution of projection work c) writing of progress d) reporting of state -type Coordinator(log : ILogger, maxPendingBatches, processorDop, ?statsInterval) = - let sleepIntervalMs = 5 - let cts = new CancellationTokenSource() - let batches = new SemaphoreSlim(maxPendingBatches) - let stats = CoordinatorStats(log, maxPendingBatches, ?statsInterval=statsInterval) - let progressWriter = ProgressWriter() - let progressState = ProgressState() - let streams = StreamStates() - let work = ConcurrentQueue<_>() - let handle = function - | Add (epoch, checkpoint,items) -> - let reqs = Dictionary() - let mutable count = 0 - for item in items do - streams.Add item - count <- count + 1 - reqs.[item.stream] <- item.index + 1L - progressState.AppendBatch((epoch,checkpoint),reqs) - work.Enqueue(Added (reqs.Count,count)) - | Added _ -> () - | Result (stream, Choice1Of2 index) -> - let batchesCompleted = progressState.MarkStreamProgress(stream,index) - if batchesCompleted <> 0 then batches.Release(batchesCompleted) |> ignore - streams.MarkCompleted(stream,index) - | Result (stream, Choice2Of2 _) -> - streams.MarkFailed stream - | ProgressResult _ -> () - - member private __.Pump(project : StreamBatch -> Async) = async { - let dispatcher = Dispatcher(processorDop) - use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) - use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) - Async.Start(progressWriter.Pump(), cts.Token) - Async.Start(dispatcher.Pump(), cts.Token) - while not cts.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let validatedPos, batches = progressState.Validate(streams.TryGetStreamWritePos) - stats.HandleValidated(Option.map fst validatedPos, batches) - validatedPos |> Option.iter progressWriter.Post - stats.HandleCommitted progressWriter.CommittedEpoch - // 3. After that, provision writers queue - let capacity = dispatcher.Capacity - if capacity <> 0 then - let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) - for batch in work do - dispatcher.Enqueue <| async { - try let! count = project batch - return batch.stream, Choice1Of2 (batch.span.index + int64 count) - with e -> return batch.stream, Choice2Of2 e } - // 4. Periodically emit status info - let busy = processorDop - dispatcher.Capacity - stats.TryDump(busy,processorDop,streams) - do! Async.Sleep sleepIntervalMs } - static member Start(rangeLog, maxPendingBatches, processorDop, project) = - let instance = new Coordinator(rangeLog, maxPendingBatches, processorDop) - Async.Start <| instance.Pump(project) - instance - member __.Submit(epoch, markBatchCompleted, events) = async { - let! _ = batches.Await() - Add (epoch, markBatchCompleted, Array.ofSeq events) |> work.Enqueue - return maxPendingBatches-batches.CurrentCount,maxPendingBatches } - - member __.Stop() = - cts.Cancel() \ No newline at end of file diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index 5e7967ea3..18288cf3a 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -12,7 +12,17 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consumer.fsproj", "{7ED94D2B-1744-48A0-9B20-94E4777617E9}" EndProject -Project("{F2A71F9B-5D33-465A-A702-920D77279786}") = "Projector.Tests", "Projector.Tests\Projector.Tests.fsproj", "{964E1EA5-9A40-422D-9673-DE169E6D49EE}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection", "Equinox.Projection\Equinox.Projection.fsproj", "{C0EDE411-0BF1-417E-8B17-806F55BE01FE}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Tests", "Equinox.Projection.Tests\Equinox.Projection.Tests.fsproj", "{D8C4B963-1415-4711-A585-80BA2BFC9010}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "..\equinox-sync\Sync\Sync.fsproj", "{C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "..\equinox-sync\Ingest\Ingest.fsproj", "{260FCBA4-C948-4D4E-92E1-B679075F6890}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Codec", "Equinox.Projection.Codec\Equinox.Projection.Codec.fsproj", "{AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Cosmos", "Equinox.Projection.Cosmos\Equinox.Projection.Cosmos.fsproj", "{2071A2C9-B5C8-4143-B437-6833666D0ACA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -28,10 +38,30 @@ Global {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.Build.0 = Debug|Any CPU {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.Build.0 = Release|Any CPU - {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {964E1EA5-9A40-422D-9673-DE169E6D49EE}.Release|Any CPU.Build.0 = Release|Any CPU + {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Release|Any CPU.Build.0 = Release|Any CPU + {D8C4B963-1415-4711-A585-80BA2BFC9010}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {D8C4B963-1415-4711-A585-80BA2BFC9010}.Debug|Any CPU.Build.0 = Debug|Any CPU + {D8C4B963-1415-4711-A585-80BA2BFC9010}.Release|Any CPU.ActiveCfg = Release|Any CPU + {D8C4B963-1415-4711-A585-80BA2BFC9010}.Release|Any CPU.Build.0 = Release|Any CPU + {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.Build.0 = Debug|Any CPU + {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.ActiveCfg = Release|Any CPU + {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.Build.0 = Release|Any CPU + {260FCBA4-C948-4D4E-92E1-B679075F6890}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {260FCBA4-C948-4D4E-92E1-B679075F6890}.Debug|Any CPU.Build.0 = Debug|Any CPU + {260FCBA4-C948-4D4E-92E1-B679075F6890}.Release|Any CPU.ActiveCfg = Release|Any CPU + {260FCBA4-C948-4D4E-92E1-B679075F6890}.Release|Any CPU.Build.0 = Release|Any CPU + {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Release|Any CPU.Build.0 = Release|Any CPU + {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 79bff3715..93aed518a 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -196,7 +196,7 @@ type Readers(conn, spec : ReaderSpec, tryMapEvent, postBatch, max : EventStore.C work.OverallStats.DumpIfIntervalExpired() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - try let! eof = work.Process(conn, tryMapEvent, postBatch, (fun _ -> true), (fun _pos -> Seq.iter postBatch), task) + try let! eof = work.Process(conn, tryMapEvent, postBatch, (fun _pos -> Seq.iter postBatch), task) if eof then remainder <- None finally dop.Release() |> ignore } return () } diff --git a/equinox-sync/Sync/Checkpoint.fs b/equinox-sync/Sync/Checkpoint.fs index 8a703df01..49d1a801d 100644 --- a/equinox-sync/Sync/Checkpoint.fs +++ b/equinox-sync/Sync/Checkpoint.fs @@ -107,24 +107,4 @@ type CheckpointSeries(name, log, resolveStream) = member __.Read = inner.Read seriesId member __.Start(freq, pos) = inner.Start(seriesId, freq, pos) member __.Override(freq, pos) = inner.Override(seriesId, freq, pos) - member __.Commit(pos) = inner.Commit(seriesId, pos) - -/// Manages writing of progress -/// - Each write attempt is of the newest token -/// - retries until success or a new item is posted -type ProgressWriter(sync, ?interval) = - let pumpSleepMs = let interval = defaultArg interval TimeSpan.FromSeconds 5. in interval.TotalMilliseconds |> int - let mutable latest = None - let result = Event>() - [] member __.Result = result.Publish - member __.Post(value : int64) = latest <- Some value - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match latest with - | Some value -> - try do! sync value - result.Trigger (Choice1Of2 value) - with e -> result.Trigger (Choice2Of2 e) - | _ -> () - do! Async.Sleep pumpSleepMs } \ No newline at end of file + member __.Commit(pos) = inner.Commit(seriesId, pos) \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs deleted file mode 100644 index 3cf041f99..000000000 --- a/equinox-sync/Sync/CosmosIngester.fs +++ /dev/null @@ -1,292 +0,0 @@ -module SyncTemplate.CosmosIngester - -open Equinox.Cosmos.Core -open Equinox.Cosmos.Store -open Serilog -open System.Collections.Concurrent -open System.Collections.Generic -open System.Threading - -let arrayBytes (x:byte[]) = if x = null then 0 else x.Length -let private mb x = float x / 1024. / 1024. - -let category (streamName : string) = streamName.Split([|'-'|],2).[0] - -let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 - -type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } -type [] Batch = { stream: string; span: Span } - -module Writer = - type [] Result = - | Ok of stream: string * updatedPos: int64 - | Duplicate of stream: string * updatedPos: int64 - | PartialDuplicate of overage: Batch - | PrefixMissing of batch: Batch * writePos: int64 - | Exn of exn: exn * batch: Batch - member __.WriteTo(log: ILogger) = - match __ with - | Ok (stream, pos) -> log.Information("Wrote {stream} up to {pos}", stream, pos) - | Duplicate (stream, pos) -> log.Information("Ignored {stream} (synced up to {pos})", stream, pos) - | PartialDuplicate overage -> log.Information("Requeing {stream} {pos} ({count} events)", - overage.stream, overage.span.index, overage.span.events.Length) - | PrefixMissing (batch,pos) -> log.Information("Waiting {stream} missing {gap} events ({count} events @ {pos})", - batch.stream, batch.span.index-pos, batch.span.events.Length, batch.span.index) - | Exn (exn, batch) -> log.Warning(exn,"Writing {stream} failed, retrying {count} events ....", batch.stream, batch.span.events.Length) - - let write (log : ILogger) (ctx : CosmosContext) ({ stream = s; span = { index = i; events = e}} as batch) = async { - let stream = ctx.CreateStream s - log.Debug("Writing {s}@{i}x{n}",s,i,e.Length) - try let! res = ctx.Sync(stream, { index = i; etag = None }, e) - let ress = - match res with - | AppendResult.Ok pos -> Ok (s, pos.index) - | AppendResult.Conflict (pos, _) | AppendResult.ConflictUnknown pos -> - match pos.index with - | actual when actual < i -> PrefixMissing (batch, actual) - | actual when actual >= i + e.LongLength -> Duplicate (s, actual) - | actual -> PartialDuplicate { stream = s; span = { index = actual; events = e |> Array.skip (actual-i |> int) } } - log.Debug("Result: {res}",ress) - return ress - with e -> return Exn (e, batch) } - let (|TimedOutMessage|RateLimitedMessage|TooLargeMessage|MalformedMessage|Other|) (e: exn) = - match string e with - | m when m.Contains "Microsoft.Azure.Documents.RequestTimeoutException" -> TimedOutMessage - | m when m.Contains "Microsoft.Azure.Documents.RequestRateTooLargeException" -> RateLimitedMessage - | m when m.Contains "Microsoft.Azure.Documents.RequestEntityTooLargeException" -> TooLargeMessage - | m when m.Contains "SyntaxError: JSON.parse Error: Unexpected input at position" - || m.Contains "SyntaxError: JSON.parse Error: Invalid character at position" -> MalformedMessage - | _ -> Other - -type [] StreamState = { isMalformed : bool; write: int64 option; queue: Span[] } with - member __.TryGap() = - if __.queue = null then None - else - match __.write, Array.tryHead __.queue with - | Some w, Some { index = i } when i > w -> Some (w, i-w) - | _ -> None - member __.IsReady = - if __.queue = null || __.isMalformed then false - else - match __.write, Array.tryHead __.queue with - | Some w, Some { index = i } -> i = w - | None, _ -> true - | _ -> false - member __.Size = - if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) - -module StreamState = - let (|NNA|) xs = if xs = null then Array.empty else xs - module Span = - let (|End|) x = x.index + if x.events = null then 0L else x.events.LongLength - let trim min = function - | x when x.index >= min -> x // don't adjust if min not within - | End n when n < min -> { index = min; events = [||] } // throw away if before min - | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice - let merge min (xs : Span seq) = - let xs = - seq { for x in xs -> { x with events = (|NNA|) x.events } } - |> Seq.map (trim min) - |> Seq.filter (fun x -> x.events.Length <> 0) - |> Seq.sortBy (fun x -> x.index) - let buffer = ResizeArray() - let mutable curr = None - for x in xs do - match curr, x with - // Not overlapping, no data buffered -> buffer - | None, _ -> - curr <- Some x - // Gap - | Some (End nextIndex as c), x when x.index > nextIndex -> - buffer.Add c - curr <- Some x - // Overlapping, join - | Some (End nextIndex as c), x -> - curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } - curr |> Option.iter buffer.Add - if buffer.Count = 0 then null else buffer.ToArray() - - let inline optionCombine f (r1: int64 option) (r2: int64 option) = - match r1, r2 with - | Some x, Some y -> f x y |> Some - | None, None -> None - | None, x | x, None -> x - let combine (s1: StreamState) (s2: StreamState) : StreamState = - let writePos = optionCombine max s1.write s2.write - let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 - { write = writePos; queue = Span.merge (defaultArg writePos 0L) items; isMalformed = s1.isMalformed || s2.isMalformed} - -/// Gathers stats relating to how many items of a given category have been observed -type CatStats() = - let cats = Dictionary() - member __.Ingest(cat,?weight) = - let weight = defaultArg weight 1L - match cats.TryGetValue cat with - | true, catCount -> cats.[cat] <- catCount + weight - | false, _ -> cats.[cat] <- weight - member __.Any = cats.Count <> 0 - member __.Clear() = cats.Clear() - member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd - -type ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Ok - -type StreamStates() = - let states = Dictionary() - let dirty = Queue() - let markDirty stream = if (not << dirty.Contains) stream then dirty.Enqueue stream - let gap = Queue() - let markGap stream = if (not << gap.Contains) stream then gap.Enqueue stream - - let update stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - markDirty stream - stream, state - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - if updated.IsReady then - markDirty stream - stream, updated - let updateWritePos stream pos isMalformed span = - update stream { write = pos; queue = span; isMalformed = isMalformed } - - member __.Add(item: Batch, ?isMalformed) = updateWritePos item.stream None (defaultArg isMalformed false) [|item.span|] - member __.TryGetStreamWritePos stream = match states.TryGetValue stream with true, value -> value.write | _ -> None - member __.HandleWriteResult = function - | Writer.Result.Ok (stream, pos) -> updateWritePos stream (Some pos) false null, Ok - | Writer.Result.Duplicate (stream, pos) -> updateWritePos stream (Some pos) false null, Ok - | Writer.Result.PartialDuplicate overage -> updateWritePos overage.stream (Some overage.span.index) false [|overage.span|], Ok - | Writer.Result.PrefixMissing (overage,pos) -> - markGap overage.stream - updateWritePos overage.stream (Some pos) false [|overage.span|], Ok - | Writer.Result.Exn (exn, batch) -> - let r, malformed = - match exn with - | Writer.RateLimitedMessage -> RateLimited, false - | Writer.TimedOutMessage -> TimedOut, false - | Writer.TooLargeMessage -> TooLarge, true - | Writer.MalformedMessage -> Malformed, true - | Writer.Other -> Ok, false - __.Add(batch, malformed), r - member __.TryGap() : (string*int64*int) option = - let rec aux () = - match gap |> Queue.tryDequeue with - | None -> None - | Some stream -> - - match states.[stream].TryGap() with - | Some (pos,count) -> Some (stream,pos,int count) - | None -> aux () - aux () - member __.TryReady(isBusy) = - let blocked = ResizeArray() - let rec aux () = - match dirty |> Queue.tryDequeue with - | None -> None - | Some stream -> - - match states.[stream] with - | state when state.IsReady -> - if (not << isBusy) stream then - let h = state.queue |> Array.head - - let mutable bytesBudget = cosmosPayloadLimit - let mutable count = 0 - let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = - bytesBudget <- bytesBudget - cosmosPayloadBytes y - count <- count + 1 - // Reduce the item count when we don't yet know the write position - count <= (if Option.isNone state.write then 10 else 4096) && (bytesBudget >= 0 || count = 1) - Some { stream = stream; span = { index = h.index; events = h.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } - else - blocked.Add(stream) |> ignore - aux () - | _ -> aux () - let res = aux () - for x in blocked do markDirty x - res - member __.Dump(log : ILogger) = - let mutable synced, ready, waiting, malformed = 0, 0, 0, 0 - let mutable readyB, waitingB, malformedB = 0L, 0L, 0L - let waitCats, readyCats, readyStreams = CatStats(), CatStats(), CatStats() - for KeyValue (stream,state) in states do - match int64 state.Size with - | 0L -> - synced <- synced + 1 - | sz when state.isMalformed -> - malformed <- malformed + 1 - malformedB <- malformedB + sz - | sz when state.IsReady -> - readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) - ready <- ready + 1 - readyB <- readyB + sz - | sz -> - waitCats.Ingest(category stream) - waiting <- waiting + 1 - waitingB <- waitingB + sz - log.Information("Ready {ready}/{readyMb:n1}MB Dirty {dirty} Awaiting prefix {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", - ready, mb readyB, dirty.Count, waiting, mb waitingB, malformed, mb malformedB, synced) - if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - if waitCats.Any then log.Warning("Waiting {waitCats}", waitCats.StatsDescending) - -type RefCounted<'T> = { mutable refCount: int; value: 'T } - -// via https://stackoverflow.com/a/31194647/11635 -type SemaphorePool(gen : unit -> SemaphoreSlim) = - let inners: Dictionary> = Dictionary() - - let getOrCreateSlot key = - lock inners <| fun () -> - match inners.TryGetValue key with - | true, inner -> - inner.refCount <- inner.refCount + 1 - inner.value - | false, _ -> - let value = gen () - inners.[key] <- { refCount = 1; value = value } - value - let slotReleaseGuard key : System.IDisposable = - { new System.IDisposable with - member __.Dispose() = - lock inners <| fun () -> - let item = inners.[key] - match item.refCount with - | 1 -> inners.Remove key |> ignore - | current -> item.refCount <- current - 1 } - - member __.ExecuteAsync(k,f) = async { - let x = getOrCreateSlot k - use _ = slotReleaseGuard k - return! f x } - - member __.Execute(k,f) = - let x = getOrCreateSlot k - use _l = slotReleaseGuard k - f x - - type Writers(write, maxDop) = - let work = ConcurrentQueue() - let result = Event() - let locks = SemaphorePool(fun () -> new SemaphoreSlim 1) - [] member __.Result = result.Publish - member __.Enqueue item = work.Enqueue item - member __.HasCapacity = work.Count < maxDop - member __.IsStreamBusy stream = - let checkBusy (x : SemaphoreSlim) = x.CurrentCount = 0 - locks.Execute(stream,checkBusy) - member __.Pump() = async { - let dop = new SemaphoreSlim(maxDop) - let dispatch item = async { let! res = write item in result.Trigger res } |> dop.Throttle - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match work.TryDequeue() with - | true, item -> - let holdStreamWhileDispatching (streamLock : SemaphoreSlim) = async { do! dispatch item |> streamLock.Throttle } - do! locks.ExecuteAsync(item.stream,holdStreamWhileDispatching) - | _ -> do! Async.Sleep 100 } \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index bdde06a4b..1ba3e9a2a 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -1,6 +1,8 @@ module SyncTemplate.EventStoreSource open Equinox.Store // AwaitTaskCorrect +open Equinox.Projection.Cosmos +open Equinox.Projection.State open EventStore.ClientAPI open System open Serilog // NB Needs to shadow ILogger @@ -11,10 +13,10 @@ open System.Collections.Generic type EventStore.ClientAPI.RecordedEvent with member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) -let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = CosmosIngester.arrayBytes x.Data + CosmosIngester.arrayBytes x.Metadata +let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 -let tryToBatch (e : RecordedEvent) : CosmosIngester.Batch option = +let tryToBatch (e : RecordedEvent) : StreamItem option = let eb = recPayloadBytes e if eb > CosmosIngester.cosmosPayloadLimit then Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", @@ -24,7 +26,7 @@ let tryToBatch (e : RecordedEvent) : CosmosIngester.Batch option = let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ - Some { stream = e.EventStreamId; span = { index = e.EventNumber; events = [| event |]} } + Some { stream = e.EventStreamId; index = e.EventNumber; event = event} let private mb x = float x / 1024. / 1024. @@ -118,7 +120,7 @@ let establishMax (conn : IEventStoreConnection) = async { Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 return Option.get max } -let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : CosmosIngester.Batch -> unit) = +let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : StreamSpan -> unit) = let rec fetchFrom pos limit = async { let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect @@ -136,7 +138,7 @@ let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int type [] PullResult = Exn of exn: exn | Eof | EndOfTranche let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) - (range:Range, once) (tryMapEvent : ResolvedEvent -> CosmosIngester.Batch option) (postBatch : Position -> CosmosIngester.Batch[] -> unit) = + (range:Range, once) (tryMapEvent : ResolvedEvent -> StreamItem option) (postBatch : Position -> StreamItem[] -> Async) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec aux () = async { let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -146,10 +148,10 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let batches = currentSlice.Events |> Seq.choose tryMapEvent |> Array.ofSeq let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms", + let! (cur,max) = postBatch currentSlice.NextPosition batches + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms {cur}/{max}", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds) + batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds, cur, max) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else sw.Restart() // restart the clock as we hand off back to the Reader return! aux () } @@ -181,7 +183,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = work.Enqueue <| Work.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() - member __.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, work) = async { + member __.Process(conn, tryMapEvent, postItem, postBatch, work) = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize match work with | StreamPrefix (name,pos,len,batchSize) -> @@ -236,7 +238,6 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let slicesStats, stats = SliceStatsBuffer(), OverallStats() use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let progressSw = Stopwatch.StartNew() - let mutable paused = false while true do let currentPos = range.Current if progressSw.ElapsedMilliseconds > progressIntervalMs then @@ -244,19 +245,12 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = count, pauses, currentPos.CommitPosition, chunk currentPos) progressSw.Restart() count <- count + 1 - if shouldTail () then - paused <- false - let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch - do! awaitInterval - match res with - | PullResult.EndOfTranche | PullResult.Eof -> () - | PullResult.Exn e -> - batchSize <- adjust batchSize - Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) - else - if not paused then Log.Information("Pausing due to backlog of incomplete batches...") - paused <- true - pauses <- pauses + 1 - do! awaitInterval + let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch + do! awaitInterval + match res with + | PullResult.EndOfTranche | PullResult.Eof -> () + | PullResult.Exn e -> + batchSize <- adjust batchSize + Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) stats.DumpIfIntervalExpired() return true } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 4a10e987b..fa8e88fcd 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -7,6 +7,9 @@ open Equinox.Cosmos.Projection //#if eventStore open Equinox.EventStore //#endif +open Equinox.Projection.Coordination +open Equinox.Projection.Cosmos +open Equinox.Projection.State //#if !eventStore open Equinox.Store //#endif @@ -298,98 +301,23 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -module Metrics = - module RuCounters = - open Equinox.Cosmos.Store - open Serilog.Events - - let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds - - let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function - | Log.Tip (Stats s) - | Log.TipNotFound (Stats s) - | Log.TipNotModified (Stats s) - | Log.Query (_,_, (Stats s)) -> CosmosReadRc s - // slices are rolled up into batches so be sure not to double-count - | Log.Response (_,(Stats s)) -> CosmosResponseRc s - | Log.SyncSuccess (Stats s) - | Log.SyncConflict (Stats s) -> CosmosWriteRc s - | Log.SyncResync (Stats s) -> CosmosResyncRc s - let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function - | (:? ScalarValue as x) -> Some x.Value - | _ -> None - let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = - match logEvent.Properties.TryGetValue("cosmosEvt") with - | true, SerilogScalar (:? Log.Event as e) -> Some e - | _ -> None - type RuCounter = - { mutable rux100: int64; mutable count: int64; mutable ms: int64 } - static member Create() = { rux100 = 0L; count = 0L; ms = 0L } - member __.Ingest (ru, ms) = - System.Threading.Interlocked.Increment(&__.count) |> ignore - System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore - System.Threading.Interlocked.Add(&__.ms, ms) |> ignore - type RuCounterSink() = - static member val Read = RuCounter.Create() with get, set - static member val Write = RuCounter.Create() with get, set - static member val Resync = RuCounter.Create() with get, set - static member Reset() = - RuCounterSink.Read <- RuCounter.Create() - RuCounterSink.Write <- RuCounter.Create() - RuCounterSink.Resync <- RuCounter.Create() - interface Serilog.Core.ILogEventSink with - member __.Emit logEvent = logEvent |> function - | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats - | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats - | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats - | _ -> () - - let dumpRuStats duration (log: Serilog.ILogger) = - let stats = - [ "Read", RuCounters.RuCounterSink.Read - "Write", RuCounters.RuCounterSink.Write - "Resync", RuCounters.RuCounterSink.Resync ] - let mutable totalCount, totalRc, totalMs = 0L, 0., 0L - let logActivity name count rc lat = - if count <> 0L then - log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", - name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) - for name, stat in stats do - let ru = float stat.rux100 / 100. - totalCount <- totalCount + stat.count - totalRc <- totalRc + ru - totalMs <- totalMs + stat.ms - logActivity name stat.count ru stat.ms - logActivity "TOTAL" totalCount totalRc totalMs - // Yes, there's a minor race here! - RuCounters.RuCounterSink.Reset() - let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] - let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) - for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) - #if !cosmos module EventStoreSource = - type [] CoordinationWork<'Pos> = - | Result of CosmosIngester.Writer.Result - | ProgressResult of Choice - | Unbatched of CosmosIngester.Batch - | BatchWithTracking of 'Pos * CosmosIngester.Batch[] - - type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> CosmosIngester.Batch option, maxDop, ?statsInterval) = + type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = let sleepIntervalMs = 100 let dop = new SemaphoreSlim(maxDop) let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) member __.HasCapacity = work.QueueCount < dop.CurrentCount member __.AddTail(startPos, max, interval) = work.AddTail(startPos, max, interval) member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) - member __.Pump(postItem, shouldTail, postBatch) = async { + member __.Pump(postItem, postBatch) = async { let! ct = Async.CancellationToken while not ct.IsCancellationRequested do work.OverallStats.DumpIfIntervalExpired() let! _ = dop.Await() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, tryMapEvent, postItem, shouldTail, postBatch, task) in () + try let! _ = work.Process(conn, tryMapEvent, postItem, postBatch, task) in () finally dop.Release() |> ignore } return () } match work.TryDequeue() with @@ -399,146 +327,68 @@ module EventStoreSource = dop.Release() |> ignore do! Async.Sleep sleepIntervalMs } + open CosmosIngester type StartMode = Starting | Resuming | Overridding - type Coordinator(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, progressWriter: Checkpoint.ProgressWriter, maxPendingBatches, ?interval) = + type Syncer(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, maxPendingBatches, commitProgress, ?interval) = let statsInterval = defaultArg interval (TimeSpan.FromMinutes 1.) - let statsIntervalMs = int64 statsInterval.TotalMilliseconds - let sleepIntervalMs = 1 - let input = new System.Collections.Concurrent.BlockingCollection<_>(System.Collections.Concurrent.ConcurrentQueue(), maxPendingBatches) - let results = System.Collections.Concurrent.ConcurrentQueue() - let buffer = CosmosIngester.StreamStates() - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) - let tailSyncState = ProgressBatcher.State() - // Yes, there is a race, but its constrained by the number of parallel readers and the fact that batches get ingested quickly here - let mutable pendingBatchCount = 0 - let shouldThrottle () = pendingBatchCount > maxPendingBatches - let mutable validatedEpoch, comittedEpoch : int64 option * int64 option = None, None - let pumpReaders = - let postWrite = (*we want to prioritize processing of catchup stream reads*) results.Enqueue << CoordinationWork.Unbatched - let postBatch pos xs = input.Add (CoordinationWork.BatchWithTracking (pos,xs)) - readers.Pump(postWrite, not << shouldThrottle, postBatch) - let postWriteResult = results.Enqueue << CoordinationWork.Result - let postProgressResult = results.Enqueue << CoordinationWork.ProgressResult member __.Pump() = async { - use _ = writers.Result.Subscribe postWriteResult - use _ = progressWriter.Result.Subscribe postProgressResult - let! _ = Async.StartChild pumpReaders - let! _ = Async.StartChild <| writers.Pump() - let! _ = Async.StartChild <| progressWriter.Pump() - let! ct = Async.CancellationToken let writerResultLog = log.ForContext() - let mutable bytesPended, bytesPendedAgg = 0L, 0L - let workPended, eventsPended, cycles = ref 0, ref 0, ref 0 - let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 - let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 - let progCommitFails, progCommits = ref 0, ref 0 - let badCats = CosmosIngester.CatStats() - let dumpStats () = - if !progCommitFails <> 0 || !progCommits <> 0 then - match comittedEpoch with + let trim (writePos : int64 option, batch : StreamSpan) = + let mutable bytesBudget = cosmosPayloadLimit + let mutable count = 0 + let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = + bytesBudget <- bytesBudget - cosmosPayloadBytes y + count <- count + 1 + // Reduce the item count when we don't yet know the write position + count <= (if Option.isNone writePos then 10 else 4096) && (bytesBudget >= 0 || count = 1) + { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } + let project batch = async { + let trimmed = trim batch + try let! res = Writer.write log cosmosContext trimmed + return trimmed.stream, Choice1Of2 res + with e -> return trimmed.stream, Choice2Of2 e } + let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = + let applyResultToStreamState = function + | stream, (Choice1Of2 (Writer.Ok pos)) -> + streams.InternalUpdate stream pos null, ResultKind.Ok + | stream, (Choice1Of2 (Writer.Duplicate pos)) -> + streams.InternalUpdate stream pos null, ResultKind.Ok + | stream, (Choice1Of2 (Writer.PartialDuplicate overage)) -> + streams.InternalUpdate stream overage.index [|overage|], ResultKind.Ok + | stream, (Choice1Of2 (Writer.PrefixMissing (overage,pos))) -> + streams.InternalUpdate stream pos [|overage|], ResultKind.Ok + | stream, (Choice2Of2 exn) -> + let kind, malformed = Writer.classify exn + streams.SetMalformed(stream,malformed), kind + match res with + | Message.Result (s,r) -> + // TODO kind ? + let (stream,updatedState), kind = applyResultToStreamState (s,r) + match updatedState.write with + | Some wp -> + let closedBatches = progressState.MarkStreamProgress(stream, wp) + if closedBatches > 0 then + batches.Release(closedBatches) |> ignore + streams.MarkCompleted(stream,wp) | None -> - log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncomitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) - | Some committed when !progCommitFails <> 0 -> - log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncomitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) - | Some committed -> - log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncomitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) - progCommits := 0; progCommitFails := 0 - else - log.Information("Progress @ {validated} (committed: {committed}) Uncomitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Cycles {cycles} Queued {queued} reqs {events} events {mb:n}MB ∑{gb:n3}GB", - !cycles, !workPended, !eventsPended, mb bytesPended, mb bytesPendedAgg / 1024.) - cycles := 0; workPended := 0; eventsPended := 0; bytesPended <- 0L - Log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; - if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - Log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", - !rateLimited, !timedOut, !tooLarge, !malformed) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 - if badCats.Any then Log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - Metrics.dumpRuStats statsInterval log - buffer.Dump log - let tryDumpStats = every statsIntervalMs dumpStats - let handle = function - | CoordinationWork.Unbatched item -> - buffer.Add item |> ignore - | CoordinationWork.BatchWithTracking(pos, items) -> - for item in items do - buffer.Add item |> ignore - tailSyncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) - | CoordinationWork.Result res -> - match res with - | CosmosIngester.Writer.Result.Ok _ -> incr resultOk - | CosmosIngester.Writer.Result.Duplicate _ -> incr resultDup - | CosmosIngester.Writer.Result.PartialDuplicate _ -> incr resultPartialDup - | CosmosIngester.Writer.Result.PrefixMissing _ -> incr resultPrefix - | CosmosIngester.Writer.Result.Exn _ -> incr resultExn - - let (stream, updatedState), kind = buffer.HandleWriteResult res - match updatedState.write with None -> () | Some wp -> tailSyncState.MarkStreamProgress(stream, wp) - match kind with - | CosmosIngester.Ok -> res.WriteTo writerResultLog - | CosmosIngester.RateLimited -> incr rateLimited - | CosmosIngester.TooLarge -> category stream |> badCats.Ingest; incr tooLarge - | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed - | CosmosIngester.TimedOut -> incr timedOut - | CoordinationWork.ProgressResult (Choice1Of2 epoch) -> - incr progCommits - comittedEpoch <- Some epoch - | CoordinationWork.ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progCommitFails - let queueWrite (w : CosmosIngester.Batch) = - incr workPended - eventsPended := !eventsPended + w.span.events.Length - bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) - writers.Enqueue w - while not ct.IsCancellationRequested do - incr cycles - // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - let mutable more, gotWork = true, false - while more do - match results.TryDequeue() with - | true, item -> handle item; gotWork <- true - | false, _ -> more <- false - // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let (_validatedPos, _pendingBatchCount) = tailSyncState.Validate buffer.TryGetStreamWritePos - pendingBatchCount <- _pendingBatchCount - validatedEpoch <- _validatedPos |> Option.map (fun x -> x.CommitPosition) - // 3. Feed latest position to store - validatedEpoch |> Option.iter progressWriter.Post - // 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - let mutable more = true - while more && readers.HasCapacity do - match buffer.TryGap() with - | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - | None -> more <- false - // 5. After that, [over] provision writers queue - let mutable more = true - while more && writers.HasCapacity do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> queueWrite w; gotWork <- true - | None -> (); more <- false - // 6. OK, we've stashed and cleaned work; now take some inputs - let x = Stopwatch.StartNew() - let mutable more = true - while more && x.ElapsedMilliseconds < int64 sleepIntervalMs do - match input.TryTake() with - | true, item -> handle item - | false, _ -> more <- false - match sleepIntervalMs - int x.ElapsedMilliseconds with - | d when d > 0 -> - if writers.HasCapacity || gotWork then do! Async.Sleep 1 - else do! Async.Sleep d + streams.MarkFailed stream + Writer.logTo writerResultLog (s,r) | _ -> () - // 7. Periodically emit status info - tryDumpStats () } - + let coordinator = Coordinator.Start(log, maxPendingBatches, maxWriters, project, handleResult, statsInterval) + let pumpReaders = + let postStreamSpan : StreamSpan -> unit = coordinator.Submit + let postBatch (pos : EventStore.ClientAPI.Position) xs = + let cp = pos.CommitPosition + coordinator.Submit(cp, commitProgress cp, Seq.ofArray xs) + readers.Pump(postStreamSpan, postBatch) + do! pumpReaders } + + //// 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + //let mutable more = true + //while more && readers.HasCapacity do + // match buffer.TryGap() with + // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + // | None -> more <- false static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn @@ -564,14 +414,13 @@ module EventStoreSource = do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) return Overridding, r, spec.checkpointInterval } - Log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m", + log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m", startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) return startPos } let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) readers.AddTail(startPos, max, spec.tailInterval) - let progress = Checkpoint.ProgressWriter(checkpoints.Commit) - let coordinator = Coordinator(log, readers, cosmosContext, maxWriters, progress, maxPendingBatches=maxPendingBatches) + let coordinator = Syncer(log, readers, cosmosContext, maxWriters, maxPendingBatches, checkpoints.Commit) do! coordinator.Pump() } #else module CosmosSource = @@ -829,7 +678,7 @@ module Logging = let cfpLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpLevel) |> fun c -> let ingesterLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information - c.MinimumLevel.Override(typeof.FullName, ingesterLevel) + c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning c.MinimumLevel.Override(typeof.FullName, generalLevel) @@ -847,7 +696,7 @@ module Logging = c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() - Log.ForContext(), Log.ForContext() + Log.ForContext(), Log.ForContext() [] let main argv = @@ -896,7 +745,7 @@ let main argv = || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e - EventStoreSource.Coordinator.Run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (args.MaxWriters, target, args.MaxPendingBatches) resolveCheckpointStream + EventStoreSource.Syncer.Run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (args.MaxWriters, target, args.MaxPendingBatches) resolveCheckpointStream #endif |> Async.RunSynchronously 0 diff --git a/equinox-sync/Sync/ProgressBatcher.fs b/equinox-sync/Sync/ProgressBatcher.fs deleted file mode 100644 index 4a5b6799b..000000000 --- a/equinox-sync/Sync/ProgressBatcher.fs +++ /dev/null @@ -1,39 +0,0 @@ -module SyncTemplate.ProgressBatcher - -open System.Collections.Generic - -type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } - -type State<'Pos>(?currentPos : 'Pos) = - let pending = Queue<_>() - let mutable validatedPos = currentPos - member __.AppendBatch(pos, streamWithRequiredIndices : (string * int64) seq) = - let byStream = streamWithRequiredIndices |> Seq.groupBy fst |> Seq.map (fun (s,xs) -> KeyValuePair(s,xs |> Seq.map snd |> Seq.max)) - pending.Enqueue { pos = pos; streamToRequiredIndex = Dictionary byStream } - member __.MarkStreamProgress(stream, index) = - for x in pending do - match x.streamToRequiredIndex.TryGetValue stream with - | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore - | _, _ -> () - let headIsComplete () = - match pending.TryPeek() with - | true, batch -> batch.streamToRequiredIndex.Count = 0 - | _ -> false - while headIsComplete () do - let headBatch = pending.Dequeue() - validatedPos <- Some headBatch.pos - member __.Validate tryGetStreamWritePos : 'Pos option * int = - let rec aux () = - match pending.TryPeek() with - | false, _ -> () - | true, batch -> - for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do - match tryGetStreamWritePos stream with - | Some index when requiredIndex <= index -> batch.streamToRequiredIndex.Remove stream |> ignore - | _ -> () - if batch.streamToRequiredIndex.Count = 0 then - let headBatch = pending.Dequeue() - validatedPos <- Some headBatch.pos - aux () - aux () - validatedPos, pending.Count \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 30aba3fcf..08e793009 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -8,18 +8,16 @@ - - - + - + @@ -27,4 +25,9 @@ + + + + + \ No newline at end of file From adc6ec0c32860622baa66324509e32ad1fdeda64 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 11:38:53 +0100 Subject: [PATCH 137/353] Filter log to console --- equinox-sync/Sync/Program.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index fa8e88fcd..dd793c856 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -689,8 +689,9 @@ module Logging = l.WriteTo.Sink(Metrics.RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke + let isWriter = Filters.Matching.FromSource().Invoke let isCheckpointing = Filters.Matching.FromSource().Invoke - (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x)) + (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x || isWriter x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) From ab4b7563c09610c79bcba1b91a179b79b69b2088 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 11:43:44 +0100 Subject: [PATCH 138/353] Fix malformed log --- equinox-projector/Equinox.Projection/State.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 5abbc5f2c..38ef1d78c 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -171,7 +171,7 @@ type StreamStates() = ready <- ready + 1 readyB <- readyB + sz log.Information("Busy {busy}/{busyMb:n1}MB Ready {ready}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", - busyCount, mb busyB, ready, mb readyB, malformed, mb, malformedB, synced) + busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB, synced) if busyCats.Any then log.Information("Busy Categories, events {busyCats}", busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) From ecaeaa368ca894adca5ffccba37327b95e2e73e3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 12:01:22 +0100 Subject: [PATCH 139/353] Limit batches --- equinox-projector/Equinox.Projection/State.fs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 38ef1d78c..e4d3d6118 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -117,13 +117,15 @@ type StreamStates() = let schedule (requestedOrder : string seq) (capacity: int) = let toSchedule = ResizeArray<_>(capacity) let xs = requestedOrder.GetEnumerator() - while xs.MoveNext() && toSchedule.Capacity <> 0 do + let mutable remaining = capacity + while xs.MoveNext() && remaining <> 0 do let x = xs.Current let state = states.[x] if not state.isMalformed && busy.Add x then let q = state.queue if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", x) toSchedule.Add(state.write, { stream = x; span = q.[0] }) + remaining <- remaining - 1 toSchedule.ToArray() let markNotBusy stream = busy.Remove stream |> ignore From 97f54cd3bbf508d5137d53672f181db73ce58756 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 12:47:10 +0100 Subject: [PATCH 140/353] Use CosmosStats --- .../Equinox.Projection.Cosmos/CosmosIngester.fs | 2 +- .../Equinox.Projection/Coordination.fs | 16 ++++++++-------- equinox-sync/Sync/Program.fs | 3 ++- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 20afd625d..a3af0e690 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -98,7 +98,7 @@ module Writer = // for x in blocked do markDirty x // res -type CosmosStats private (log : ILogger, writerResultLog, maxPendingBatches, statsInterval) = +type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = inherit Stats(log, maxPendingBatches, statsInterval) let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index a4509d2dd..6be370663 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -80,7 +80,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = /// c) writing of progress /// d) reporting of state /// The key bit that's managed externally is the reading/accepting of incoming data -type Coordinator<'R>(log : ILogger, maxPendingBatches, processorDop, project : int64 option * StreamSpan -> Async>, handleResult, statsInterval) = +type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * StreamSpan -> Async>, handleResult) = let sleepIntervalMs = 5 let cts = new CancellationTokenSource() let batches = new SemaphoreSlim(maxPendingBatches) @@ -89,7 +89,6 @@ type Coordinator<'R>(log : ILogger, maxPendingBatches, processorDop, project : i let dispatcher = Dispatcher(processorDop) let progressState = ProgressState() let progressWriter = ProgressWriter<_>() - let stats = Stats(log, maxPendingBatches, statsInterval) let handle = function | Add (epoch, checkpoint,items) -> let reqs = Dictionary() @@ -106,7 +105,7 @@ type Coordinator<'R>(log : ILogger, maxPendingBatches, processorDop, project : i | Added _ | ProgressResult _ | Result _ -> () - member private __.Pump() = async { + member private __.Pump(stats : Stats<'R>) = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) @@ -133,11 +132,11 @@ type Coordinator<'R>(log : ILogger, maxPendingBatches, processorDop, project : i let busy = processorDop - dispatcher.Capacity stats.TryDump(busy,processorDop,streams) do! Async.Sleep sleepIntervalMs } - static member Start<'R>(rangeLog, maxPendingBatches, processorDop, project, handleResult, statsInterval) = - let instance = new Coordinator<'R>(rangeLog, maxPendingBatches, processorDop, project, handleResult, statsInterval) - Async.Start <| instance.Pump() + static member Start<'R>(stats, maxPendingBatches, processorDop, project, handleResult) = + let instance = new Coordinator<'R>(maxPendingBatches, processorDop, project, handleResult) + Async.Start <| instance.Pump(stats) instance - static member Start(rangeLog, maxPendingBatches, processorDop, project : StreamSpan -> Async, statsInterval) = + static member Start(log, maxPendingBatches, processorDop, project : StreamSpan -> Async, statsInterval) = let project (_maybeWritePos, batch) = async { try let! count = project batch return batch.stream, Choice1Of2 (batch.span.index + int64 count) @@ -149,7 +148,8 @@ type Coordinator<'R>(log : ILogger, maxPendingBatches, processorDop, project : i | Result (stream, Choice2Of2 _) -> streams.MarkFailed stream | _ -> () - Coordinator.Start(rangeLog, maxPendingBatches, processorDop, project, handleResult, statsInterval) + let stats = Stats(log, maxPendingBatches, statsInterval) + Coordinator.Start(stats, maxPendingBatches, processorDop, project, handleResult) member __.Submit(epoch, markBatchCompleted, events) = async { let! _ = batches.Await() diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index dd793c856..b1bd42c82 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -374,7 +374,8 @@ module EventStoreSource = streams.MarkFailed stream Writer.logTo writerResultLog (s,r) | _ -> () - let coordinator = Coordinator.Start(log, maxPendingBatches, maxWriters, project, handleResult, statsInterval) + let stats = CosmosStats(log, maxPendingBatches, statsInterval) + let coordinator = Coordinator.Start(stats, maxPendingBatches, maxWriters, project, handleResult) let pumpReaders = let postStreamSpan : StreamSpan -> unit = coordinator.Submit let postBatch (pos : EventStore.ClientAPI.Position) xs = From 86789f5cb78d2291d56c6d73c24b6333258321bd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 15:32:19 +0100 Subject: [PATCH 141/353] Tidy result handling --- .../CosmosIngester.fs | 25 ++++++------ .../Equinox.Projection/Coordination.fs | 38 +++++++++---------- equinox-projector/Equinox.Projection/State.fs | 2 +- equinox-sync/Sync/Program.fs | 15 ++++---- 4 files changed, 41 insertions(+), 39 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index a3af0e690..907333c16 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -9,7 +9,7 @@ open Serilog let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 -type [] ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Other | Ok +type [] ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Other module Writer = type [] Result = @@ -54,11 +54,14 @@ module Writer = | _ -> Other let classify = function - | RateLimitedMessage -> ResultKind.RateLimited, false - | TimedOutMessage -> ResultKind.TimedOut, false - | TooLargeMessage -> ResultKind.TooLarge, true - | MalformedMessage -> ResultKind.Malformed, true - | Other -> ResultKind.Other, false + | RateLimitedMessage -> ResultKind.RateLimited + | TimedOutMessage -> ResultKind.TimedOut + | TooLargeMessage -> ResultKind.TooLarge + | MalformedMessage -> ResultKind.Malformed + | Other -> ResultKind.Other + let isMalformed = function + | ResultKind.RateLimited | ResultKind.TimedOut | ResultKind.Other -> false + | ResultKind.TooLarge | ResultKind.Malformed -> true //member __.TryGap() : (string*int64*int) option = // let rec aux () = @@ -127,9 +130,9 @@ type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = | Writer.Result.PrefixMissing _ -> incr resultPrefix | Result (stream, Choice2Of2 exn) -> match Writer.classify exn with - | ResultKind.Ok, _ | ResultKind.Other, _ -> () - | ResultKind.RateLimited, _ -> incr rateLimited - | ResultKind.TooLarge, _ -> category stream |> badCats.Ingest; incr tooLarge - | ResultKind.Malformed, _ -> category stream |> badCats.Ingest; incr malformed - | ResultKind.TimedOut, _ -> incr timedOut + | ResultKind.Other -> () + | ResultKind.RateLimited -> incr rateLimited + | ResultKind.TooLarge -> category stream |> badCats.Ingest; incr tooLarge + | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed + | ResultKind.TimedOut -> incr timedOut | Add _ | AddStream _ | Added _ | ProgressResult _ -> () \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 6be370663..520de6fb0 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -89,21 +89,6 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S let dispatcher = Dispatcher(processorDop) let progressState = ProgressState() let progressWriter = ProgressWriter<_>() - let handle = function - | Add (epoch, checkpoint,items) -> - let reqs = Dictionary() - let mutable count = 0 - for item in items do - streams.Add item - count <- count + 1 - reqs.[item.stream] <- item.index + 1L - progressState.AppendBatch((epoch,checkpoint),reqs) - work.Enqueue(Added (reqs.Count,count)) - | AddStream streamSpan -> - streams.Add(streamSpan,false) |> ignore - work.Enqueue(Added (1,streamSpan.span.events.Length)) - | Added _ | ProgressResult _ | Result _ -> - () member private __.Pump(stats : Stats<'R>) = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) @@ -111,12 +96,27 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S Async.Start(progressWriter.Pump(), cts.Token) Async.Start(dispatcher.Pump(), cts.Token) let handle x = - handle x - match x with Result _ as r -> handleResult (streams, progressState, batches) r | _ -> () - stats.Handle x + match x with + | Add (epoch, checkpoint, items) -> + let reqs = Dictionary() + let mutable count = 0 + for item in items do + streams.Add item + count <- count + 1 + reqs.[item.stream] <- item.index + 1L + progressState.AppendBatch((epoch,checkpoint),reqs) + work.Enqueue(Added (reqs.Count,count)) + | AddStream streamSpan -> + streams.Add(streamSpan,false) |> ignore + work.Enqueue(Added (1,streamSpan.span.events.Length)) + | Added _ | ProgressResult _ -> + () + | Result _ as r -> + handleResult (streams, progressState, batches) r + while not cts.IsCancellationRequested do // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - work |> ConcurrentQueue.drain handle + work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) let validatedPos, batches = progressState.Validate(streams.TryGetStreamWritePos) stats.HandleValidated(Option.map fst validatedPos, batches) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index e4d3d6118..966512f68 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -214,7 +214,7 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do match tryGetStreamWritePos stream with | Some index when requiredIndex <= index -> - Log.Warning("Validation had to remove {stream}", stream) + Log.Warning("Validation had to remove {stream} as required {req} has been met by {index}", stream, requiredIndex, index) batch.streamToRequiredIndex.Remove stream |> ignore | _ -> () if batch.streamToRequiredIndex.Count = 0 then diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index b1bd42c82..2541f64c9 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -350,20 +350,19 @@ module EventStoreSource = let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = let applyResultToStreamState = function | stream, (Choice1Of2 (Writer.Ok pos)) -> - streams.InternalUpdate stream pos null, ResultKind.Ok + streams.InternalUpdate stream pos null | stream, (Choice1Of2 (Writer.Duplicate pos)) -> - streams.InternalUpdate stream pos null, ResultKind.Ok + streams.InternalUpdate stream pos null | stream, (Choice1Of2 (Writer.PartialDuplicate overage)) -> - streams.InternalUpdate stream overage.index [|overage|], ResultKind.Ok + streams.InternalUpdate stream overage.index [|overage|] | stream, (Choice1Of2 (Writer.PrefixMissing (overage,pos))) -> - streams.InternalUpdate stream pos [|overage|], ResultKind.Ok + streams.InternalUpdate stream pos [|overage|] | stream, (Choice2Of2 exn) -> - let kind, malformed = Writer.classify exn - streams.SetMalformed(stream,malformed), kind + let malformed = Writer.classify exn |> Writer.isMalformed + streams.SetMalformed(stream,malformed) match res with | Message.Result (s,r) -> - // TODO kind ? - let (stream,updatedState), kind = applyResultToStreamState (s,r) + let stream,updatedState = applyResultToStreamState (s,r) match updatedState.write with | Some wp -> let closedBatches = progressState.MarkStreamProgress(stream, wp) From 793eee6a6ba68d03bd653b9ed4874bc4a98e4e53 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 15:57:39 +0100 Subject: [PATCH 142/353] Skip empty batches --- .../Equinox.Projection/Coordination.fs | 32 ++++++++++++------- equinox-projector/Equinox.Projection/State.fs | 3 +- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 520de6fb0..be80d5420 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -13,7 +13,7 @@ type Message<'R> = | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq | AddStream of StreamSpan /// Log stats about an ingested batch - | Added of streams: int * events: int + | Added of streams: int * skip: int * events: int /// Result of processing on stream - specified number of items or threw `exn` | Result of stream: string * outcome: Choice<'R,exn> /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` @@ -22,7 +22,7 @@ type Message<'R> = type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None let progCommitFails, progCommits = ref 0, ref 0 - let cycles, batchesPended, streamsPended, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (busy,capacity) (streams : StreamStates) = if !progCommitFails <> 0 || !progCommits <> 0 then @@ -40,18 +40,19 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}e) Busy {busy}/{processors} Completed {completed} ({passed} passed {exns} exn)", - !cycles, !batchesPended, !streamsPended, !eventsPended, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}-{skipped}e) Busy {busy}/{processors} Completed {completed} ({passed} passed {exns} exn)", + !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log abstract member Handle : Message<'R> -> unit default __.Handle res = match res with | Add _ | AddStream _ -> () - | Added (streams, events) -> + | Added (streams, skipped, events) -> incr batchesPended - eventsPended := !eventsPended + events streamsPended := !streamsPended + streams + eventsPended := !eventsPended + events + eventsSkipped := !eventsSkipped + skipped | Result (_stream, Choice1Of2 _) -> incr resultCompleted | Result (_stream, Choice2Of2 _) -> @@ -95,17 +96,24 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) Async.Start(dispatcher.Pump(), cts.Token) + let canSkip (streamState : StreamState) (item : StreamItem) = + match streamState.write, item.index + 1L with + | Some cw, required -> cw >= required + | _ -> false + let handle x = match x with | Add (epoch, checkpoint, items) -> let reqs = Dictionary() - let mutable count = 0 + let mutable count, skipCount = 0, 0 for item in items do - streams.Add item - count <- count + 1 - reqs.[item.stream] <- item.index + 1L + let streamState = streams.Add item + if canSkip streamState item then skipCount <- skipCount + 1 + else + count <- count + 1 + reqs.[item.stream] <- item.index+1L progressState.AppendBatch((epoch,checkpoint),reqs) - work.Enqueue(Added (reqs.Count,count)) + work.Enqueue(Added (reqs.Count,skipCount,count)) | AddStream streamSpan -> streams.Add(streamSpan,false) |> ignore work.Enqueue(Added (1,streamSpan.span.events.Length)) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 966512f68..b2fbb3c6b 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -111,7 +111,6 @@ type StreamStates() = stream, updated let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore - let enqueue isMalformed (item : StreamItem) = updateWritePos item.stream isMalformed None [| { index = item.index; events = [| item.event |] } |] |> ignore let busy = HashSet() let schedule (requestedOrder : string seq) (capacity: int) = @@ -132,7 +131,7 @@ type StreamStates() = member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } member __.Add(item: StreamItem, ?isMalformed) = - enqueue (defaultArg isMalformed false) item + updateWritePos item.stream (defaultArg isMalformed false) None [| { index = item.index; events = [| item.event |] } |] member __.Add(batch: StreamSpan, isMalformed) = updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] member __.SetMalformed(stream,isMalformed) = From 8994cd62f5dde5e23070b0f9ab1944df29560790 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 16:01:28 +0100 Subject: [PATCH 143/353] Fix --- equinox-projector/Equinox.Projection/Coordination.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index be80d5420..772db7226 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -107,7 +107,7 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S let reqs = Dictionary() let mutable count, skipCount = 0, 0 for item in items do - let streamState = streams.Add item + let _,streamState = streams.Add item if canSkip streamState item then skipCount <- skipCount + 1 else count <- count + 1 @@ -116,7 +116,7 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S work.Enqueue(Added (reqs.Count,skipCount,count)) | AddStream streamSpan -> streams.Add(streamSpan,false) |> ignore - work.Enqueue(Added (1,streamSpan.span.events.Length)) + //work.Enqueue(Added (1,streamSpan.span.events.Length)) | Added _ | ProgressResult _ -> () | Result _ as r -> From 5aa7ecd138a8d6f87ddcc306f4a502d9fc100318 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 16:27:57 +0100 Subject: [PATCH 144/353] Handle empty batch completion --- .../Equinox.Projection/Coordination.fs | 30 +++++++++++-------- equinox-projector/Equinox.Projection/State.fs | 14 +++++---- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 772db7226..674ba9a3e 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -96,26 +96,31 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) Async.Start(dispatcher.Pump(), cts.Token) - let canSkip (streamState : StreamState) (item : StreamItem) = + let validVsSkip (streamState : StreamState) (item : StreamItem) = match streamState.write, item.index + 1L with - | Some cw, required -> cw >= required - | _ -> false - + | Some cw, required when cw >= required -> 0, 1 + | _ -> 1, 0 + //let validVsSkipSpan (streamState : StreamState) (batch : StreamSpan) = + // match streamState.write, item.index + item.span.events.LongLength with + // | Some cw, required when cw >= required -> 0, 1 + // | _ -> 1, 0 let handle x = match x with | Add (epoch, checkpoint, items) -> let reqs = Dictionary() let mutable count, skipCount = 0, 0 for item in items do - let _,streamState = streams.Add item - if canSkip streamState item then skipCount <- skipCount + 1 - else - count <- count + 1 + let _stream,streamState = streams.Add item + match validVsSkip streamState item with + | 0, skip -> + skipCount <- skipCount + skip + | required, _ -> + count <- count + required reqs.[item.stream] <- item.index+1L progressState.AppendBatch((epoch,checkpoint),reqs) work.Enqueue(Added (reqs.Count,skipCount,count)) - | AddStream streamSpan -> - streams.Add(streamSpan,false) |> ignore + | AddStream streamSpan ->() + //let _stream,streamState = streams.Add(streamSpan,false) //work.Enqueue(Added (1,streamSpan.span.events.Length)) | Added _ | ProgressResult _ -> () @@ -126,8 +131,9 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let validatedPos, batches = progressState.Validate(streams.TryGetStreamWritePos) - stats.HandleValidated(Option.map fst validatedPos, batches) + let completed, validatedPos, pendingBatches = progressState.Validate(streams.TryGetStreamWritePos) + if completed <> 0 then batches.Release(completed) |> ignore + stats.HandleValidated(Option.map fst validatedPos, pendingBatches) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch // 3. After that, provision writers queue diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index b2fbb3c6b..20acd4705 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -206,9 +206,9 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = if streams.Add s then yield s,(batch,getStreamQueueLength s) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst - member __.Validate tryGetStreamWritePos : 'Pos option * int = - let rec aux () = - if pending.Count = 0 then () else + member __.Validate tryGetStreamWritePos : int * 'Pos option * int = + let rec aux completed = + if pending.Count = 0 then 0 else let batch = pending.Peek() for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do match tryGetStreamWritePos stream with @@ -219,9 +219,11 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = if batch.streamToRequiredIndex.Count = 0 then let headBatch = pending.Dequeue() validatedPos <- Some headBatch.pos - aux () - aux () - validatedPos, pending.Count + aux (completed + 1) + else + completed + let completed = aux 0 + completed, validatedPos, pending.Count /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher<'R>(maxDop) = From 0cb68e807272ede7dfd1c5a101bf19b053ffcceb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 16:54:33 +0100 Subject: [PATCH 145/353] Fix Batch counting --- equinox-projector/Equinox.Projection/Coordination.fs | 8 ++++---- equinox-projector/Equinox.Projection/Infrastructure.fs | 2 +- equinox-projector/Equinox.Projection/State.fs | 8 ++++---- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 674ba9a3e..8bbf93f93 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -131,8 +131,8 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let completed, validatedPos, pendingBatches = progressState.Validate(streams.TryGetStreamWritePos) - if completed <> 0 then batches.Release(completed) |> ignore + let completedBatches, validatedPos, pendingBatches = progressState.Validate(streams.TryGetStreamWritePos) + if completedBatches > 0 then batches.Release(completedBatches) |> ignore stats.HandleValidated(Option.map fst validatedPos, pendingBatches) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch @@ -167,11 +167,11 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S member __.Submit(epoch, markBatchCompleted, events) = async { let! _ = batches.Await() - Add (epoch, markBatchCompleted, Array.ofSeq events) |> work.Enqueue + work.Enqueue <| Add (epoch, markBatchCompleted, Array.ofSeq events) return maxPendingBatches-batches.CurrentCount,maxPendingBatches } member __.Submit(streamSpan) = - AddStream streamSpan |> work.Enqueue + work.Enqueue <| AddStream streamSpan member __.Stop() = cts.Cancel() \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Infrastructure.fs b/equinox-projector/Equinox.Projection/Infrastructure.fs index ea928dc24..15a93cb57 100644 --- a/equinox-projector/Equinox.Projection/Infrastructure.fs +++ b/equinox-projector/Equinox.Projection/Infrastructure.fs @@ -10,8 +10,8 @@ module ConcurrentQueue = let drain handle (xs : ConcurrentQueue<_>) = let rec aux () = match xs.TryDequeue() with - | true, x -> handle x; aux () | false, _ -> () + | true, x -> handle x; aux () aux () type Async with diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 20acd4705..dd48aa67f 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -208,7 +208,7 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst member __.Validate tryGetStreamWritePos : int * 'Pos option * int = let rec aux completed = - if pending.Count = 0 then 0 else + if pending.Count = 0 then completed else let batch = pending.Peek() for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do match tryGetStreamWritePos stream with @@ -216,12 +216,12 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = Log.Warning("Validation had to remove {stream} as required {req} has been met by {index}", stream, requiredIndex, index) batch.streamToRequiredIndex.Remove stream |> ignore | _ -> () - if batch.streamToRequiredIndex.Count = 0 then + if batch.streamToRequiredIndex.Count <> 0 then + completed + else let headBatch = pending.Dequeue() validatedPos <- Some headBatch.pos aux (completed + 1) - else - completed let completed = aux 0 completed, validatedPos, pending.Count From 92b242003dbde058d2070ac07fff734434ae584d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 26 Apr 2019 17:34:55 +0100 Subject: [PATCH 146/353] Fix exception stats --- .../Equinox.Projection.Cosmos/CosmosIngester.fs | 12 ++++++------ equinox-projector/Equinox.Projection/Coordination.fs | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 907333c16..53a806e80 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -103,15 +103,15 @@ module Writer = type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = inherit Stats(log, maxPendingBatches, statsInterval) - let resultOk, resultDup, resultPartialDup, resultPrefix, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0 + let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 let badCats = CatStats() override __.DumpExtraStats() = - let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExn - log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} Missing {exns} Exns)", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExn) - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExn := 0; + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExnOther + log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} awaiting prefix {exns} exceptions)", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExnOther) + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExnOther := 0; if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", !rateLimited, !timedOut, !tooLarge, !malformed) @@ -130,7 +130,7 @@ type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = | Writer.Result.PrefixMissing _ -> incr resultPrefix | Result (stream, Choice2Of2 exn) -> match Writer.classify exn with - | ResultKind.Other -> () + | ResultKind.Other -> incr resultExnOther | ResultKind.RateLimited -> incr rateLimited | ResultKind.TooLarge -> category stream |> badCats.Ingest; incr tooLarge | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 8bbf93f93..642bbdff3 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -40,7 +40,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}-{skipped}e) Busy {busy}/{processors} Completed {completed} ({passed} passed {exns} exn)", + log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}-{skipped}e) Busy {busy}/{processors} Completed {completed} ({passed} success {exns} exn)", !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log From 08a46caa1f148ea9397384faf85afcb7663ac522 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 00:47:09 +0100 Subject: [PATCH 147/353] Tidy labels --- .../CosmosIngester.fs | 39 +++---------------- .../Equinox.Projection/Coordination.fs | 2 +- equinox-projector/Equinox.Projection/State.fs | 8 ++-- equinox-sync/Sync/Program.fs | 8 ++-- 4 files changed, 15 insertions(+), 42 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 53a806e80..cccbb4593 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -73,33 +73,6 @@ module Writer = // | Some (pos,count) -> Some (stream,pos,int count) // | None -> aux () // aux () - //member __.TryReady(isBusy) = - // let blocked = ResizeArray() - // let rec aux () = - // match dirty |> Queue.tryDequeue with - // | None -> None - // | Some stream -> - - // match states.[stream] with - // | state when state.IsReady -> - // if (not << isBusy) stream then - // let h = state.queue |> Array.head - - // let mutable bytesBudget = cosmosPayloadLimit - // let mutable count = 0 - // let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = - // bytesBudget <- bytesBudget - cosmosPayloadBytes y - // count <- count + 1 - // // Reduce the item count when we don't yet know the write position - // count <= (if Option.isNone state.write then 10 else 4096) && (bytesBudget >= 0 || count = 1) - // Some { stream = stream; span = { index = h.index; events = h.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } - // else - // blocked.Add(stream) |> ignore - // aux () - // | _ -> aux () - // let res = aux () - // for x in blocked do markDirty x - // res type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = inherit Stats(log, maxPendingBatches, statsInterval) @@ -109,13 +82,13 @@ type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExnOther - log.Information("Wrote {completed} ({ok} ok {dup} redundant {partial} partial {prefix} awaiting prefix {exns} exceptions)", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, !resultExnOther) - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; resultExnOther := 0; + log.Information("CosmosDb {completed:n0} ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} awaiting prefix)", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - log.Warning("Exceptions {rateLimited} rate-limited, {timedOut} timed out, {tooLarge} too large, {malformed} malformed", - !rateLimited, !timedOut, !tooLarge, !malformed) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0 + log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", + !rateLimited, !timedOut, !tooLarge, !malformed, !resultExnOther) + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0; if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() Metrics.dumpRuStats statsInterval log diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 642bbdff3..fbe2e7494 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -40,7 +40,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - log.Information("Cycles {cycles} Ingested {batches} ({streams}s {events}-{skipped}e) Busy {busy}/{processors} Completed {completed} ({passed} success {exns} exn)", + log.Information("Cycles {cycles} Ingested {batches}b {streams:n0}s {events:n0}-{skipped:n0}e Active {busy}/{processors} Completed {completed:n0} ({passed} success {exns} exn)", !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index dd48aa67f..799ca66f6 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -171,10 +171,10 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) ready <- ready + 1 readyB <- readyB + sz - log.Information("Busy {busy}/{busyMb:n1}MB Ready {ready}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB Synced {synced}", - busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB, synced) - if busyCats.Any then log.Information("Busy Categories, events {busyCats}", busyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Categories, events {readyCats}", readyCats.StatsDescending) + log.Information("Synced {synced:n0} In flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) + if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2541f64c9..ba2554502 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -101,8 +101,8 @@ module CmdParser = | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" - | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 64" + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 128" + | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" @@ -125,8 +125,8 @@ module CmdParser = member __.VerboseConsole = a.Contains VerboseConsole member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,64) - member __.MaxWriters = a.GetResult(MaxWriters,64) + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,128) + member __.MaxWriters = a.GetResult(MaxWriters,512) member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.StreamReaders = a.GetResult(StreamReaders,8) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds From 354ae7cdd81f45ad98da5a581e4fb802ffc72239 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 01:48:43 +0100 Subject: [PATCH 148/353] Event counts+sizes --- .../CosmosIngester.fs | 23 ++++++++------ .../Equinox.Projection.Tests/ProgressTests.fs | 12 ++++--- .../Equinox.Projection/Coordination.fs | 6 ++-- equinox-projector/Equinox.Projection/State.fs | 8 +++-- equinox-sync/Sync/Program.fs | 31 ++++++++++--------- 5 files changed, 45 insertions(+), 35 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index cccbb4593..8a727f554 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -17,15 +17,15 @@ module Writer = | Duplicate of updatedPos: int64 | PartialDuplicate of overage: Span | PrefixMissing of batch: Span * writePos: int64 - let logTo (log: ILogger) (res : string * Choice) = + let logTo (log: ILogger) (res : string * Choice<(int*int)*Result,exn>) = match res with - | stream, (Choice1Of2 (Ok pos)) -> + | stream, (Choice1Of2 (_, Ok pos)) -> log.Information("Wrote {stream} up to {pos}", stream, pos) - | stream, (Choice1Of2 (Duplicate updatedPos)) -> + | stream, (Choice1Of2 (_, Duplicate updatedPos)) -> log.Information("Ignored {stream} (synced up to {pos})", stream, updatedPos) - | stream, (Choice1Of2 (PartialDuplicate overage)) -> + | stream, (Choice1Of2 (_, PartialDuplicate overage)) -> log.Information("Requeing {stream} {pos} ({count} events)", stream, overage.index, overage.events.Length) - | stream, (Choice1Of2 (PrefixMissing (batch,pos))) -> + | stream, (Choice1Of2 (_, PrefixMissing (batch,pos))) -> log.Information("Waiting {stream} missing {gap} events ({count} events @ {pos})", stream, batch.index-pos, batch.events.Length, batch.index) | stream, Choice2Of2 exn -> log.Warning(exn,"Writing {stream} failed, retrying", stream) @@ -74,16 +74,17 @@ module Writer = // | None -> aux () // aux () -type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = - inherit Stats(log, maxPendingBatches, statsInterval) +type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = + inherit Stats<(int*int)*Writer.Result>(log, maxPendingBatches, statsInterval) let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 + let mutable events, bytes = 0, 0L let badCats = CatStats() override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExnOther - log.Information("CosmosDb {completed:n0} ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} awaiting prefix)", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + log.Information("Requests {completed:n0} {events}e {mb}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} awaiting prefix)", + results, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", @@ -95,7 +96,9 @@ type CosmosStats (log : ILogger, maxPendingBatches, statsInterval) = override __.Handle message = base.Handle message match message with - | Message.Result (_stream, Choice1Of2 r) -> + | Message.Result (_stream, Choice1Of2 ((es,bs),r)) -> + events <- events + es + bytes <- bytes + int64 bs match r with | Writer.Result.Ok _ -> incr resultOk | Writer.Result.Duplicate _ -> incr resultDup diff --git a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs index ab1bdde2f..cc82c7964 100644 --- a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs +++ b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs @@ -10,7 +10,8 @@ let mkDictionary xs = Dictionary(dict xs) let [] ``Empty has zero streams pending or progress to write`` () = let sut = ProgressState<_>() - let validatedPos, batches = sut.Validate(fun _ -> None) + let completed,validatedPos, batches = sut.Validate(fun _ -> None) + 0 =! completed None =! validatedPos 0 =! batches @@ -18,7 +19,8 @@ let [] ``Can add multiple batches`` () = let sut = ProgressState<_>() sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) sut.AppendBatch(1,mkDictionary ["b",2L; "c",3L]) - let validatedPos, batches = sut.Validate(fun _ -> None) + let completed,validatedPos, batches = sut.Validate(fun _ -> None) + 0 =! completed None =! validatedPos 2 =! batches @@ -27,7 +29,8 @@ let [] ``Marking Progress Removes batches and updates progress`` () = sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) sut.MarkStreamProgress("a",1L) |> ignore sut.MarkStreamProgress("b",1L) |> ignore - let validatedPos, batches = sut.Validate(fun _ -> None) + let completed, validatedPos, batches = sut.Validate(fun _ -> None) + 0 =! completed None =! validatedPos 1 =! batches @@ -36,6 +39,7 @@ let [] ``Marking progress is not persistent`` () = sut.AppendBatch(0, mkDictionary ["a",1L]) sut.MarkStreamProgress("a",2L) |> ignore sut.AppendBatch(1, mkDictionary ["a",1L; "b",2L]) - let validatedPos, batches = sut.Validate(fun _ -> None) + let completed, validatedPos, batches = sut.Validate(fun _ -> None) + 0 =! completed Some 0 =! validatedPos 1 =! batches \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index fbe2e7494..e64225f72 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -25,6 +25,9 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (busy,capacity) (streams : StreamStates) = + log.Information("Cycles {cycles} Ingested {batches}b {streams:n0}s {events:n0}-{skipped:n0}e Active {busy}/{processors} Completed {completed:n0} ({passed} success {exns} exn)", + !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -40,9 +43,6 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) - log.Information("Cycles {cycles} Ingested {batches}b {streams:n0}s {events:n0}-{skipped:n0}e Active {busy}/{processors} Completed {completed:n0} ({passed} success {exns} exn)", - !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log abstract member Handle : Message<'R> -> unit default __.Handle res = diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 799ca66f6..6e1195929 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -21,7 +21,7 @@ let expiredMs ms = due let arrayBytes (x:byte[]) = if x = null then 0 else x.Length -let private mb x = float x / 1024. / 1024. +let mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } @@ -154,12 +154,13 @@ type StreamStates() = schedule requestedOrder capacity member __.Dump(log : ILogger) = let mutable busyCount, busyB, ready, readyB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0 - let busyCats, readyCats, readyStreams = CatStats(), CatStats(), CatStats() + let busyCats, readyCats, readyStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats() for KeyValue (stream,state) in states do match int64 state.Size with | 0L -> synced <- synced + 1 | sz when state.isMalformed -> + malformedStreams.Ingest(stream, mb sz |> int64) malformed <- malformed + 1 malformedB <- malformedB + sz | sz when busy.Contains stream -> @@ -171,11 +172,12 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) ready <- ready + 1 readyB <- readyB + sz - log.Information("Synced {synced:n0} In flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + log.Information("Synced {synced:n0} In-flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index ba2554502..091c60556 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -269,7 +269,7 @@ module CmdParser = | Database _ -> "specify a database name for Cosmos account (default: envvar:EQUINOX_COSMOS_DATABASE)." | Collection _ -> "specify a collection name for Cosmos account (default: envvar:EQUINOX_COSMOS_COLLECTION)." | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 1)." + | Retries _ -> "specify operation retries (default: 0)." | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." and DestinationArguments(a : ParseResults) = @@ -280,7 +280,7 @@ module CmdParser = member __.Collection = match a.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" member __.Timeout = a.GetResult(Timeout, 5.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(Retries, 1) + member __.Retries = a.GetResult(Retries, 0) member __.MaxRetryWaitTime = a.GetResult(RetriesWaitTime, 5) /// Connect with the provided parameters and/or environment variables @@ -345,24 +345,26 @@ module EventStoreSource = let project batch = async { let trimmed = trim batch try let! res = Writer.write log cosmosContext trimmed - return trimmed.stream, Choice1Of2 res + let ctx = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes + return trimmed.stream, Choice1Of2 (ctx,res) with e -> return trimmed.stream, Choice2Of2 e } + let stats = CosmosStats(log, maxPendingBatches, statsInterval) let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = let applyResultToStreamState = function - | stream, (Choice1Of2 (Writer.Ok pos)) -> - streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (Writer.Duplicate pos)) -> - streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (Writer.PartialDuplicate overage)) -> - streams.InternalUpdate stream overage.index [|overage|] - | stream, (Choice1Of2 (Writer.PrefixMissing (overage,pos))) -> - streams.InternalUpdate stream pos [|overage|] + | stream, (Choice1Of2 (ctx, Writer.Ok pos)) -> + Some ctx,streams.InternalUpdate stream pos null + | stream, (Choice1Of2 (ctx, Writer.Duplicate pos)) -> + Some ctx,streams.InternalUpdate stream pos null + | stream, (Choice1Of2 (ctx, Writer.PartialDuplicate overage)) -> + Some ctx,streams.InternalUpdate stream overage.index [|overage|] + | stream, (Choice1Of2 (ctx, Writer.PrefixMissing (overage,pos))) -> + Some ctx,streams.InternalUpdate stream pos [|overage|] | stream, (Choice2Of2 exn) -> let malformed = Writer.classify exn |> Writer.isMalformed - streams.SetMalformed(stream,malformed) + None,streams.SetMalformed(stream,malformed) match res with | Message.Result (s,r) -> - let stream,updatedState = applyResultToStreamState (s,r) + let ctx,(stream,updatedState) = applyResultToStreamState (s,r) match updatedState.write with | Some wp -> let closedBatches = progressState.MarkStreamProgress(stream, wp) @@ -373,8 +375,7 @@ module EventStoreSource = streams.MarkFailed stream Writer.logTo writerResultLog (s,r) | _ -> () - let stats = CosmosStats(log, maxPendingBatches, statsInterval) - let coordinator = Coordinator.Start(stats, maxPendingBatches, maxWriters, project, handleResult) + let coordinator = Coordinator<(int*int)*Writer.Result>.Start(stats, maxPendingBatches, maxWriters, project, handleResult) let pumpReaders = let postStreamSpan : StreamSpan -> unit = coordinator.Submit let postBatch (pos : EventStore.ClientAPI.Position) xs = From c7b99975209ce994f7676dea27a2f1c10fb33bcb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 01:58:21 +0100 Subject: [PATCH 149/353] Tidy req stats --- equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs | 4 ++-- equinox-projector/Equinox.Projection/Coordination.fs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 8a727f554..5fe202eac 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -83,8 +83,8 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExnOther - log.Information("Requests {completed:n0} {events}e {mb}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} awaiting prefix)", - results, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + log.Information("Reqs {completed:n0} ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting) {events:n0}e {mb:n1}MB", + results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, events, mb bytes) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index e64225f72..8b8d89d63 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -25,7 +25,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (busy,capacity) (streams : StreamStates) = - log.Information("Cycles {cycles} Ingested {batches}b {streams:n0}s {events:n0}-{skipped:n0}e Active {busy}/{processors} Completed {completed:n0} ({passed} success {exns} exn)", + log.Information("Cycles {cycles} Batches {batches} {streams:n0}s {events:n0}-{skipped:n0}e Active {busy}/{processors} Completed {completed:n0} ({passed} success {exns} exn)", !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 if !progCommitFails <> 0 || !progCommits <> 0 then From 18dbc9b251807780662a3d41b497f1ae666797f8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 02:08:31 +0100 Subject: [PATCH 150/353] Tidy progress --- .../CosmosIngester.fs | 4 ++-- .../Equinox.Projection/Coordination.fs | 18 +++++++++--------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 5fe202eac..65f235598 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -83,8 +83,8 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExnOther - log.Information("Reqs {completed:n0} ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting) {events:n0}e {mb:n1}MB", - results, !resultOk, !resultDup, !resultPartialDup, !resultPrefix, events, mb bytes) + log.Information("Reqs {completed:n0} {mb:n1}MB {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + results, mb bytes, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 8b8d89d63..8edae5cc8 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -31,18 +31,18 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> - log.Error("Progress @ {validated}; writing failing: {failures} failures ({commits} successful commits) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, !progCommitFails, !progCommits, pendingBatchCount, maxPendingBatches) + log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) | Some committed when !progCommitFails <> 0 -> - log.Warning("Progress @ {validated} (committed: {committed}, {commits} commits, {failures} failures) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails, pendingBatchCount, maxPendingBatches) + log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) | Some committed -> - log.Information("Progress @ {validated} (committed: {committed}, {commits} commits) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, committed, !progCommits, pendingBatchCount, maxPendingBatches) + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) progCommits := 0; progCommitFails := 0 else - log.Information("Progress @ {validated} (committed: {committed}) Uncommitted {pendingBatches}/{maxPendingBatches}", - Option.toNullable validatedEpoch, Option.toNullable comittedEpoch, pendingBatchCount, maxPendingBatches) + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) streams.Dump log abstract member Handle : Message<'R> -> unit default __.Handle res = @@ -82,7 +82,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = /// d) reporting of state /// The key bit that's managed externally is the reading/accepting of incoming data type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * StreamSpan -> Async>, handleResult) = - let sleepIntervalMs = 5 + let sleepIntervalMs = 1 let cts = new CancellationTokenSource() let batches = new SemaphoreSlim(maxPendingBatches) let work = ConcurrentQueue>() From e067ce1da0c3a7992e909184285fdc04ced8bf26 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 02:15:34 +0100 Subject: [PATCH 151/353] Tidy --- equinox-projector/Equinox.Projection/Coordination.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 8edae5cc8..28668c259 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -25,9 +25,6 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (busy,capacity) (streams : StreamStates) = - log.Information("Cycles {cycles} Batches {batches} {streams:n0}s {events:n0}-{skipped:n0}e Active {busy}/{processors} Completed {completed:n0} ({passed} success {exns} exn)", - !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity, !resultCompleted + !resultExn, !resultCompleted, !resultExn) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -43,6 +40,9 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + log.Information("Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Active {busy}/{processors} Completed {completed:n0} Exceptions {exns})", + !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity,!resultCompleted, !resultExn) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log abstract member Handle : Message<'R> -> unit default __.Handle res = From b9c0d44e4392290aca6206c940e352449d9a9c0b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 02:30:16 +0100 Subject: [PATCH 152/353] Tidy stream list --- equinox-projector/Equinox.Projection/Coordination.fs | 2 +- equinox-projector/Equinox.Projection/State.fs | 2 +- equinox-sync/Sync/EventStoreSource.fs | 6 +++--- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 28668c259..7740ac7d8 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -40,7 +40,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - log.Information("Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Active {busy}/{processors} Completed {completed:n0} Exceptions {exns})", + log.Information("Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Active {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity,!resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 6e1195929..81f578809 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -169,7 +169,7 @@ type StreamStates() = busyB <- busyB + sz | sz -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%d" stream (defaultArg state.write 0L), mb sz |> int64) + readyStreams.Ingest(sprintf "%sx%d@%d" stream state.queue.Length (defaultArg state.write 0L), mb sz |> int64) ready <- ready + 1 readyB <- readyB + sz log.Information("Synced {synced:n0} In-flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 1ba3e9a2a..d15bc1994 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -226,7 +226,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = __.AddTranche(range, bs) return false | Tail (pos, max, interval, batchSize) -> - let mutable count, pauses, batchSize, range = 0, 0, batchSize, Range(pos, None, max) + let mutable count, batchSize, range = 0, batchSize, Range(pos, None, max) let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds let tailSw = Stopwatch.StartNew() @@ -241,8 +241,8 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = while true do let currentPos = range.Current if progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Information("Tailed {count} times ({pauses} waits) @ {pos} (chunk {chunk})", - count, pauses, currentPos.CommitPosition, chunk currentPos) + Log.Information("Tailed {count} times @ {pos} (chunk {chunk})", + count, currentPos.CommitPosition, chunk currentPos) progressSw.Restart() count <- count + 1 let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch From 84a83bb4811436987001af1c15eb333fef4e6e9f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 03:03:51 +0100 Subject: [PATCH 153/353] start Cosmos sync --- .../Equinox.Projection/Coordination.fs | 10 ++-- equinox-projector/Equinox.Projection/State.fs | 4 +- equinox-sync/Sync/Infrastructure.fs | 54 +++++++++---------- equinox-sync/Sync/Program.fs | 21 +++----- equinox-sync/Sync/Sync.fsproj | 1 + 5 files changed, 39 insertions(+), 51 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index 7740ac7d8..fb33388e2 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -100,10 +100,6 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S match streamState.write, item.index + 1L with | Some cw, required when cw >= required -> 0, 1 | _ -> 1, 0 - //let validVsSkipSpan (streamState : StreamState) (batch : StreamSpan) = - // match streamState.write, item.index + item.span.events.LongLength with - // | Some cw, required when cw >= required -> 0, 1 - // | _ -> 1, 0 let handle x = match x with | Add (epoch, checkpoint, items) -> @@ -119,9 +115,9 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S reqs.[item.stream] <- item.index+1L progressState.AppendBatch((epoch,checkpoint),reqs) work.Enqueue(Added (reqs.Count,skipCount,count)) - | AddStream streamSpan ->() - //let _stream,streamState = streams.Add(streamSpan,false) - //work.Enqueue(Added (1,streamSpan.span.events.Length)) + | AddStream streamSpan -> + let _stream,_streamState = streams.Add(streamSpan,false) + work.Enqueue(Added (1,0,streamSpan.span.events.Length)) // Yes, need to compute skip | Added _ | ProgressResult _ -> () | Result _ as r -> diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 81f578809..8c02297e9 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -169,12 +169,12 @@ type StreamStates() = busyB <- busyB + sz | sz -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%sx%d@%d" stream state.queue.Length (defaultArg state.write 0L), mb sz |> int64) + readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, mb sz |> int64) ready <- ready + 1 readyB <- readyB + sz log.Information("Synced {synced:n0} In-flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) - if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) + if busyCats.Any then log.Information("In-flight Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 21fbdb72a..41449ff19 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -6,29 +6,29 @@ open System open System.Threading open System.Threading.Tasks -#nowarn "21" // re AwaitKeyboardInterrupt -#nowarn "40" // re AwaitKeyboardInterrupt +//#nowarn "21" // re AwaitKeyboardInterrupt +//#nowarn "40" // re AwaitKeyboardInterrupt -type Async with - static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) - /// Asynchronously awaits the next keyboard interrupt event - static member AwaitKeyboardInterrupt () : Async = - Async.FromContinuations(fun (sc,_,_) -> - let isDisposed = ref 0 - let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore - and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) +//type Async with +// static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) +// /// Asynchronously awaits the next keyboard interrupt event +// static member AwaitKeyboardInterrupt () : Async = +// Async.FromContinuations(fun (sc,_,_) -> +// let isDisposed = ref 0 +// let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore +// and d : IDisposable = Console.CancelKeyPress.Subscribe callback +// in ()) -module Queue = - let tryDequeue (x : System.Collections.Generic.Queue<'T>) = -#if NET461 - if x.Count = 0 then None - else x.Dequeue() |> Some -#else - match x.TryDequeue() with - | false, _ -> None - | true, res -> Some res -#endif +//module Queue = +// let tryDequeue (x : System.Collections.Generic.Queue<'T>) = +//#if NET461 +// if x.Count = 0 then None +// else x.Dequeue() |> Some +//#else +// match x.TryDequeue() with +// | false, _ -> None +// | true, res -> Some res +//#endif type SemaphoreSlim with /// F# friendly semaphore await function @@ -39,9 +39,9 @@ type SemaphoreSlim with return! Async.AwaitTaskCorrect task } - /// Throttling wrapper which waits asynchronously until the semaphore has available capacity - member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { - let! _ = semaphore.Await() - try return! workflow - finally semaphore.Release() |> ignore - } \ No newline at end of file +// /// Throttling wrapper which waits asynchronously until the semaphore has available capacity +// member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { +// let! _ = semaphore.Await() +// try return! workflow +// finally semaphore.Release() |> ignore +// } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 091c60556..aee550f8e 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -3,8 +3,7 @@ open Equinox.Cosmos //#if !eventStore open Equinox.Cosmos.Projection -//#endif -//#if eventStore +//#else open Equinox.EventStore //#endif open Equinox.Projection.Coordination @@ -18,18 +17,8 @@ open System //#if !eventStore open System.Collections.Generic //#endif -open System.Diagnostics open System.Threading -let mb x = float x / 1024. / 1024. -let category (streamName : string) = streamName.Split([|'-'|],2).[0] -let every ms f = - let timer = Stopwatch.StartNew() - fun () -> - if timer.ElapsedMilliseconds > ms then - f () - timer.Restart() - //#if eventStore type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint @@ -70,6 +59,7 @@ module CmdParser = | [] LagFreqS of float | [] ChangeFeedVerbose #else + | [] ForceRestart | [] MinBatchSize of int | [] MaxPendingBatches of int | [] MaxWriters of int @@ -80,7 +70,7 @@ module CmdParser = | [] Tail of intervalS: float | [] VerboseConsole #endif - | [] ForceRestart + | [] FromTail | [] BatchSize of int | [] Verbose | [] Source of ParseResults @@ -90,7 +80,7 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" #if cosmos - | ForceStartFromHere _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." + | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | BatchSize _ -> "maximum item count to request from feed. Default: 1000" | LeaseCollectionSource _ ->"specify Collection Name for Leases collection, within `source` connection/database (default: `source`'s `collection` + `-aux`)." | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." @@ -117,7 +107,7 @@ module CmdParser = #if cosmos member __.LeaseId = a.GetResult ConsumerGroupName member __.BatchSize = a.GetResult(BatchSize,1000) - member __.StartFromHere = a.Contains ForceStartFromHere + member __.StartFromHere = a.Contains FromTail member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose #else @@ -390,6 +380,7 @@ module EventStoreSource = // match buffer.TryGap() with // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) // | None -> more <- false + static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 08e793009..5b8025ee7 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,6 +4,7 @@ Exe netcoreapp2.1 5 + cosmos2 From 4d4be0f22014340a32f7acbe40491d282032fa82 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 03:09:33 +0100 Subject: [PATCH 154/353] Default to not from tail --- equinox-sync/Sync/Program.fs | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index aee550f8e..2a89af10d 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -59,6 +59,7 @@ module CmdParser = | [] LagFreqS of float | [] ChangeFeedVerbose #else + | [] FromTail | [] ForceRestart | [] MinBatchSize of int | [] MaxPendingBatches of int @@ -70,7 +71,6 @@ module CmdParser = | [] Tail of intervalS: float | [] VerboseConsole #endif - | [] FromTail | [] BatchSize of int | [] Verbose | [] Source of ParseResults @@ -88,6 +88,7 @@ module CmdParser = | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | Source _ -> "CosmosDb input parameters." #else + | FromTail _ -> "Start the processing from the Tail" | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" @@ -143,11 +144,12 @@ module CmdParser = #else member x.BuildFeedParams() : ReaderSpec = let startPos = - match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent with - | Some p, _, _ -> Absolute p - | _, Some c, _ -> StartPos.Chunk c - | _, _, Some p -> Percentage p - | None, None, None -> StartPos.TailOrCheckpoint + match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains FromTail with + | Some p, _, _, _ -> Absolute p + | _, Some c, _, _ -> StartPos.Chunk c + | _, _, Some p, _ -> Percentage p + | None, None, None, true -> StartPos.TailOrCheckpoint + | None, None, None, _ -> Absolute 0L Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", From d2f6e8338353404e4a20544e6b9c6daa4fced6a7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 03:12:11 +0100 Subject: [PATCH 155/353] Fix start message --- equinox-sync/Sync/Program.fs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2a89af10d..a4511f034 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -20,7 +20,7 @@ open System.Collections.Generic open System.Threading //#if eventStore -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint type ReaderSpec = { /// Identifier for this projection and it's state @@ -149,7 +149,7 @@ module CmdParser = | _, Some c, _, _ -> StartPos.Chunk c | _, _, Some p, _ -> Percentage p | None, None, None, true -> StartPos.TailOrCheckpoint - | None, None, None, _ -> Absolute 0L + | None, None, None, _ -> StartPos.StartOrCheckpoint Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", @@ -396,6 +396,7 @@ module EventStoreSource = | Chunk c -> EventStoreSource.posFromChunk c | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) | TailOrCheckpoint -> max + | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start let! startMode, startPos, checkpointFreq = async { match initialCheckpointState, requestedStartPos with | Checkpoint.Folds.NotStarted, r -> From 393ec84fbac100a524944f4c336e7c878250faa7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 16:28:14 +0100 Subject: [PATCH 156/353] Generalize Ingestion --- .../Equinox.Projection.Cosmos.fsproj | 2 +- .../{CosmosIngester.fs => Ingestion.fs} | 61 +++++++- .../Equinox.Projection.Cosmos/Metrics.fs | 2 + equinox-projector/Equinox.Projection/State.fs | 4 +- equinox-projector/Projector/Infrastructure.fs | 13 +- equinox-projector/Projector/Program.fs | 1 - equinox-sync/Ingest/Program.fs | 1 - equinox-sync/Sync/EventStoreSource.fs | 5 +- equinox-sync/Sync/Infrastructure.fs | 18 +-- equinox-sync/Sync/Program.fs | 137 +++++++++++------- equinox-sync/Sync/Sync.fsproj | 2 +- 11 files changed, 161 insertions(+), 85 deletions(-) rename equinox-projector/Equinox.Projection.Cosmos/{CosmosIngester.fs => Ingestion.fs} (64%) diff --git a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj index 4242a0a46..7d1b24529 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj +++ b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj @@ -11,7 +11,7 @@ - + diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs similarity index 64% rename from equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs rename to equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs index 65f235598..ecbdd5aba 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs @@ -1,10 +1,12 @@ -module Equinox.Projection.Cosmos.CosmosIngester +module Equinox.Projection.Cosmos.Ingestion open Equinox.Cosmos.Core open Equinox.Cosmos.Store open Equinox.Projection.Coordination open Equinox.Projection.State open Serilog +open System +open System.Threading let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 @@ -82,14 +84,14 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = let badCats = CatStats() override __.DumpExtraStats() = - let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + !resultExnOther - log.Information("Reqs {completed:n0} {mb:n1}MB {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", - results, mb bytes, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix + log.Information("Reqs {completed:n0} {mb:n3}GB {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + results, mb bytes/1024., events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", !rateLimited, !timedOut, !tooLarge, !malformed, !resultExnOther) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0; + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0; events <- 0; bytes <- 0L if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() Metrics.dumpRuStats statsInterval log @@ -111,4 +113,51 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = | ResultKind.TooLarge -> category stream |> badCats.Ingest; incr tooLarge | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed | ResultKind.TimedOut -> incr timedOut - | Add _ | AddStream _ | Added _ | ProgressResult _ -> () \ No newline at end of file + | Add _ | AddStream _ | Added _ | ProgressResult _ -> () + +module CosmosIngestionCoordinator = + let create (log : Serilog.ILogger, cosmosContext, maxWriters, maxPendingBatches, statsInterval) = + let writerResultLog = log.ForContext() + let trim (writePos : int64 option, batch : StreamSpan) = + let mutable bytesBudget = cosmosPayloadLimit + let mutable count = 0 + let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = + bytesBudget <- bytesBudget - cosmosPayloadBytes y + count <- count + 1 + // Reduce the item count when we don't yet know the write position + count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) + { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } + let project batch = async { + let trimmed = trim batch + try let! res = Writer.write log cosmosContext trimmed + let ctx = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes + return trimmed.stream, Choice1Of2 (ctx,res) + with e -> return trimmed.stream, Choice2Of2 e } + let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = + let applyResultToStreamState = function + | stream, (Choice1Of2 (ctx, Writer.Ok pos)) -> + Some ctx,streams.InternalUpdate stream pos null + | stream, (Choice1Of2 (ctx, Writer.Duplicate pos)) -> + Some ctx,streams.InternalUpdate stream pos null + | stream, (Choice1Of2 (ctx, Writer.PartialDuplicate overage)) -> + Some ctx,streams.InternalUpdate stream overage.index [|overage|] + | stream, (Choice1Of2 (ctx, Writer.PrefixMissing (overage,pos))) -> + Some ctx,streams.InternalUpdate stream pos [|overage|] + | stream, (Choice2Of2 exn) -> + let malformed = Writer.classify exn |> Writer.isMalformed + None,streams.SetMalformed(stream,malformed) + match res with + | Message.Result (s,r) -> + let _ctx,(stream,updatedState) = applyResultToStreamState (s,r) + match updatedState.write with + | Some wp -> + let closedBatches = progressState.MarkStreamProgress(stream, wp) + if closedBatches > 0 then + batches.Release(closedBatches) |> ignore + streams.MarkCompleted(stream,wp) + | None -> + streams.MarkFailed stream + Writer.logTo writerResultLog (s,r) + | _ -> () + let stats = CosmosStats(log, maxPendingBatches, statsInterval) + Coordinator<(int*int)*Writer.Result>.Start(stats, maxPendingBatches, maxWriters, project, handleResult) \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs b/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs index 8e554d856..6b8d9e610 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs @@ -1,5 +1,7 @@ module Equinox.Projection.Cosmos.Metrics +// TODO move into equinox.cosmos + open System module RuCounters = diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 8c02297e9..d74649be5 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -169,14 +169,14 @@ type StreamStates() = busyB <- busyB + sz | sz -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, mb sz |> int64) + readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) ready <- ready + 1 readyB <- readyB + sz log.Information("Synced {synced:n0} In-flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) if busyCats.Any then log.Information("In-flight Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams, MB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } diff --git a/equinox-projector/Projector/Infrastructure.fs b/equinox-projector/Projector/Infrastructure.fs index 970315bf7..858bfce82 100644 --- a/equinox-projector/Projector/Infrastructure.fs +++ b/equinox-projector/Projector/Infrastructure.fs @@ -29,7 +29,6 @@ type Async with else ek(Exception "invalid Task state!")) |> ignore - type SemaphoreSlim with /// F# friendly semaphore await function member semaphore.Await(?timeout : TimeSpan) = async { @@ -39,9 +38,9 @@ type SemaphoreSlim with return! Async.AwaitTaskCorrect task } - /// Throttling wrapper which waits asynchronously until the semaphore has available capacity - member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { - let! _ = semaphore.Await() - try return! workflow - finally semaphore.Release() |> ignore - } \ No newline at end of file + ///// Throttling wrapper that waits asynchronously until the semaphore has available capacity + //member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { + // let! _ = semaphore.Await() + // try return! workflow + // finally semaphore.Release() |> ignore + //} \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index c71686724..b8501af59 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -212,7 +212,6 @@ module Logging = let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ChangeFeedVerbose - if not (System.Threading.ThreadPool.SetMaxThreads(1024,256)) then raise <| CmdParser.MissingArg "Cannot set MaxThreads to 1024" let discovery, connector, source = args.Cosmos.BuildConnectionDetails() let aux, leaseId, startFromHere, batchSize, maxPendingBatches, processorDop, lagFrequency = args.BuildChangeFeedParams() //#if kafka diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 93aed518a..78e40253c 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -11,7 +11,6 @@ open EventStoreSource type StartPos = Position of int64 | Chunk of int | Percentage of float | StreamList of string list | Start type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; minBatchSize: int } -let mb x = float x / 1024. / 1024. module CmdParser = open Argu diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index d15bc1994..dbb587253 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -2,6 +2,7 @@ open Equinox.Store // AwaitTaskCorrect open Equinox.Projection.Cosmos +open Equinox.Projection.Cosmos.Ingestion open Equinox.Projection.State open EventStore.ClientAPI open System @@ -18,9 +19,9 @@ let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadByte let tryToBatch (e : RecordedEvent) : StreamItem option = let eb = recPayloadBytes e - if eb > CosmosIngester.cosmosPayloadLimit then + if eb > cosmosPayloadLimit then Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", - e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, CosmosIngester.cosmosPayloadLimit) + e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, cosmosPayloadLimit) None else let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 41449ff19..3a00ff679 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -9,15 +9,15 @@ open System.Threading.Tasks //#nowarn "21" // re AwaitKeyboardInterrupt //#nowarn "40" // re AwaitKeyboardInterrupt -//type Async with -// static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) -// /// Asynchronously awaits the next keyboard interrupt event -// static member AwaitKeyboardInterrupt () : Async = -// Async.FromContinuations(fun (sc,_,_) -> -// let isDisposed = ref 0 -// let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore -// and d : IDisposable = Console.CancelKeyPress.Subscribe callback -// in ()) +type Async with + static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) + /// Asynchronously awaits the next keyboard interrupt event + static member AwaitKeyboardInterrupt () : Async = + Async.FromContinuations(fun (sc,_,_) -> + let isDisposed = ref 0 + let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore + and d : IDisposable = Console.CancelKeyPress.Subscribe callback + in ()) //module Queue = // let tryDequeue (x : System.Collections.Generic.Queue<'T>) = diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index a4511f034..b1034a5a8 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -1,6 +1,7 @@ module SyncTemplate.Program open Equinox.Cosmos +open Equinox.Projection.Cosmos.Ingestion //#if !eventStore open Equinox.Cosmos.Projection //#else @@ -16,6 +17,8 @@ open Serilog open System //#if !eventStore open System.Collections.Generic +//#else +open System.Diagnostics //#endif open System.Threading @@ -53,13 +56,16 @@ module CmdParser = type Parameters = | [] ConsumerGroupName of string | [] LocalSeq + | [] Verbose + | [] FromTail + | [] BatchSize of int #if cosmos + | [] ChangeFeedVerbose | [] LeaseCollectionSource of string | [] LeaseCollectionDestination of string | [] LagFreqS of float - | [] ChangeFeedVerbose #else - | [] FromTail + | [] VerboseConsole | [] ForceRestart | [] MinBatchSize of int | [] MaxPendingBatches of int @@ -69,28 +75,27 @@ module CmdParser = | [] Percent of float | [] StreamReaders of int | [] Tail of intervalS: float - | [] VerboseConsole #endif - | [] BatchSize of int - | [] Verbose | [] Source of ParseResults interface IArgParserTemplate with member a.Usage = match a with | ConsumerGroupName _ -> "Projector consumer group name." | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" + | Verbose -> "request Verbose Logging. Default: off" #if cosmos + | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | BatchSize _ -> "maximum item count to request from feed. Default: 1000" | LeaseCollectionSource _ ->"specify Collection Name for Leases collection, within `source` connection/database (default: `source`'s `collection` + `-aux`)." | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" - | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | Source _ -> "CosmosDb input parameters." #else - | FromTail _ -> "Start the processing from the Tail" - | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." + | VerboseConsole -> "request Verbose Console Logging. Default: off" + | FromTail -> "Start the processing from the Tail" | BatchSize _ -> "maximum item count to request from feed. Default: 4096" + | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 128" | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" @@ -99,21 +104,20 @@ module CmdParser = | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" | StreamReaders _ -> "number of concurrent readers. Default: 8" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" - | VerboseConsole -> "request Verbose Console Logging. Default: off" | Source _ -> "EventStore input parameters." #endif - | Verbose -> "request Verbose Logging. Default: off" and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None + member __.Verbose = a.Contains Verbose #if cosmos + member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName member __.BatchSize = a.GetResult(BatchSize,1000) member __.StartFromHere = a.Contains FromTail member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds - member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose #else - member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.VerboseConsole = a.Contains VerboseConsole + member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,128) @@ -125,8 +129,6 @@ module CmdParser = member __.ForceRestart = a.Contains ForceRestart #endif - member __.Verbose = a.Contains Verbose - member val Source : SourceArguments = SourceArguments(a.GetResult Source) member __.Destination : DestinationArguments = __.Source.Destination #if cosmos @@ -319,9 +321,59 @@ module EventStoreSource = dop.Release() |> ignore do! Async.Sleep sleepIntervalMs } - open CosmosIngester type StartMode = Starting | Resuming | Overridding - type Syncer(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, maxPendingBatches, commitProgress, ?interval) = + + //// 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) + //let mutable more = true + //while more && readers.HasCapacity do + // match buffer.TryGap() with + // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) + // | None -> more <- false + + let run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { + let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) + let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn + let! initialCheckpointState = checkpoints.Read + let! max = maxInParallel + let! startPos = async { + let mkPos x = EventStore.ClientAPI.Position(x, 0L) + let requestedStartPos = + match spec.start with + | Absolute p -> mkPos p + | Chunk c -> EventStoreSource.posFromChunk c + | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) + | TailOrCheckpoint -> max + | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start + let! startMode, startPos, checkpointFreq = async { + match initialCheckpointState, requestedStartPos with + | Checkpoint.Folds.NotStarted, r -> + if spec.forceRestart then raise <| CmdParser.InvalidArguments ("Cannot specify --forceRestart when no progress yet committed") + do! checkpoints.Start(spec.checkpointInterval, r.CommitPosition) + return Starting, r, spec.checkpointInterval + | Checkpoint.Folds.Running s, _ when not spec.forceRestart -> + return Resuming, mkPos s.state.pos, TimeSpan.FromSeconds(float s.config.checkpointFreqS) + | Checkpoint.Folds.Running _, r -> + do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) + return Overridding, r, spec.checkpointInterval + } + log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m", + startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, + float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) + return startPos } + let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) + readers.AddTail(startPos, max, spec.tailInterval) + let coordinator = CosmosIngestionCoordinator.create (log, cosmosContext, maxWriters, maxPendingBatches, TimeSpan.FromMinutes 1.) + let postStreamSpan : StreamSpan -> unit = coordinator.Submit + let postBatch (pos : EventStore.ClientAPI.Position) xs = + let cp = pos.CommitPosition + coordinator.Submit(cp, checkpoints.Commit cp, Seq.ofArray xs) + do! readers.Pump(postStreamSpan, postBatch) } +#else +module CosmosSource = + open Microsoft.Azure.Documents + open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing + + type Ingester(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, maxPendingBatches, commitProgress, ?interval) = let statsInterval = defaultArg interval (TimeSpan.FromMinutes 1.) member __.Pump() = async { let writerResultLog = log.ForContext() @@ -417,36 +469,8 @@ module EventStoreSource = readers.AddTail(startPos, max, spec.tailInterval) let coordinator = Syncer(log, readers, cosmosContext, maxWriters, maxPendingBatches, checkpoints.Commit) do! coordinator.Pump() } -#else -module CosmosSource = - open Microsoft.Azure.Documents - open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing - type [] CoordinationWork<'Pos> = - | Result of CosmosIngester.Writer.Result - | ProgressResult of Choice - | BatchWithTracking of 'Pos * CosmosIngester.Batch[] - /// Manages writing of progress - /// - Each write attempt is of the newest token - /// - retries until success or a new item is posted - type ProgressWriter() = - let pumpSleepMs = 100 - let mutable lastCompleted = -1 - let mutable latest = None - let result = Event<_>() - [] member __.Result = result.Publish - member __.Post(version,f) = latest <- Some (version,f) - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match latest with - | Some (v,f) when v <> lastCompleted -> - try do! f - lastCompleted <- v - result.Trigger (Choice1Of2 v) - with e -> result.Trigger (Choice2Of2 e) - | _ -> do! Async.Sleep pumpSleepMs } type PendingWork = { batches : int; streams : int } type Coordinator private (cosmosContext, cts : CancellationTokenSource, ?maxWriters, ?interval) = let pumpSleepMs = 100 @@ -547,11 +571,11 @@ module CosmosSource = let coordinator = new Coordinator(cosmosContext, cts) Async.Start(coordinator.Pump log,cts.Token) coordinator - member __.Submit(checkpoint : Async, batches : CosmosIngester.Batch[]) = postBatch checkpoint batches + member __.Submit(checkpoint : Async, batches : StreamItem[]) = postBatch checkpoint batches member __.PendingBatches = pendingBatches interface IDisposable with member __.Dispose() = cts.Cancel() - let createRangeSyncHandler (log:ILogger) (ctx: Core.CosmosContext) (transform : Microsoft.Azure.Documents.Document -> CosmosIngester.Batch seq) = + let createRangeSyncHandler (log:ILogger) (ctx: Core.CosmosContext) (transform : Microsoft.Azure.Documents.Document -> StreamItem seq) = let busyPauseMs = 500 let maxUnconfirmedBatches = 10 let mutable coordinator = Unchecked.defaultof<_> @@ -646,17 +670,17 @@ module CosmosSource = member __.Timestamp = x.c member __.Stream = x.s } - let transformV0 catFilter (v0SchemaDocument: Document) : CosmosIngester.Batch seq = seq { + let transformV0 catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { let parsed = EventV0Parser.parse v0SchemaDocument let streamName = (*if parsed.Stream.Contains '-' then parsed.Stream else "Prefixed-"+*)parsed.Stream if catFilter (category streamName) then - yield { stream = streamName; span = { index = parsed.Index; events = [| parsed |] } } } + yield { stream = streamName; index = parsed.Index; event = parsed } } //#else - let transformOrFilter catFilter (changeFeedDocument: Document) : CosmosIngester.Batch seq = seq { + let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq = seq { for e in DocumentParser.enumEvents changeFeedDocument do if catFilter (category e.Stream) then // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level - yield { stream = e.Stream; span = { index = e.Index; events = [| e |] } } } + yield { stream = e.Stream; index = e.Index; event = e } } //#endif #endif @@ -676,7 +700,7 @@ module Logging = c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning - c.MinimumLevel.Override(typeof.FullName, generalLevel) + c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, LogEventLevel.Information) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let configure (a : Configuration.LoggerSinkConfiguration) : unit = @@ -684,7 +708,7 @@ module Logging = l.WriteTo.Sink(Metrics.RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke - let isWriter = Filters.Matching.FromSource().Invoke + let isWriter = Filters.Matching.FromSource().Invoke let isCheckpointing = Filters.Matching.FromSource().Invoke (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x || isWriter x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) @@ -697,7 +721,11 @@ module Logging = [] let main argv = try let args = CmdParser.parse argv - let log, storeLog = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint +#if cosmos + let log,storeLog = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint +#else + let log,storeLog = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint +#endif let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) let resolveCheckpointStream = @@ -712,7 +740,6 @@ let main argv = Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, storeLog) #if cosmos - let log = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() let auxDiscovery, aux, leaseId, startFromHere, batchSize, lagFrequency = args.BuildChangeFeedParams() #if marveleqx @@ -741,7 +768,7 @@ let main argv = || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e - EventStoreSource.Syncer.Run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (args.MaxWriters, target, args.MaxPendingBatches) resolveCheckpointStream + EventStoreSource.run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (args.MaxWriters, target, args.MaxPendingBatches) resolveCheckpointStream #endif |> Async.RunSynchronously 0 diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 5b8025ee7..af03744f3 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,7 +4,7 @@ Exe netcoreapp2.1 5 - cosmos2 + cosmos1 From 35e100846d8e1168f10c2c1301b35d69900df076 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 16:42:13 +0100 Subject: [PATCH 157/353] Tidy --- equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs | 4 ++-- equinox-projector/Equinox.Projection/Coordination.fs | 4 ++-- equinox-projector/Equinox.Projection/State.fs | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs index ecbdd5aba..c3bfd2494 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs @@ -85,8 +85,8 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Reqs {completed:n0} {mb:n3}GB {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", - results, mb bytes/1024., events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + log.Information("Streams {completed:n0} {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + results, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Coordination.fs index fb33388e2..29992c6fb 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Coordination.fs @@ -40,8 +40,8 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - log.Information("Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Active {busy}/{processors} Completed {completed} Exceptions {exns}", - !cycles, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, busy, capacity,!resultCompleted, !resultExn) + log.Information("Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + !cycles, busy, capacity, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 streams.Dump log abstract member Handle : Message<'R> -> unit diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index d74649be5..8ea573e99 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -172,9 +172,9 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) ready <- ready + 1 readyB <- readyB + sz - log.Information("Synced {synced:n0} In-flight {busy:n0}/{busyMb:n1}MB Queued {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + log.Information("Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) - if busyCats.Any then log.Information("In-flight Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) + if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) From 5accf321237e6c21ad823a5b6c1a910de4b570f8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 16:52:33 +0100 Subject: [PATCH 158/353] Rename to Engine --- equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs | 4 ++-- .../Equinox.Projection/{Coordination.fs => Engine.fs} | 4 ++-- .../Equinox.Projection/Equinox.Projection.fsproj | 2 +- equinox-projector/Equinox.Projection/State.fs | 2 +- equinox-projector/Projector/Program.fs | 2 +- equinox-sync/Sync/Program.fs | 2 +- 6 files changed, 8 insertions(+), 8 deletions(-) rename equinox-projector/Equinox.Projection/{Coordination.fs => Engine.fs} (99%) diff --git a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs index c3bfd2494..12c15d551 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs @@ -2,7 +2,7 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Store -open Equinox.Projection.Coordination +open Equinox.Projection.Engine open Equinox.Projection.State open Serilog open System @@ -85,7 +85,7 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Streams {completed:n0} {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + log.Information("Requests {completed:n0} {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", results, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then diff --git a/equinox-projector/Equinox.Projection/Coordination.fs b/equinox-projector/Equinox.Projection/Engine.fs similarity index 99% rename from equinox-projector/Equinox.Projection/Coordination.fs rename to equinox-projector/Equinox.Projection/Engine.fs index 29992c6fb..9d59e4560 100644 --- a/equinox-projector/Equinox.Projection/Coordination.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -1,4 +1,4 @@ -module Equinox.Projection.Coordination +module Equinox.Projection.Engine open Equinox.Projection.State open Serilog @@ -43,7 +43,6 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = log.Information("Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", !cycles, busy, capacity, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 - streams.Dump log abstract member Handle : Message<'R> -> unit default __.Handle res = match res with @@ -72,6 +71,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = if statsDue () then dumpStats (busy,capacity) streams __.DumpExtraStats() + streams.Dump log abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () diff --git a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj index c26c3d2c8..3035b7d4b 100644 --- a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj +++ b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj @@ -12,7 +12,7 @@ - + diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 8ea573e99..ec8010ceb 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -172,7 +172,7 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) ready <- ready + 1 readyB <- readyB + sz - log.Information("Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index b8501af59..e8f571d45 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -5,7 +5,7 @@ open Confluent.Kafka //#endif open Equinox.Cosmos open Equinox.Cosmos.Projection -open Equinox.Projection.Coordination +open Equinox.Projection.Engine //#if kafka open Equinox.Projection.Codec open Equinox.Store diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index b1034a5a8..4119563e3 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -7,8 +7,8 @@ open Equinox.Cosmos.Projection //#else open Equinox.EventStore //#endif -open Equinox.Projection.Coordination open Equinox.Projection.Cosmos +open Equinox.Projection.Engine open Equinox.Projection.State //#if !eventStore open Equinox.Store From 788dbaa6139c43a04dc73df3183a47bfe6d2ea02 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 16:53:50 +0100 Subject: [PATCH 159/353] Fix --- equinox-projector/Equinox.Projection/Engine.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 9d59e4560..e71e211ad 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -71,7 +71,7 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = if statsDue () then dumpStats (busy,capacity) streams __.DumpExtraStats() - streams.Dump log + streams.Dump log abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () From 650ff9b03cc1b95c15f2cb00f09925caf4c6c61b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 27 Apr 2019 17:00:16 +0100 Subject: [PATCH 160/353] reorder --- equinox-projector/Equinox.Projection/Engine.fs | 4 ++-- equinox-projector/Equinox.Projection/State.fs | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index e71e211ad..2ef21abdd 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -106,13 +106,13 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S let reqs = Dictionary() let mutable count, skipCount = 0, 0 for item in items do - let _stream,streamState = streams.Add item + let stream,streamState = streams.Add item match validVsSkip streamState item with | 0, skip -> skipCount <- skipCount + skip | required, _ -> count <- count + required - reqs.[item.stream] <- item.index+1L + reqs.[stream] <- item.index+1L progressState.AppendBatch((epoch,checkpoint),reqs) work.Enqueue(Added (reqs.Count,skipCount,count)) | AddStream streamSpan -> diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index ec8010ceb..1c80be402 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -172,12 +172,12 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) ready <- ready + 1 readyB <- readyB + sz - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", - synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } From cbf87d2763e2574135a752d30cc2050fd1f1f8b5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Apr 2019 18:58:21 +0100 Subject: [PATCH 161/353] Split read/progress batching from ingestion --- ...roj => Equinox.Cosmos.ProjectionEx.fsproj} | 0 .../Equinox.Projection.Cosmos/Ingestion.fs | 123 ++++--- .../Equinox.Projection.Cosmos/Metrics.fs | 2 +- .../Equinox.Projection.Tests/ProgressTests.fs | 36 +- .../Equinox.Projection/Engine.fs | 316 +++++++++++++----- equinox-projector/Equinox.Projection/State.fs | 87 ++--- equinox-projector/Projector/Program.fs | 46 +-- .../equinox-projector-consumer.sln | 2 +- equinox-sync/Ingest/Program.fs | 3 +- equinox-sync/Sync/EventStoreSource.fs | 30 +- equinox-sync/Sync/Program.fs | 288 +++------------- equinox-sync/Sync/Sync.fsproj | 5 +- 12 files changed, 427 insertions(+), 511 deletions(-) rename equinox-projector/Equinox.Projection.Cosmos/{Equinox.Projection.Cosmos.fsproj => Equinox.Cosmos.ProjectionEx.fsproj} (100%) diff --git a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj similarity index 100% rename from equinox-projector/Equinox.Projection.Cosmos/Equinox.Projection.Cosmos.fsproj rename to equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj diff --git a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs index 12c15d551..d65f09af3 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs @@ -1,11 +1,10 @@ -module Equinox.Projection.Cosmos.Ingestion +module Equinox.Cosmos.Projection.Ingestion open Equinox.Cosmos.Core open Equinox.Cosmos.Store open Equinox.Projection.Engine open Equinox.Projection.State open Serilog -open System open System.Threading let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 @@ -65,19 +64,8 @@ module Writer = | ResultKind.RateLimited | ResultKind.TimedOut | ResultKind.Other -> false | ResultKind.TooLarge | ResultKind.Malformed -> true - //member __.TryGap() : (string*int64*int) option = - // let rec aux () = - // match gap |> Queue.tryDequeue with - // | None -> None - // | Some stream -> - - // match states.[stream].TryGap() with - // | Some (pos,count) -> Some (stream,pos,int count) - // | None -> aux () - // aux () - -type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = - inherit Stats<(int*int)*Writer.Result>(log, maxPendingBatches, statsInterval) +type CosmosStats(log : ILogger, statsInterval) = + inherit Stats<(int*int)*Writer.Result>(log, statsInterval) let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 let mutable events, bytes = 0, 0L @@ -85,20 +73,23 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Requests {completed:n0} {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + log.Information("Completed {completed:n0} {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", results, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0 + resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", !rateLimited, !timedOut, !tooLarge, !malformed, !resultExnOther) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0; events <- 0; bytes <- 0L + rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0 if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - Metrics.dumpRuStats statsInterval log + Equinox.Cosmos.Metrics.dumpRuStats statsInterval log override __.Handle message = base.Handle message match message with - | Message.Result (_stream, Choice1Of2 ((es,bs),r)) -> + | Message.Add (_,_) + | Message.AddStream _ + | Message.Added _ -> () + | Result (_stream, Choice1Of2 ((es,bs),r)) -> events <- events + es bytes <- bytes + int64 bs match r with @@ -113,51 +104,49 @@ type CosmosStats(log : ILogger, maxPendingBatches, statsInterval) = | ResultKind.TooLarge -> category stream |> badCats.Ingest; incr tooLarge | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed | ResultKind.TimedOut -> incr timedOut - | Add _ | AddStream _ | Added _ | ProgressResult _ -> () -module CosmosIngestionCoordinator = - let create (log : Serilog.ILogger, cosmosContext, maxWriters, maxPendingBatches, statsInterval) = - let writerResultLog = log.ForContext() - let trim (writePos : int64 option, batch : StreamSpan) = - let mutable bytesBudget = cosmosPayloadLimit - let mutable count = 0 - let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = - bytesBudget <- bytesBudget - cosmosPayloadBytes y - count <- count + 1 - // Reduce the item count when we don't yet know the write position - count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) - { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } - let project batch = async { - let trimmed = trim batch - try let! res = Writer.write log cosmosContext trimmed - let ctx = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes - return trimmed.stream, Choice1Of2 (ctx,res) - with e -> return trimmed.stream, Choice2Of2 e } - let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = - let applyResultToStreamState = function - | stream, (Choice1Of2 (ctx, Writer.Ok pos)) -> - Some ctx,streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (ctx, Writer.Duplicate pos)) -> - Some ctx,streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (ctx, Writer.PartialDuplicate overage)) -> - Some ctx,streams.InternalUpdate stream overage.index [|overage|] - | stream, (Choice1Of2 (ctx, Writer.PrefixMissing (overage,pos))) -> - Some ctx,streams.InternalUpdate stream pos [|overage|] - | stream, (Choice2Of2 exn) -> - let malformed = Writer.classify exn |> Writer.isMalformed - None,streams.SetMalformed(stream,malformed) - match res with - | Message.Result (s,r) -> - let _ctx,(stream,updatedState) = applyResultToStreamState (s,r) - match updatedState.write with - | Some wp -> - let closedBatches = progressState.MarkStreamProgress(stream, wp) - if closedBatches > 0 then - batches.Release(closedBatches) |> ignore - streams.MarkCompleted(stream,wp) - | None -> - streams.MarkFailed stream - Writer.logTo writerResultLog (s,r) - | _ -> () - let stats = CosmosStats(log, maxPendingBatches, statsInterval) - Coordinator<(int*int)*Writer.Result>.Start(stats, maxPendingBatches, maxWriters, project, handleResult) \ No newline at end of file +let startIngestionEngine (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, statsInterval) = + let writerResultLog = log.ForContext() + let trim (writePos : int64 option, batch : StreamSpan) = + let mutable bytesBudget = cosmosPayloadLimit + let mutable count = 0 + let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = + bytesBudget <- bytesBudget - cosmosPayloadBytes y + count <- count + 1 + // Reduce the item count when we don't yet know the write position + count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) + { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } + let project batch = async { + let trimmed = trim batch + try let! res = Writer.write log cosmosContext trimmed + let ctx = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes + return trimmed.stream, Choice1Of2 (ctx,res) + with e -> return trimmed.stream, Choice2Of2 e } + let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = + let applyResultToStreamState = function + | stream, (Choice1Of2 (ctx, Writer.Ok pos)) -> + Some ctx,streams.InternalUpdate stream pos null + | stream, (Choice1Of2 (ctx, Writer.Duplicate pos)) -> + Some ctx,streams.InternalUpdate stream pos null + | stream, (Choice1Of2 (ctx, Writer.PartialDuplicate overage)) -> + Some ctx,streams.InternalUpdate stream overage.index [|overage|] + | stream, (Choice1Of2 (ctx, Writer.PrefixMissing (overage,pos))) -> + Some ctx,streams.InternalUpdate stream pos [|overage|] + | stream, (Choice2Of2 exn) -> + let malformed = Writer.classify exn |> Writer.isMalformed + None,streams.SetMalformed(stream,malformed) + match res with + | Message.Result (s,r) -> + let _ctx,(stream,updatedState) = applyResultToStreamState (s,r) + match updatedState.write with + | Some wp -> + let closedBatches = progressState.MarkStreamProgress(stream, wp) + if closedBatches > 0 then + batches.Release(closedBatches) |> ignore + streams.MarkCompleted(stream,wp) + | None -> + streams.MarkFailed stream + Writer.logTo writerResultLog (s,r) + | _ -> () + let ingesterStats = CosmosStats(log, statsInterval) + ProjectionEngine<(int*int)*Writer.Result>.Start(ingesterStats, maxPendingBatches, maxWriters, project, handleResult) \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs b/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs index 6b8d9e610..48c5d40ac 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs @@ -1,4 +1,4 @@ -module Equinox.Projection.Cosmos.Metrics +module Equinox.Cosmos.Metrics // TODO move into equinox.cosmos diff --git a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs index cc82c7964..06657c1b5 100644 --- a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs +++ b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs @@ -1,45 +1,43 @@ module ProgressTests open Equinox.Projection.State - open Swensen.Unquote -open Xunit open System.Collections.Generic +open Xunit let mkDictionary xs = Dictionary(dict xs) let [] ``Empty has zero streams pending or progress to write`` () = let sut = ProgressState<_>() - let completed,validatedPos, batches = sut.Validate(fun _ -> None) + let completed = sut.Validate(fun _ -> None) 0 =! completed - None =! validatedPos - 0 =! batches let [] ``Can add multiple batches`` () = let sut = ProgressState<_>() - sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) - sut.AppendBatch(1,mkDictionary ["b",2L; "c",3L]) - let completed,validatedPos, batches = sut.Validate(fun _ -> None) + let noBatchesComplete () = failwith "No bathes should complete" + sut.AppendBatch(noBatchesComplete, mkDictionary ["a",1L; "b",2L]) + sut.AppendBatch(noBatchesComplete, mkDictionary ["b",2L; "c",3L]) + let completed = sut.Validate(fun _ -> None) 0 =! completed - None =! validatedPos - 2 =! batches let [] ``Marking Progress Removes batches and updates progress`` () = let sut = ProgressState<_>() - sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) + let callbacks = ref 0 + let complete () = incr callbacks + sut.AppendBatch(complete, mkDictionary ["a",1L; "b",2L]) sut.MarkStreamProgress("a",1L) |> ignore sut.MarkStreamProgress("b",1L) |> ignore - let completed, validatedPos, batches = sut.Validate(fun _ -> None) + let completed = sut.Validate(fun _ -> None) 0 =! completed - None =! validatedPos - 1 =! batches + 1 =! !callbacks let [] ``Marking progress is not persistent`` () = let sut = ProgressState<_>() - sut.AppendBatch(0, mkDictionary ["a",1L]) + let callbacks = ref 0 + let complete () = incr callbacks + sut.AppendBatch(complete, mkDictionary ["a",1L]) sut.MarkStreamProgress("a",2L) |> ignore - sut.AppendBatch(1, mkDictionary ["a",1L; "b",2L]) - let completed, validatedPos, batches = sut.Validate(fun _ -> None) + sut.AppendBatch(complete, mkDictionary ["a",1L; "b",2L]) + let completed = sut.Validate(fun _ -> None) 0 =! completed - Some 0 =! validatedPos - 1 =! batches \ No newline at end of file + 1 =! !callbacks \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 2ef21abdd..00ff04753 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -7,41 +7,24 @@ open System.Collections.Concurrent open System.Collections.Generic open System.Threading +type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } + [] type Message<'R> = - /// Enqueue a batch of items with supplied tag and progress marking function - | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq + /// Enqueue a batch of items with supplied progress marking function + | Add of markCompleted: (unit -> unit) * items: StreamItem[] | AddStream of StreamSpan - /// Log stats about an ingested batch + /// Feed stats about an ingested batch to relevant listeners | Added of streams: int * skip: int * events: int - /// Result of processing on stream - specified number of items or threw `exn` + /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> - /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` - | ProgressResult of Choice -type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = - let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None - let progCommitFails, progCommits = ref 0, ref 0 +type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (busy,capacity) (streams : StreamStates) = - if !progCommitFails <> 0 || !progCommits <> 0 then - match comittedEpoch with - | None -> - log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) - | Some committed when !progCommitFails <> 0 -> - log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) - | Some committed -> - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) - progCommits := 0; progCommitFails := 0 - else - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + let dumpStats (available,maxDop) = log.Information("Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, busy, capacity, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : Message<'R> -> unit default __.Handle res = @@ -56,45 +39,28 @@ type Stats<'R>(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = incr resultCompleted | Result (_stream, Choice2Of2 _) -> incr resultExn - | ProgressResult (Choice1Of2 epoch) -> - incr progCommits - comittedEpoch <- Some epoch - | ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progCommitFails - member __.HandleValidated(epoch, pendingBatches) = - incr cycles - pendingBatchCount <- pendingBatches - validatedEpoch <- epoch - member __.HandleCommitted epoch = - comittedEpoch <- epoch - member __.TryDump(busy,capacity,streams) = + member __.TryDump((available,maxDop),streams : StreamStates) = if statsDue () then - dumpStats (busy,capacity) streams + dumpStats (available,maxDop) __.DumpExtraStats() streams.Dump log abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () -/// Single instance per Source; Coordinates -/// a) ingestion of events -/// b) execution of projection/ingestion work -/// c) writing of progress -/// d) reporting of state -/// The key bit that's managed externally is the reading/accepting of incoming data -type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * StreamSpan -> Async>, handleResult) = +/// Consolidates ingested events into streams; coordinates dispatching of these in priority dictated by the needs of the checkpointing approach in force +/// a) does not itself perform any readin activities +/// b) manages writing of progress +/// x) periodically reports state (with hooks for ingestion engines to report same) +type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, handleResult) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() let batches = new SemaphoreSlim(maxPendingBatches) let work = ConcurrentQueue>() let streams = StreamStates() - let dispatcher = Dispatcher(processorDop) let progressState = ProgressState() - let progressWriter = ProgressWriter<_>() member private __.Pump(stats : Stats<'R>) = async { - use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) - Async.Start(progressWriter.Pump(), cts.Token) Async.Start(dispatcher.Pump(), cts.Token) let validVsSkip (streamState : StreamState) (item : StreamItem) = match streamState.write, item.index + 1L with @@ -102,23 +68,23 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S | _ -> 1, 0 let handle x = match x with - | Add (epoch, checkpoint, items) -> + | Add (checkpoint, items) -> let reqs = Dictionary() let mutable count, skipCount = 0, 0 for item in items do - let stream,streamState = streams.Add item + let stream,streamState = streams.Add(item.stream,item.index,item.event) match validVsSkip streamState item with | 0, skip -> skipCount <- skipCount + skip | required, _ -> count <- count + required reqs.[stream] <- item.index+1L - progressState.AppendBatch((epoch,checkpoint),reqs) + progressState.AppendBatch(checkpoint,reqs) work.Enqueue(Added (reqs.Count,skipCount,count)) | AddStream streamSpan -> let _stream,_streamState = streams.Add(streamSpan,false) work.Enqueue(Added (1,0,streamSpan.span.events.Length)) // Yes, need to compute skip - | Added _ | ProgressResult _ -> + | Added _ -> () | Result _ as r -> handleResult (streams, progressState, batches) r @@ -127,47 +93,237 @@ type Coordinator<'R>(maxPendingBatches, processorDop, project : int64 option * S // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let completedBatches, validatedPos, pendingBatches = progressState.Validate(streams.TryGetStreamWritePos) + let completedBatches = progressState.Validate(streams.TryGetStreamWritePos) if completedBatches > 0 then batches.Release(completedBatches) |> ignore - stats.HandleValidated(Option.map fst validatedPos, pendingBatches) - validatedPos |> Option.iter progressWriter.Post - stats.HandleCommitted progressWriter.CommittedEpoch // 3. After that, provision writers queue - let capacity = dispatcher.Capacity + let capacity,_ = dispatcher.AvailableCapacity if capacity <> 0 then let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) - for batch in work do - dispatcher.Enqueue(project batch) + let xs = (Seq.ofArray work).GetEnumerator() + let mutable ok = true + while xs.MoveNext() && ok do + let! succeeded = dispatcher.TryAdd(project xs.Current) + ok <- succeeded // 4. Periodically emit status info - let busy = processorDop - dispatcher.Capacity - stats.TryDump(busy,processorDop,streams) + stats.TryDump(dispatcher.AvailableCapacity,streams) do! Async.Sleep sleepIntervalMs } static member Start<'R>(stats, maxPendingBatches, processorDop, project, handleResult) = - let instance = new Coordinator<'R>(maxPendingBatches, processorDop, project, handleResult) + let dispatcher = Dispatcher(processorDop) + let instance = new ProjectionEngine<'R>(maxPendingBatches, dispatcher, project, handleResult) Async.Start <| instance.Pump(stats) instance - static member Start(log, maxPendingBatches, processorDop, project : StreamSpan -> Async, statsInterval) = - let project (_maybeWritePos, batch) = async { - try let! count = project batch - return batch.stream, Choice1Of2 (batch.span.index + int64 count) - with e -> return batch.stream, Choice2Of2 e } - let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) = function - | Result (stream, Choice1Of2 index) -> - match progressState.MarkStreamProgress(stream,index) with 0 -> () | batchesCompleted -> batches.Release(batchesCompleted) |> ignore - streams.MarkCompleted(stream,index) - | Result (stream, Choice2Of2 _) -> - streams.MarkFailed stream - | _ -> () - let stats = Stats(log, maxPendingBatches, statsInterval) - Coordinator.Start(stats, maxPendingBatches, processorDop, project, handleResult) - member __.Submit(epoch, markBatchCompleted, events) = async { + member __.TrySubmit(markCompleted, events) = async { + let! got = batches.Await(TimeSpan.Zero) + if got then work.Enqueue <| Add (markCompleted, events); return true + else return false } + + member __.Submit(markCompleted, events) = async { let! _ = batches.Await() - work.Enqueue <| Add (epoch, markBatchCompleted, Array.ofSeq events) + work.Enqueue <| Add (markCompleted, Array.ofSeq events) return maxPendingBatches-batches.CurrentCount,maxPendingBatches } + member __.AllStreams = streams.All + member __.Submit(streamSpan) = work.Enqueue <| AddStream streamSpan + member __.Stop() = + cts.Cancel() + +let startProjectionEngine (log, maxPendingBatches, processorDop, project : StreamSpan -> Async, statsInterval) = + let project (_maybeWritePos, batch) = async { + try let! count = project batch + return batch.stream, Choice1Of2 (batch.span.index + int64 count) + with e -> return batch.stream, Choice2Of2 e } + let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) = function + | Result (stream, Choice1Of2 index) -> + match progressState.MarkStreamProgress(stream,index) with 0 -> () | batchesCompleted -> batches.Release(batchesCompleted) |> ignore + streams.MarkCompleted(stream,index) + | Result (stream, Choice2Of2 _) -> + streams.MarkFailed stream + | _ -> () + let stats = Stats(log, statsInterval) + ProjectionEngine.Start(stats, maxPendingBatches, processorDop, project, handleResult) + +type Sem(max) = + let inner = new SemaphoreSlim(max) + member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore + member __.State = max-inner.CurrentCount,max + member __.Await() = inner.Await() |> Async.Ignore + member __.HasCapacity = inner.CurrentCount > 0 + member __.TryAwait(?timeout) = inner.Await(?timeout=timeout) + +type TrancheStreamBuffer() = + let states = Dictionary() + let merge stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + + member __.Merge(items : StreamItem seq) = + for item in items do + merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = Array.singleton item.event } |] } + + member __.Take(set : ISet<_>) = seq { + for x in states do + if set.Contains x.Key then + states.Remove x.Key |> ignore + yield x } + + member __.Dump(log : ILogger) = + let mutable waiting, waitingB = 0, 0L + let waitingCats, waitingStreams = CatStats(), CatStats() + for KeyValue (stream,state) in states do + let sz = int64 state.Size + waitingCats.Ingest(category stream) + waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) + waiting <- waiting + 1 + waitingB <- waitingB + sz + log.Information("Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) + if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) + if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) + +/// Manages writing of progress +/// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) +/// - retries until success or a new item is posted +type ProgressWriter<'Res when 'Res: equality>() = + let pumpSleepMs = 100 + let due = expiredMs 5000L + let mutable committedEpoch = None + let mutable validatedPos = None + let result = Event>() + [] member __.Result = result.Publish + member __.Post(version,f) = + Volatile.Write(&validatedPos,Some (version,f)) + member __.CommittedEpoch = Volatile.Read(&committedEpoch) + member __.Pump() = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + match Volatile.Read &validatedPos with + | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> + try do! f + Volatile.Write(&committedEpoch, Some v) + result.Trigger (Choice1Of2 v) + with e -> result.Trigger (Choice2Of2 e) + | _ -> do! Async.Sleep pumpSleepMs } + +[] +type SeriesMessage = + | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq + | Added of streams: int * events: int + /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` + | ProgressResult of Choice + +type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = + let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None + let progCommitFails, progCommits = ref 0, ref 0 + let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + let dumpStats (available,maxDop) = + if !progCommitFails <> 0 || !progCommits <> 0 then + match comittedEpoch with + | None -> + log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) + | Some committed when !progCommitFails <> 0 -> + log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) + | Some committed -> + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) + progCommits := 0; progCommitFails := 0 + else + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + log.Information("Ingest Cycles {cycles} Batches {batches} ({streams:n0}s {events:n0})", + !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; + member __.Handle : SeriesMessage -> unit = function + | Add _ -> () // Enqueuing of an event is not interesting - we assume it'll get processed and mapped to an `Added` in the same cycle + | ProgressResult (Choice1Of2 epoch) -> + incr progCommits + comittedEpoch <- Some epoch + | ProgressResult (Choice2Of2 (_exn : exn)) -> + incr progCommitFails + | Added (streams,events) -> + incr batchesPended + streamsPended := !streamsPended + streams + eventsPended := !eventsPended + events + member __.HandleValidated(epoch, pendingBatches) = + incr cycles + pendingBatchCount <- pendingBatches + validatedEpoch <- epoch + member __.HandleCommitted epoch = + comittedEpoch <- epoch + member __.TryDump((available,maxDop),streams : TrancheStreamBuffer) = + if statsDue () then + dumpStats (available,maxDop) + streams.Dump log + +/// Holds batches away from Core processing to limit in-flight processsing +type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, maxSubmissions, statsInterval : TimeSpan, ?pumpDelayMs) = + let cts = new CancellationTokenSource() + let work = ConcurrentQueue() + let read = new Sem(maxQueued) + let write = new Sem(maxSubmissions) + let streams = TrancheStreamBuffer() + let pending = Queue<_>() + let mutable validatedPos = None + let progressWriter = ProgressWriter<_>() + let stats = TrancheStats(log, maxQueued, statsInterval) + let pumpDelayMs = defaultArg pumpDelayMs 5 + + member private __.Pump() = async { + let handle x = + match x with + | Add (epoch, checkpoint, items) -> + let items = Array.ofSeq items + streams.Merge items + let markCompleted () = + write.Release() + validatedPos <- Some (epoch,checkpoint) + let streams = HashSet() + for x in items do streams.Add x.stream |> ignore + work.Enqueue(Added (streams.Count,items.Length)) + pending.Enqueue((markCompleted,items)) + | Added _ | ProgressResult _ -> () + use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) + Async.Start(progressWriter.Pump(), cts.Token) + while not cts.IsCancellationRequested do + work |> ConcurrentQueue.drain handle + let mutable ingesterAccepting = true + // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted + while pending.Count <> 0 && write.HasCapacity && ingesterAccepting do + let markCompleted, events = pending.Peek() + let! sumbitted = ingester.TrySubmit(markCompleted, events) + if sumbitted then + pending.Dequeue() |> ignore + // mark off a write as being in progress + do! write.Await() + else + ingesterAccepting <- false + // 2. Update any progress into the stats + stats.HandleValidated(Option.map fst validatedPos, fst write.State) + validatedPos |> Option.iter progressWriter.Post + stats.HandleCommitted progressWriter.CommittedEpoch + // 4. Periodically emit status info + stats.TryDump(write.State,streams) + do! Async.Sleep pumpDelayMs } + + /// Awaits space in `read` to limit reading ahead - yields present state of Read and Write phases + member __.Submit(epoch, markBatchCompleted, events) = async { + do! read.Await() + work.Enqueue <| Add (epoch, markBatchCompleted, events) + return read.State } + + static member Start<'R>(log, ingester, maxRead, maxWrite, statsInterval) = + let instance = new TrancheEngine<'R>(log, ingester, maxRead, maxWrite, statsInterval) + Async.Start <| instance.Pump() + instance + member __.Stop() = cts.Cancel() \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 1c80be402..e7c979d4f 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -4,7 +4,6 @@ open Serilog open System.Collections.Generic open System.Diagnostics open System.Threading -open System open System.Collections.Concurrent let every ms f = @@ -24,7 +23,6 @@ let arrayBytes (x:byte[]) = if x = null then 0 else x.Length let mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] -type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } type [] StreamSpan = { stream: string; span: Span } type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with @@ -99,11 +97,13 @@ type CatStats() = member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd type StreamStates() = + let mutable streams = Set.empty let states = Dictionary() let update stream (state : StreamState) = match states.TryGetValue stream with | false, _ -> states.Add(stream, state) + streams <- streams.Add stream stream, state | true, current -> let updated = StreamState.combine current state @@ -129,9 +129,10 @@ type StreamStates() = let markNotBusy stream = busy.Remove stream |> ignore + member __.All = streams member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } - member __.Add(item: StreamItem, ?isMalformed) = - updateWritePos item.stream (defaultArg isMalformed false) None [| { index = item.index; events = [| item.event |] } |] + member __.Add(stream, index, event, ?isMalformed) = + updateWritePos stream (defaultArg isMalformed false) None [| { index = index; events = [| event |] } |] member __.Add(batch: StreamSpan, isMalformed) = updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] member __.SetMalformed(stream,isMalformed) = @@ -172,20 +173,30 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) ready <- ready + 1 readyB <- readyB + sz + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", - synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) -type [] internal Chunk<'Pos> = { pos: 'Pos; streamToRequiredIndex : Dictionary } + //member __.TryGap() : (string*int64*int) option = + // let rec aux () = + // match gap |> Queue.tryDequeue with + // | None -> None + // | Some stream -> + + // match states.[stream].TryGap() with + // | Some (pos,count) -> Some (stream,pos,int count) + // | None -> aux () + // aux () -type ProgressState<'Pos>(?currentPos : 'Pos) = +type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } + +type ProgressState<'Pos>() = let pending = Queue<_>() - let mutable validatedPos = currentPos - member __.AppendBatch(pos, reqs : Dictionary) = - pending.Enqueue { pos = pos; streamToRequiredIndex = reqs } + member __.AppendBatch(markCompleted, reqs : Dictionary) = + pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } member __.MarkStreamProgress(stream, index) = for x in pending do match x.streamToRequiredIndex.TryGetValue stream with @@ -194,9 +205,8 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = let headIsComplete () = pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 let mutable completed = 0 while headIsComplete () do + let item = pending.Dequeue() in item.markCompleted() completed <- completed + 1 - let headBatch = pending.Dequeue() - validatedPos <- Some headBatch.pos completed member __.ScheduledOrder getStreamQueueLength = let raw = seq { @@ -208,7 +218,7 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = if streams.Add s then yield s,(batch,getStreamQueueLength s) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst - member __.Validate tryGetStreamWritePos : int * 'Pos option * int = + member __.Validate tryGetStreamWritePos = let rec aux completed = if pending.Count = 0 then completed else let batch = pending.Peek() @@ -221,15 +231,12 @@ type ProgressState<'Pos>(?currentPos : 'Pos) = if batch.streamToRequiredIndex.Count <> 0 then completed else - let headBatch = pending.Dequeue() - validatedPos <- Some headBatch.pos + let item = pending.Dequeue() in item.markCompleted() aux (completed + 1) - let completed = aux 0 - completed, validatedPos, pending.Count + aux 0 /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher<'R>(maxDop) = - let cancellationCheckInterval = TimeSpan.FromMilliseconds 5. let work = new BlockingCollection<_>(ConcurrentQueue<_>()) let result = Event<'R>() let dop = new SemaphoreSlim(maxDop) @@ -238,37 +245,15 @@ type Dispatcher<'R>(maxDop) = result.Trigger res dop.Release() |> ignore } [] member __.Result = result.Publish - member __.Capacity = dop.CurrentCount - member __.Enqueue item = work.Add item + member __.AvailableCapacity = + let available = dop.CurrentCount + 1 + available,maxDop + member __.TryAdd(item,?timeout) = async { + let! got = dop.Await(?timeout=timeout) + if got then + work.Add(item) + return got} member __.Pump () = async { let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - let! got = dop.Await(cancellationCheckInterval) - if got then - let mutable item = Unchecked.defaultof> - if work.TryTake(&item, cancellationCheckInterval) then Async.Start(dispatch item) - else dop.Release() |> ignore } - -/// Manages writing of progress -/// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) -/// - retries until success or a new item is posted -type ProgressWriter<'Res when 'Res: equality>() = - let pumpSleepMs = 100 - let due = expiredMs 5000L - let mutable committedEpoch = None - let mutable validatedPos = None - let result = Event>() - [] member __.Result = result.Publish - member __.Post(version,f) = - Volatile.Write(&validatedPos,Some (version,f)) - member __.CommittedEpoch = Volatile.Read(&committedEpoch) - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match Volatile.Read &validatedPos with - | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> - try do! f - Volatile.Write(&committedEpoch, Some v) - result.Trigger (Choice1Of2 v) - with e -> result.Trigger (Choice2Of2 e) - | _ -> do! Async.Sleep pumpSleepMs } \ No newline at end of file + for item in work.GetConsumingEnumerable ct do + Async.Start(dispatch item) } \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index e8f571d45..c6f3db823 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -5,13 +5,13 @@ open Confluent.Kafka //#endif open Equinox.Cosmos open Equinox.Cosmos.Projection -open Equinox.Projection.Engine +open Equinox.Projection.State //#if kafka open Equinox.Projection.Codec open Equinox.Store open Jet.ConfluentKafka.FSharp //#else -open Equinox.Projection.State +open Equinox.Projection.Engine //#endif open Microsoft.Azure.Documents.ChangeFeedProcessor open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing @@ -154,7 +154,7 @@ let run (log : ILogger) discovery connectionPolicy source do! Async.AwaitKeyboardInterrupt() } //#if kafka -let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broker, topic) = +let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = CompressionType.Lz4) let producer = KafkaProducer.Create(Log.Logger, cfg, topic) @@ -175,23 +175,29 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_busyPause,_project) (broke } ChangeFeedObserver.Create(log, projectBatch, dispose = disposeProducer) //#else -let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) () = - let mutable coordinator = Unchecked.defaultof> - let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok - let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let pt = Stopwatch.StartNew() - // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved - let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } - let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - let! index,max = coordinator.Submit(epoch,checkpoint,seq { for x in Seq.collect DocumentParser.enumEvents docs -> { stream = x.Stream; index = x.Index; event = x } }) - log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Ingest {index}/{max} {p:n1}s", - epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., index, max, (let e = pt.Elapsed in e.TotalSeconds)) - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - } - let init rangeLog = coordinator <- Coordinator<_>.Start(rangeLog, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) - let dispose () = coordinator.Stop() - ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) +let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) = + let projectionEngine = startProjectionEngine (log, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) + fun () -> + let mutable trancheEngine = Unchecked.defaultof> + let init rangeLog = trancheEngine <- TrancheEngine.Start (rangeLog, projectionEngine, maxPendingBatches, processorDop, TimeSpan.FromMinutes 1.) + let ingest epoch checkpoint docs = + let events = Seq.collect DocumentParser.enumEvents docs + let items = seq { for x in events -> { stream = x.Stream; index = x.Index; event = x } } + trancheEngine.Submit(epoch, checkpoint, items) + let dispose () = trancheEngine.Stop() + let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok + let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 + // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved + let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } + let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time + log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Post {pt:n3}s {cur}/{max}", + epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., + let e = pt.Elapsed in e.TotalSeconds, cur, max) + sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + } + ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) //#endif // Illustrates how to emit direct to the Console using Serilog diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index 18288cf3a..8800cd964 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -22,7 +22,7 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "..\equinox-sync\I EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Codec", "Equinox.Projection.Codec\Equinox.Projection.Codec.fsproj", "{AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Cosmos", "Equinox.Projection.Cosmos\Equinox.Projection.Cosmos.fsproj", "{2071A2C9-B5C8-4143-B437-6833666D0ACA}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos.ProjectionEx", "Equinox.Projection.Cosmos\Equinox.Cosmos.ProjectionEx.fsproj", "{2071A2C9-B5C8-4143-B437-6833666D0ACA}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs index 78e40253c..77602542b 100644 --- a/equinox-sync/Ingest/Program.fs +++ b/equinox-sync/Ingest/Program.fs @@ -320,10 +320,11 @@ let main argv = || e.EventStreamId.StartsWith("$") || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.EndsWith("_checkpoint") + || e.EventStreamId.StartsWith("marvel_bookmark_") || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused || e.EventStreamId = "ReloadBatchId" // does not start at 0 || e.EventStreamId = "PurchaseOrder-5791" // Too large - || e.EventStreamId.EndsWith("_checkpoint") || not (catFilter e.EventStreamId) -> None | e -> EventStoreSource.tryToBatch e Coordinator.Run log source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerCount, readerQueueLen) |> Async.RunSynchronously diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index dbb587253..9c8929a99 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -1,9 +1,7 @@ module SyncTemplate.EventStoreSource open Equinox.Store // AwaitTaskCorrect -open Equinox.Projection.Cosmos -open Equinox.Projection.Cosmos.Ingestion -open Equinox.Projection.State +open Equinox.Projection open EventStore.ClientAPI open System open Serilog // NB Needs to shadow ILogger @@ -14,20 +12,14 @@ open System.Collections.Generic type EventStore.ClientAPI.RecordedEvent with member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) -let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata +let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = State.arrayBytes x.Data + State.arrayBytes x.Metadata let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 -let tryToBatch (e : RecordedEvent) : StreamItem option = - let eb = recPayloadBytes e - if eb > cosmosPayloadLimit then - Log.Error("ES Event Id {eventId} (#{index} in {stream}, type {type}) size {eventSize} exceeds Cosmos ingestion limit {maxCosmosBytes}", - e.EventId, e.EventNumber, e.EventStreamId, e.EventType, eb, cosmosPayloadLimit) - None - else - let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata - let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data - let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ - Some { stream = e.EventStreamId; index = e.EventNumber; event = event} +let toIngestionItem (e : RecordedEvent) : Engine.StreamItem = + let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata + let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data + let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ + { stream = e.EventStreamId; index = e.EventNumber; event = event} let private mb x = float x / 1024. / 1024. @@ -121,7 +113,7 @@ let establishMax (conn : IEventStoreConnection) = async { Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 return Option.get max } -let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : StreamSpan -> unit) = +let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : State.StreamSpan -> unit) = let rec fetchFrom pos limit = async { let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect @@ -139,7 +131,7 @@ let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int type [] PullResult = Exn of exn: exn | Eof | EndOfTranche let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) - (range:Range, once) (tryMapEvent : ResolvedEvent -> StreamItem option) (postBatch : Position -> StreamItem[] -> Async) = + (range:Range, once) (tryMapEvent : ResolvedEvent -> Engine.StreamItem option) (postBatch : Position -> Engine.StreamItem[] -> Async) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec aux () = async { let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -150,9 +142,9 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length let! (cur,max) = postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n0}ms {cur}/{max}", + Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n3}s {cur}/{max}", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, batches.Length, postSw.ElapsedMilliseconds, cur, max) + batchEvents, usedCats, usedStreams, batches.Length, let e = postSw.Elapsed in e.TotalSeconds, cur, max) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else sw.Restart() // restart the clock as we hand off back to the Reader return! aux () } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 4119563e3..b7a2eaa18 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -1,13 +1,12 @@ module SyncTemplate.Program open Equinox.Cosmos -open Equinox.Projection.Cosmos.Ingestion //#if !eventStore open Equinox.Cosmos.Projection +open Equinox.Cosmos.Projection.Ingestion //#else open Equinox.EventStore //#endif -open Equinox.Projection.Cosmos open Equinox.Projection.Engine open Equinox.Projection.State //#if !eventStore @@ -19,8 +18,10 @@ open System open System.Collections.Generic //#else open System.Diagnostics -//#endif open System.Threading +open Equinox.Cosmos.Projection.Ingestion + +//#endif //#if eventStore type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint @@ -59,6 +60,9 @@ module CmdParser = | [] Verbose | [] FromTail | [] BatchSize of int + | [] MaxPendingBatches of int + | [] MaxProcessing of int + | [] MaxWriters of int #if cosmos | [] ChangeFeedVerbose | [] LeaseCollectionSource of string @@ -68,8 +72,6 @@ module CmdParser = | [] VerboseConsole | [] ForceRestart | [] MinBatchSize of int - | [] MaxPendingBatches of int - | [] MaxWriters of int | [] Position of int64 | [] Chunk of int | [] Percent of float @@ -83,6 +85,9 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Verbose -> "request Verbose Logging. Default: off" + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" + | MaxProcessing _ -> "Maximum number of batches to process concurrently. Default: 128" + | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" #if cosmos | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." @@ -97,8 +102,6 @@ module CmdParser = | BatchSize _ -> "maximum item count to request from feed. Default: 4096" | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 128" - | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" @@ -109,6 +112,9 @@ module CmdParser = and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.Verbose = a.Contains Verbose + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) + member __.MaxProcessing = a.GetResult(MaxProcessing,128) + member __.MaxWriters = a.GetResult(MaxWriters,512) #if cosmos member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName @@ -120,8 +126,6 @@ module CmdParser = member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,128) - member __.MaxWriters = a.GetResult(MaxWriters,512) member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.StreamReaders = a.GetResult(StreamReaders,8) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds @@ -330,7 +334,7 @@ module EventStoreSource = // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) // | None -> more <- false - let run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { + let run (log : Serilog.ILogger) (conn, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn let! initialCheckpointState = checkpoints.Read @@ -362,254 +366,38 @@ module EventStoreSource = return startPos } let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) readers.AddTail(startPos, max, spec.tailInterval) - let coordinator = CosmosIngestionCoordinator.create (log, cosmosContext, maxWriters, maxPendingBatches, TimeSpan.FromMinutes 1.) - let postStreamSpan : StreamSpan -> unit = coordinator.Submit + let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) + let postStreamSpan : StreamSpan -> unit = failwith "TODO" // coordinator.Submit let postBatch (pos : EventStore.ClientAPI.Position) xs = let cp = pos.CommitPosition - coordinator.Submit(cp, checkpoints.Commit cp, Seq.ofArray xs) + trancheEngine.Submit(cp, checkpoints.Commit cp, xs) do! readers.Pump(postStreamSpan, postBatch) } #else module CosmosSource = open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing - type Ingester(log : Serilog.ILogger, readers : TailAndPrefixesReader, cosmosContext, maxWriters, maxPendingBatches, commitProgress, ?interval) = - let statsInterval = defaultArg interval (TimeSpan.FromMinutes 1.) - member __.Pump() = async { - let writerResultLog = log.ForContext() - let trim (writePos : int64 option, batch : StreamSpan) = - let mutable bytesBudget = cosmosPayloadLimit - let mutable count = 0 - let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = - bytesBudget <- bytesBudget - cosmosPayloadBytes y - count <- count + 1 - // Reduce the item count when we don't yet know the write position - count <= (if Option.isNone writePos then 10 else 4096) && (bytesBudget >= 0 || count = 1) - { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } - let project batch = async { - let trimmed = trim batch - try let! res = Writer.write log cosmosContext trimmed - let ctx = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes - return trimmed.stream, Choice1Of2 (ctx,res) - with e -> return trimmed.stream, Choice2Of2 e } - let stats = CosmosStats(log, maxPendingBatches, statsInterval) - let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = - let applyResultToStreamState = function - | stream, (Choice1Of2 (ctx, Writer.Ok pos)) -> - Some ctx,streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (ctx, Writer.Duplicate pos)) -> - Some ctx,streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (ctx, Writer.PartialDuplicate overage)) -> - Some ctx,streams.InternalUpdate stream overage.index [|overage|] - | stream, (Choice1Of2 (ctx, Writer.PrefixMissing (overage,pos))) -> - Some ctx,streams.InternalUpdate stream pos [|overage|] - | stream, (Choice2Of2 exn) -> - let malformed = Writer.classify exn |> Writer.isMalformed - None,streams.SetMalformed(stream,malformed) - match res with - | Message.Result (s,r) -> - let ctx,(stream,updatedState) = applyResultToStreamState (s,r) - match updatedState.write with - | Some wp -> - let closedBatches = progressState.MarkStreamProgress(stream, wp) - if closedBatches > 0 then - batches.Release(closedBatches) |> ignore - streams.MarkCompleted(stream,wp) - | None -> - streams.MarkFailed stream - Writer.logTo writerResultLog (s,r) - | _ -> () - let coordinator = Coordinator<(int*int)*Writer.Result>.Start(stats, maxPendingBatches, maxWriters, project, handleResult) - let pumpReaders = - let postStreamSpan : StreamSpan -> unit = coordinator.Submit - let postBatch (pos : EventStore.ClientAPI.Position) xs = - let cp = pos.CommitPosition - coordinator.Submit(cp, commitProgress cp, Seq.ofArray xs) - readers.Pump(postStreamSpan, postBatch) - do! pumpReaders } - - //// 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - //let mutable more = true - //while more && readers.HasCapacity do - // match buffer.TryGap() with - // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - // | None -> more <- false - - static member Run (log : Serilog.ILogger) (conn, spec, tryMapEvent) (maxWriters, cosmosContext, maxPendingBatches) resolveCheckpointStream = async { - let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) - let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn - let! initialCheckpointState = checkpoints.Read - let! max = maxInParallel - let! startPos = async { - let mkPos x = EventStore.ClientAPI.Position(x, 0L) - let requestedStartPos = - match spec.start with - | Absolute p -> mkPos p - | Chunk c -> EventStoreSource.posFromChunk c - | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) - | TailOrCheckpoint -> max - | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start - let! startMode, startPos, checkpointFreq = async { - match initialCheckpointState, requestedStartPos with - | Checkpoint.Folds.NotStarted, r -> - if spec.forceRestart then raise <| CmdParser.InvalidArguments ("Cannot specify --forceRestart when no progress yet committed") - do! checkpoints.Start(spec.checkpointInterval, r.CommitPosition) - return Starting, r, spec.checkpointInterval - | Checkpoint.Folds.Running s, _ when not spec.forceRestart -> - return Resuming, mkPos s.state.pos, TimeSpan.FromSeconds(float s.config.checkpointFreqS) - | Checkpoint.Folds.Running _, r -> - do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) - return Overridding, r, spec.checkpointInterval - } - log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m", - startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, - float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) - return startPos } - let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) - readers.AddTail(startPos, max, spec.tailInterval) - let coordinator = Syncer(log, readers, cosmosContext, maxWriters, maxPendingBatches, checkpoints.Commit) - do! coordinator.Pump() } - - - type PendingWork = { batches : int; streams : int } - type Coordinator private (cosmosContext, cts : CancellationTokenSource, ?maxWriters, ?interval) = - let pumpSleepMs = 100 - let maxWriters = defaultArg maxWriters 256 - let statsIntervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let work = System.Collections.Concurrent.ConcurrentQueue() - let buffer = CosmosIngester.StreamStates() - let progressWriter = ProgressWriter() - let syncState = ProgressBatcher.State() - let mutable epoch = 0 - let postBatch pos xs = - let batchStamp = Interlocked.Increment &epoch - work.Enqueue(CoordinationWork.BatchWithTracking ((batchStamp,pos),xs)) - let postWriteResult = work.Enqueue << CoordinationWork.Result - let postWriteProgressResult = work.Enqueue << CoordinationWork.ProgressResult - let mutable pendingBatches = 0 - member __.Pump(log : ILogger) = async { - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log cosmosContext, maxWriters) - let writerResultLog = log.ForContext() - use _ = writers.Result.Subscribe postWriteResult - use _ = progressWriter.Result.Subscribe postWriteProgressResult - let! _ = Async.StartChild <| writers.Pump() - let! _ = Async.StartChild <| progressWriter.Pump() - let! ct = Async.CancellationToken - let mutable bytesPended = 0L - let resultsHandled, workPended, eventsPended = ref 0, ref 0, ref 0 - let rateLimited, timedOut, malformed = ref 0, ref 0, ref 0 - let progressFailed, progressWritten = ref 0, ref 0 - let mutable progressEpoch = None - let badCats = CosmosIngester.CatStats() - let dumpStats () = - if !rateLimited <> 0 || !timedOut <> 0 || !malformed <> 0 then - log.Warning("Writer Exceptions {rateLimited} rate-limited, {timedOut} timed out, {malformed} malformed",!rateLimited, !timedOut, !malformed) - rateLimited := 0; timedOut := 0; malformed := 0 - if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - let tl = if !workPended = 0 && !eventsPended = 0 && !resultsHandled = 0 then Events.LogEventLevel.Debug else Events.LogEventLevel.Information - log.Write(tl, "Writer Throughput {queued} requests, {events} events; Completed {completed} reqs; Egress {gb:n3}GB", - !workPended, !eventsPended,!resultsHandled, mb bytesPended / 1024.) - resultsHandled := 0; workPended := 0; eventsPended := 0 - if !progressFailed <> 0 || !progressWritten <> 0 then - match progressEpoch with - | None -> log.Error("Progress writing failing: {failures} failures", !progressWritten, !progressFailed) - | Some epoch -> - if !progressFailed <> 0 then log.Warning("Progress Epoch {epoch} ({updates} updates, {failures} failures", epoch, !progressWritten, !progressFailed) - else log.Information("Progress Epoch {epoch} ({updates} updates)", epoch, !progressWritten) - progressFailed := 0; progressWritten := 0 - buffer.Dump log - let tryDumpStats = every statsIntervalMs dumpStats - let handle = function - | CoordinationWork.BatchWithTracking(pos, items) -> - for item in items do - buffer.Add item |> ignore - syncState.AppendBatch(pos, [|for x in items -> x.stream, x.span.index + int64 x.span.events.Length |]) - | CoordinationWork.ProgressResult (Choice1Of2 epoch) -> - incr progressWritten - progressEpoch <- Some epoch - | CoordinationWork.ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progressFailed - | CoordinationWork.Result res -> - incr resultsHandled - let (stream, updatedState), kind = buffer.HandleWriteResult res - match updatedState.write with None -> () | Some wp -> syncState.MarkStreamProgress(stream, wp) - match kind with - | CosmosIngester.Ok -> res.WriteTo writerResultLog - | CosmosIngester.RateLimited -> incr rateLimited - | CosmosIngester.Malformed -> category stream |> badCats.Ingest; incr malformed - | CosmosIngester.TimedOut -> incr timedOut - let queueWrite (w : CosmosIngester.Batch) = - incr workPended - eventsPended := !eventsPended + w.span.events.Length - bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) - writers.Enqueue w - while not ct.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write results to buffer + Progress - match work.TryDequeue() with - | true, item -> - handle item - | false, _ -> - let validatedPos, pendingBatchCount = syncState.Validate buffer.TryGetStreamWritePos - pendingBatches <- pendingBatchCount - // 2. After that, [over] provision writers queue - let mutable more = writers.HasCapacity - while more do - match buffer.TryReady(writers.IsStreamBusy) with - | Some w -> queueWrite w; more <- writers.HasCapacity - | None -> more <- false - // 3. Periodically emit status info - tryDumpStats () - // 4. Feed latest state to progress writer - validatedPos |> Option.iter progressWriter.Post - // 5. Sleep if we've nothing else to do - do! Async.Sleep pumpSleepMs - dumpStats () - log.Warning("... Coordinator exiting") } - - static member Start(log : Serilog.ILogger, cosmosContext) : Coordinator = - let cts = new CancellationTokenSource() - let coordinator = new Coordinator(cosmosContext, cts) - Async.Start(coordinator.Pump log,cts.Token) - coordinator - member __.Submit(checkpoint : Async, batches : StreamItem[]) = postBatch checkpoint batches - member __.PendingBatches = pendingBatches - interface IDisposable with member __.Dispose() = cts.Cancel() - - let createRangeSyncHandler (log:ILogger) (ctx: Core.CosmosContext) (transform : Microsoft.Azure.Documents.Document -> StreamItem seq) = - let busyPauseMs = 500 + let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Core.CosmosContext, maxWriters) (transform : Microsoft.Azure.Documents.Document -> StreamItem seq) = + let ingestionEngine = Equinox.Cosmos.Projection.Ingestion.startIngestionEngine (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let maxUnconfirmedBatches = 10 - let mutable coordinator = Unchecked.defaultof<_> + let mutable trancheEngine = Unchecked.defaultof<_> let init rangeLog = - coordinator <- Coordinator.Start(rangeLog, ctx) - let ingest docs checkpoint : (*events*)int * (*streams*)int = + trancheEngine <- Equinox.Projection.Engine.TrancheEngine.Start (rangeLog, ingestionEngine, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) + let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq - coordinator.Submit(checkpoint,events) - events.Length, HashSet(seq { for x in events -> x.stream }).Count - let dispose () = (coordinator :> IDisposable).Dispose() + trancheEngine.Submit(epoch, checkpoint, events) + let dispose () = trancheEngine.Stop () let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } - let pt, (events,streams) = Stopwatch.Time(fun () -> ingest docs checkpoint) - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Gen {events,5} events {p:n3}s Sync {streams,5} streams", - ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), - float sw.ElapsedMilliseconds / 1000., events, (let e = pt.Elapsed in e.TotalSeconds), streams) - // Only hand back control to the CFP iff our processing backlog is under control - // no point getting too far ahead and/or overloading ourselves if we can't log our progress - let mutable first = true - let pauseTimer = Stopwatch.StartNew() - let backlogIsExcessive pendingBatches = - let tooMuch = pendingBatches >= maxUnconfirmedBatches - if first && tooMuch then log.Information("Pausing due to backlog of incomplete batches...") - let longDelay = pauseTimer.ElapsedMilliseconds > 5000L - let level = if tooMuch && (first || longDelay) then Events.LogEventLevel.Information else Events.LogEventLevel.Debug - first <- false - if longDelay then pauseTimer.Reset() - log.Write(level, "Pending Batches {pb}", pendingBatches) - tooMuch - while backlogIsExcessive coordinator.PendingBatches do - do! Async.Sleep busyPauseMs + let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time + log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", + epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., + let e = pt.Elapsed in e.TotalSeconds, cur, max) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) @@ -743,9 +531,9 @@ let main argv = let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() let auxDiscovery, aux, leaseId, startFromHere, batchSize, lagFrequency = args.BuildChangeFeedParams() #if marveleqx - let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformV0 catFilter) + let createSyncHandler () = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformV0 catFilter) #else - let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformOrFilter catFilter) + let createSyncHandler () = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformOrFilter catFilter) // Uncomment to test marveleqx mode // let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformV0 catFilter) #endif @@ -761,14 +549,16 @@ let main argv = | e when not e.IsJson || e.EventStreamId.StartsWith("$") || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.EndsWith("_checkpoints") + || e.EventStreamId.StartsWith "marvel_bookmark" + || e.EventStreamId.EndsWith "_checkpoints" + || e.EventStreamId.EndsWith "_checkpoint" || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused || e.EventStreamId = "ReloadBatchId" // does not start at 0 - || e.EventStreamId = "PurchaseOrder-5791" // Too large - || e.EventStreamId.EndsWith("_checkpoint") + || e.EventStreamId = "PurchaseOrder-5791" // item too large + || e.EventStreamId = "Inventory-FC000" // Too long || not (catFilter e.EventStreamId) -> None - | e -> EventStoreSource.tryToBatch e - EventStoreSource.run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) (args.MaxWriters, target, args.MaxPendingBatches) resolveCheckpointStream + | e -> e |> EventStoreSource.toIngestionItem |> Some + EventStoreSource.run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream #endif |> Async.RunSynchronously 0 diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index af03744f3..9717a0c72 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,7 +4,7 @@ Exe netcoreapp2.1 5 - cosmos1 + cosmos_ @@ -27,8 +27,7 @@ - - + \ No newline at end of file From c004adf2d4719dbf3a8b4db516a803b69b8ad894 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Apr 2019 21:48:51 +0100 Subject: [PATCH 162/353] fix --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index b7a2eaa18..8d8f50960 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -368,7 +368,7 @@ module EventStoreSource = readers.AddTail(startPos, max, spec.tailInterval) let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) - let postStreamSpan : StreamSpan -> unit = failwith "TODO" // coordinator.Submit + let postStreamSpan : StreamSpan -> unit = fun _ -> failwith "TODO" // coordinator.Submit let postBatch (pos : EventStore.ClientAPI.Position) xs = let cp = pos.CommitPosition trancheEngine.Submit(cp, checkpoints.Commit cp, xs) From 606dae50f9afa5c4543fe40ee460d9eeb471fcdb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Apr 2019 21:55:49 +0100 Subject: [PATCH 163/353] Show max batch limits --- equinox-sync/Sync/EventStoreSource.fs | 2 +- equinox-sync/Sync/Program.fs | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 9c8929a99..6112239a2 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -144,7 +144,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn let! (cur,max) = postBatch currentSlice.NextPosition batches Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n3}s {cur}/{max}", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, batches.Length, let e = postSw.Elapsed in e.TotalSeconds, cur, max) + batchEvents, usedCats, usedStreams, batches.Length, (let e = postSw.Elapsed in e.TotalSeconds), cur, max) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else sw.Restart() // restart the clock as we hand off back to the Reader return! aux () } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 8d8f50960..7fbfb28a1 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -160,6 +160,8 @@ module CmdParser = x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", x.MinBatchSize, x.StartingBatchSize, x.StreamReaders) + Log.Information("Max read-ahead for ingester: {maxPendingBatches} batches, max batches to process concurrently: {maxProcessing}", + x.MaxPendingBatches, x.MaxProcessing) { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; streamReaders = x.StreamReaders } #endif From 355242b1d00442c0ea3bb0fb43ec166e794a0bcb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Apr 2019 22:55:35 +0100 Subject: [PATCH 164/353] Drain Active --- .../Equinox.Projection/Engine.fs | 28 +++++++++++++------ equinox-projector/Equinox.Projection/State.fs | 1 + 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 00ff04753..6c2a48e97 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -14,6 +14,7 @@ type Message<'R> = /// Enqueue a batch of items with supplied progress marking function | Add of markCompleted: (unit -> unit) * items: StreamItem[] | AddStream of StreamSpan + | AddActive of KeyValuePair[] /// Feed stats about an ingested batch to relevant listeners | Added of streams: int * skip: int * events: int /// Result of processing on stream - result (with basic stats) or the `exn` encountered @@ -29,7 +30,7 @@ type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = abstract member Handle : Message<'R> -> unit default __.Handle res = match res with - | Add _ | AddStream _ -> () + | Add _ | AddStream _ | AddActive _ -> () | Added (streams, skipped, events) -> incr batchesPended streamsPended := !streamsPended + streams @@ -81,6 +82,9 @@ type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project reqs.[stream] <- item.index+1L progressState.AppendBatch(checkpoint,reqs) work.Enqueue(Added (reqs.Count,skipCount,count)) + | AddActive events -> + for e in events do + streams.InternalMerge(e.Key,e.Value) | AddStream streamSpan -> let _stream,_streamState = streams.Add(streamSpan,false) work.Enqueue(Added (1,0,streamSpan.span.events.Length)) // Yes, need to compute skip @@ -125,6 +129,9 @@ type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project member __.AllStreams = streams.All + member __.AddActiveStreams(events) = + work.Enqueue <| AddActive events + member __.Submit(streamSpan) = work.Enqueue <| AddStream streamSpan @@ -168,9 +175,9 @@ type TrancheStreamBuffer() = for item in items do merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = Array.singleton item.event } |] } - member __.Take(set : ISet<_>) = seq { + member __.Take(processingContains) = Array.ofSeq <| seq { for x in states do - if set.Contains x.Key then + if processingContains x.Key then states.Remove x.Key |> ignore yield x } @@ -221,9 +228,12 @@ type SeriesMessage = type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None let progCommitFails, progCommits = ref 0, ref 0 - let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0 + let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (available,maxDop) = + log.Information("Tranche Cycles {cycles} Active {active}/{writers} Batches {batches} ({streams:n0}s {events:n0}e)", + !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsPended) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -239,9 +249,6 @@ type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = else log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - log.Information("Ingest Cycles {cycles} Batches {batches} ({streams:n0}s {events:n0})", - !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; member __.Handle : SeriesMessage -> unit = function | Add _ -> () // Enqueuing of an event is not interesting - we assume it'll get processed and mapped to an `Added` in the same cycle | ProgressResult (Choice1Of2 epoch) -> @@ -299,8 +306,8 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted while pending.Count <> 0 && write.HasCapacity && ingesterAccepting do let markCompleted, events = pending.Peek() - let! sumbitted = ingester.TrySubmit(markCompleted, events) - if sumbitted then + let! submitted = ingester.TrySubmit(markCompleted, events) + if submitted then pending.Dequeue() |> ignore // mark off a write as being in progress do! write.Await() @@ -310,6 +317,9 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, stats.HandleValidated(Option.map fst validatedPos, fst write.State) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch + // 3. Forward content for any active streams into processor immediately + let relevantBufferedStreams = streams.Take(ingester.AllStreams.Contains) + ingester.AddActiveStreams(relevantBufferedStreams) // 4. Periodically emit status info stats.TryDump(write.State,streams) do! Async.Sleep pumpDelayMs } diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index e7c979d4f..99e6c19dd 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -130,6 +130,7 @@ type StreamStates() = busy.Remove stream |> ignore member __.All = streams + member __.InternalMerge(stream, state) = update stream state |> ignore member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } member __.Add(stream, index, event, ?isMalformed) = updateWritePos stream (defaultArg isMalformed false) None [| { index = index; events = [| event |] } |] From d076e5cd42f81213ec02ce9d741198c23ac46fb3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Apr 2019 22:57:07 +0100 Subject: [PATCH 165/353] fix --- equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs index d65f09af3..33903c231 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs @@ -88,6 +88,7 @@ type CosmosStats(log : ILogger, statsInterval) = match message with | Message.Add (_,_) | Message.AddStream _ + | Message.AddActive _ | Message.Added _ -> () | Result (_stream, Choice1Of2 ((es,bs),r)) -> events <- events + es From b8026e2d5a7cc92432115971bbbd57b1a0b2eb1e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 29 Apr 2019 23:02:20 +0100 Subject: [PATCH 166/353] fix collection modification --- equinox-projector/Equinox.Projection/Engine.fs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 6c2a48e97..effcca5ae 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -175,11 +175,10 @@ type TrancheStreamBuffer() = for item in items do merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = Array.singleton item.event } |] } - member __.Take(processingContains) = Array.ofSeq <| seq { - for x in states do - if processingContains x.Key then - states.Remove x.Key |> ignore - yield x } + member __.Take(processingContains) = + let forward = [| for x in states do if processingContains x.Key then yield x |] + for x in forward do states.Remove x.Key |> ignore + forward member __.Dump(log : ILogger) = let mutable waiting, waitingB = 0, 0L From b90b7c1c68eb98193cde59e7827249d027b76c7d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 00:48:13 +0100 Subject: [PATCH 167/353] release Read and write --- equinox-projector/Equinox.Projection/Engine.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index effcca5ae..df6b2d03a 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -291,6 +291,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, streams.Merge items let markCompleted () = write.Release() + read.Release() validatedPos <- Some (epoch,checkpoint) let streams = HashSet() for x in items do streams.Add x.stream |> ignore From 61aebbd4a90471cd7e4fcea9987c6ac4b79d11d7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 01:36:43 +0100 Subject: [PATCH 168/353] Fix buffering stats --- .../Equinox.Projection/Engine.fs | 22 +++++++++---------- 1 file changed, 10 insertions(+), 12 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index df6b2d03a..b57fff20e 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -24,7 +24,7 @@ type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (available,maxDop) = - log.Information("Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + log.Information("Projection Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : Message<'R> -> unit @@ -41,6 +41,7 @@ type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = | Result (_stream, Choice2Of2 _) -> incr resultExn member __.TryDump((available,maxDop),streams : StreamStates) = + incr cycles if statsDue () then dumpStats (available,maxDop) __.DumpExtraStats() @@ -189,7 +190,7 @@ type TrancheStreamBuffer() = waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) waiting <- waiting + 1 waitingB <- waitingB + sz - log.Information("Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) + log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) @@ -231,8 +232,8 @@ type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (available,maxDop) = log.Information("Tranche Cycles {cycles} Active {active}/{writers} Batches {batches} ({streams:n0}s {events:n0}e)", - !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsPended) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0; + !cycles, available, maxDop, !batchesPended, !streamsPended, !eventsPended) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -260,12 +261,12 @@ type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = streamsPended := !streamsPended + streams eventsPended := !eventsPended + events member __.HandleValidated(epoch, pendingBatches) = - incr cycles - pendingBatchCount <- pendingBatches validatedEpoch <- epoch + pendingBatchCount <- pendingBatches member __.HandleCommitted epoch = comittedEpoch <- epoch member __.TryDump((available,maxDop),streams : TrancheStreamBuffer) = + incr cycles if statsDue () then dumpStats (available,maxDop) streams.Dump log @@ -284,8 +285,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, let pumpDelayMs = defaultArg pumpDelayMs 5 member private __.Pump() = async { - let handle x = - match x with + let handle = function | Add (epoch, checkpoint, items) -> let items = Array.ofSeq items streams.Merge items @@ -293,9 +293,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, write.Release() read.Release() validatedPos <- Some (epoch,checkpoint) - let streams = HashSet() - for x in items do streams.Add x.stream |> ignore - work.Enqueue(Added (streams.Count,items.Length)) + work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) pending.Enqueue((markCompleted,items)) | Added _ | ProgressResult _ -> () use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) @@ -314,7 +312,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, else ingesterAccepting <- false // 2. Update any progress into the stats - stats.HandleValidated(Option.map fst validatedPos, fst write.State) + stats.HandleValidated(Option.map fst validatedPos, fst read.State) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch // 3. Forward content for any active streams into processor immediately From ec528fe1c8c63b68a3ab80f9c0d85209d073652c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 01:40:41 +0100 Subject: [PATCH 169/353] Hook stats --- equinox-projector/Equinox.Projection/Engine.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index b57fff20e..db1e3fde3 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -299,7 +299,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) while not cts.IsCancellationRequested do - work |> ConcurrentQueue.drain handle + work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) let mutable ingesterAccepting = true // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted while pending.Count <> 0 && write.HasCapacity && ingesterAccepting do From a6818825ed70ec5e947ac063bca84ebe2e825057 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 02:43:51 +0100 Subject: [PATCH 170/353] Comments, naming --- .../Equinox.Projection.Cosmos/Ingestion.fs | 11 ++-- .../Equinox.Projection/Engine.fs | 64 +++++++++---------- equinox-projector/Equinox.Projection/State.fs | 5 +- 3 files changed, 38 insertions(+), 42 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs index 33903c231..69ab0c02b 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs @@ -86,10 +86,9 @@ type CosmosStats(log : ILogger, statsInterval) = override __.Handle message = base.Handle message match message with - | Message.Add (_,_) - | Message.AddStream _ - | Message.AddActive _ - | Message.Added _ -> () + | ProjectionMessage.Add (_,_) + | ProjectionMessage.AddActive _ + | ProjectionMessage.Added _ -> () | Result (_stream, Choice1Of2 ((es,bs),r)) -> events <- events + es bytes <- bytes + int64 bs @@ -114,7 +113,7 @@ let startIngestionEngine (log : Serilog.ILogger, maxPendingBatches, cosmosContex let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 - // Reduce the item count when we don't yet know the write position + // Reduce the item count when we don't yet know the write position in order to efficiently discover the redundancy where data is already present count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } let project batch = async { @@ -137,7 +136,7 @@ let startIngestionEngine (log : Serilog.ILogger, maxPendingBatches, cosmosContex let malformed = Writer.classify exn |> Writer.isMalformed None,streams.SetMalformed(stream,malformed) match res with - | Message.Result (s,r) -> + | ProjectionMessage.Result (s,r) -> let _ctx,(stream,updatedState) = applyResultToStreamState (s,r) match updatedState.write with | Some wp -> diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index db1e3fde3..abff4c04f 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -7,19 +7,22 @@ open System.Collections.Concurrent open System.Collections.Generic open System.Threading +/// Item from a reader as supplied to the projector/ingestor loop for aggregation type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } +/// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners [] -type Message<'R> = +type ProjectionMessage<'R> = /// Enqueue a batch of items with supplied progress marking function | Add of markCompleted: (unit -> unit) * items: StreamItem[] - | AddStream of StreamSpan - | AddActive of KeyValuePair[] - /// Feed stats about an ingested batch to relevant listeners + /// Stats per submitted batch for stats listeners to aggregate | Added of streams: int * skip: int * events: int + /// Submit new data pertaining to a stream that has commenced processing + | AddActive of KeyValuePair[] /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> - + +/// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) @@ -27,10 +30,9 @@ type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = log.Information("Projection Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 - abstract member Handle : Message<'R> -> unit - default __.Handle res = - match res with - | Add _ | AddStream _ | AddActive _ -> () + abstract member Handle : ProjectionMessage<'R> -> unit + default __.Handle msg = msg |> function + | Add _ | AddActive _ -> () | Added (streams, skipped, events) -> incr batchesPended streamsPended := !streamsPended + streams @@ -46,18 +48,20 @@ type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = dumpStats (available,maxDop) __.DumpExtraStats() streams.Dump log + /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () -/// Consolidates ingested events into streams; coordinates dispatching of these in priority dictated by the needs of the checkpointing approach in force -/// a) does not itself perform any readin activities -/// b) manages writing of progress -/// x) periodically reports state (with hooks for ingestion engines to report same) +/// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order +/// a) does not itself perform any reading activities +/// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) +/// c) submits work to the supplied Dispatcher (which it triggers pumping of) +/// d) periodically reports state (with hooks for ingestion engines to report same) type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, handleResult) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() let batches = new SemaphoreSlim(maxPendingBatches) - let work = ConcurrentQueue>() + let work = ConcurrentQueue>() let streams = StreamStates() let progressState = ProgressState() @@ -86,9 +90,6 @@ type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project | AddActive events -> for e in events do streams.InternalMerge(e.Key,e.Value) - | AddStream streamSpan -> - let _stream,_streamState = streams.Add(streamSpan,false) - work.Enqueue(Added (1,0,streamSpan.span.events.Length)) // Yes, need to compute skip | Added _ -> () | Result _ as r -> @@ -100,17 +101,18 @@ type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) let completedBatches = progressState.Validate(streams.TryGetStreamWritePos) if completedBatches > 0 then batches.Release(completedBatches) |> ignore - // 3. After that, provision writers queue + // 3. After that, top up provisioning of writers queue let capacity,_ = dispatcher.AvailableCapacity if capacity <> 0 then let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) let xs = (Seq.ofArray work).GetEnumerator() - let mutable ok = true - while xs.MoveNext() && ok do + let mutable addsBeingAccepted = true + while xs.MoveNext() && addsBeingAccepted do let! succeeded = dispatcher.TryAdd(project xs.Current) - ok <- succeeded + addsBeingAccepted <- succeeded // 4. Periodically emit status info stats.TryDump(dispatcher.AvailableCapacity,streams) + // 5. Do a minimal sleep so we don't run completely hot when emprt do! Async.Sleep sleepIntervalMs } static member Start<'R>(stats, maxPendingBatches, processorDop, project, handleResult) = let dispatcher = Dispatcher(processorDop) @@ -118,23 +120,17 @@ type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project Async.Start <| instance.Pump(stats) instance + /// Attempt to feed in a batch (subject to there being capacity to do so) member __.TrySubmit(markCompleted, events) = async { let! got = batches.Await(TimeSpan.Zero) - if got then work.Enqueue <| Add (markCompleted, events); return true - else return false } - - member __.Submit(markCompleted, events) = async { - let! _ = batches.Await() - work.Enqueue <| Add (markCompleted, Array.ofSeq events) - return maxPendingBatches-batches.CurrentCount,maxPendingBatches } + if got then + work.Enqueue <| Add (markCompleted, events) + return got } - member __.AllStreams = streams.All - - member __.AddActiveStreams(events) = + member __.AddOpenStreamData(events) = work.Enqueue <| AddActive events - member __.Submit(streamSpan) = - work.Enqueue <| AddStream streamSpan + member __.AllStreams = streams.All member __.Stop() = cts.Cancel() @@ -317,7 +313,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, stats.HandleCommitted progressWriter.CommittedEpoch // 3. Forward content for any active streams into processor immediately let relevantBufferedStreams = streams.Take(ingester.AllStreams.Contains) - ingester.AddActiveStreams(relevantBufferedStreams) + ingester.AddOpenStreamData(relevantBufferedStreams) // 4. Periodically emit status info stats.TryDump(write.State,streams) do! Async.Sleep pumpDelayMs } diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 99e6c19dd..4e07d195a 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -181,6 +181,7 @@ type StreamStates() = if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) + // Used to trigger catch-up reading of streams which have events missing prior to the observed write position //member __.TryGap() : (string*int64*int) option = // let rec aux () = // match gap |> Queue.tryDequeue with @@ -247,13 +248,13 @@ type Dispatcher<'R>(maxDop) = dop.Release() |> ignore } [] member __.Result = result.Publish member __.AvailableCapacity = - let available = dop.CurrentCount + 1 + let available = dop.CurrentCount available,maxDop member __.TryAdd(item,?timeout) = async { let! got = dop.Await(?timeout=timeout) if got then work.Add(item) - return got} + return got } member __.Pump () = async { let! ct = Async.CancellationToken for item in work.GetConsumingEnumerable ct do From 241d181543ffb61cfb9a14d6bc195fe785f414c7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 09:25:38 +0100 Subject: [PATCH 171/353] Fix length calcs --- equinox-projector/Equinox.Projection/Engine.fs | 2 +- equinox-projector/Equinox.Projection/State.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index abff4c04f..b07fc8b1a 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -183,7 +183,7 @@ type TrancheStreamBuffer() = for KeyValue (stream,state) in states do let sz = int64 state.Size waitingCats.Ingest(category stream) - waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) + waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) waiting <- waiting + 1 waitingB <- waitingB + sz log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 4e07d195a..6c2c97680 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -171,7 +171,7 @@ type StreamStates() = busyB <- busyB + sz | sz -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.Length, (sz + 512L) / 1024L) + readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) ready <- ready + 1 readyB <- readyB + sz log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", From 95fefcc88fc17de7b4321f3a217606af3a17539b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 16:47:52 +0100 Subject: [PATCH 172/353] Up default max batches --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 7fbfb28a1..6d19a1d20 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -112,7 +112,7 @@ module CmdParser = and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.Verbose = a.Contains Verbose - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,5000) member __.MaxProcessing = a.GetResult(MaxProcessing,128) member __.MaxWriters = a.GetResult(MaxWriters,512) #if cosmos From d557b4212c358764e89ec25bda83b6cfe8077721 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 30 Apr 2019 22:36:13 +0100 Subject: [PATCH 173/353] Add Gorge Mode --- equinox-sync/Sync/EventStoreSource.fs | 3 ++ equinox-sync/Sync/Program.fs | 68 +++++++++++++++++++++------ 2 files changed, 57 insertions(+), 14 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 6112239a2..979c4516c 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -95,6 +95,9 @@ let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 let posFromChunk (chunk: int) = let chunkBase = int64 chunk * 1024L * 1024L * 256L Position(chunkBase,0L) +let posFromChunkAfter (pos: EventStore.ClientAPI.Position) = + let nextChunk = 1 + int (chunk pos) + posFromChunk nextChunk let posFromPercentage (pct,max : Position) = let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 6d19a1d20..eb79090ac 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -36,8 +36,9 @@ type ReaderSpec = checkpointInterval: TimeSpan /// Delay when reading yields an empty batch tailInterval: TimeSpan - /// Maximum number of stream readers to permit - streamReaders: int + gorge: bool + /// Maximum number of striped readers to permit + stripes: int /// Initial batch size to use when commencing reading batchSize: int /// Smallest batch size to degrade to in the presence of failures @@ -75,7 +76,8 @@ module CmdParser = | [] Position of int64 | [] Chunk of int | [] Percent of float - | [] StreamReaders of int + | [] Gorge + | [] Stripes of int | [] Tail of intervalS: float #endif | [] Source of ParseResults @@ -105,7 +107,8 @@ module CmdParser = | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | StreamReaders _ -> "number of concurrent readers. Default: 8" + | Gorge -> "Parallel readers (instead of reading by stream)" + | Stripes _ -> "number of concurrent readers to run one chunk (256MB) apart. Default: 1" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | Source _ -> "EventStore input parameters." #endif @@ -127,7 +130,8 @@ module CmdParser = member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) member __.MinBatchSize = a.GetResult(MinBatchSize,512) - member __.StreamReaders = a.GetResult(StreamReaders,8) + member __.Gorge = a.Contains(Gorge) + member __.Stripes = a.GetResult(Stripes,1) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds member __.CheckpointInterval = TimeSpan.FromHours 1. member __.ForceRestart = a.Contains ForceRestart @@ -158,12 +162,12 @@ module CmdParser = | None, None, None, _ -> StartPos.StartOrCheckpoint Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) - Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} stream readers", - x.MinBatchSize, x.StartingBatchSize, x.StreamReaders) - Log.Information("Max read-ahead for ingester: {maxPendingBatches} batches, max batches to process concurrently: {maxProcessing}", - x.MaxPendingBatches, x.MaxProcessing) + Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} concurrent readers reading up to {maxPendingBatches} uncommitted batches ahead", + x.MinBatchSize, x.StartingBatchSize, x.Stripes, x.MaxPendingBatches) + Log.Information("Max batches to process concurrently: {maxProcessing}", + x.MaxProcessing) { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart - batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; streamReaders = x.StreamReaders } + batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; gorge = x.Gorge; stripes = x.Stripes } #endif and [] SourceParameters = #if cosmos @@ -327,6 +331,37 @@ module EventStoreSource = dop.Release() |> ignore do! Async.Sleep sleepIntervalMs } + type StripedReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = + let dop = new SemaphoreSlim(maxDop) + let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) + + member __.Pump(postBatch, startPos, max) = async { + let! ct = Async.CancellationToken + let mutable remainder = + let nextPos = EventStoreSource.posFromChunkAfter startPos + work.AddTranche(startPos, nextPos, max) + Some nextPos + let mutable finished = false + while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do + work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + let postItem : StreamSpan -> unit = fun _ -> failwith "NA" + try let! eof = work.Process(conn, tryMapEvent, postItem, postBatch, task) in () + if eof then remainder <- None + finally dop.Release() |> ignore } + return () } + match remainder with + | Some pos -> + let nextPos = EventStoreSource.posFromChunkAfter pos + remainder <- Some nextPos + do! forkRunRelease <| EventStoreSource.Work.Tranche (EventStoreSource.Range(pos, Some nextPos, max), batchSize) + | None -> + if finished then do! Async.Sleep 1000 + else Log.Error("No further ingestion work to commence") + finished <- true } + type StartMode = Starting | Resuming | Overridding //// 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) @@ -366,15 +401,19 @@ module EventStoreSource = startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) return startPos } - let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.streamReaders + 1) - readers.AddTail(startPos, max, spec.tailInterval) let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) - let postStreamSpan : StreamSpan -> unit = fun _ -> failwith "TODO" // coordinator.Submit let postBatch (pos : EventStore.ClientAPI.Position) xs = let cp = pos.CommitPosition trancheEngine.Submit(cp, checkpoints.Commit cp, xs) - do! readers.Pump(postStreamSpan, postBatch) } + if spec.gorge then + let readers = StripedReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) + do! readers.Pump(postBatch, startPos, max) + else + let postStreamSpan : StreamSpan -> unit = fun _ -> failwith "TODO" // coordinator.Submit // TODO StreeamReaders config + let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) + readers.AddTail(startPos, max, spec.tailInterval) + do! readers.Pump(postStreamSpan, postBatch) } #else module CosmosSource = open Microsoft.Azure.Documents @@ -557,6 +596,7 @@ let main argv = || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused || e.EventStreamId = "ReloadBatchId" // does not start at 0 || e.EventStreamId = "PurchaseOrder-5791" // item too large + || e.EventStreamId = "SkuFileUpload-99682b9cdbba4b09881d1d87dfdc1ded" || e.EventStreamId = "Inventory-FC000" // Too long || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some From 5f6770d2062b23e3f4efcb3deea846cda62a56e7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 06:04:14 +0100 Subject: [PATCH 174/353] blacklist stream --- equinox-sync/Sync/Program.fs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index eb79090ac..d9e9a78c3 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -19,8 +19,6 @@ open System.Collections.Generic //#else open System.Diagnostics open System.Threading -open Equinox.Cosmos.Projection.Ingestion - //#endif //#if eventStore @@ -597,6 +595,7 @@ let main argv = || e.EventStreamId = "ReloadBatchId" // does not start at 0 || e.EventStreamId = "PurchaseOrder-5791" // item too large || e.EventStreamId = "SkuFileUpload-99682b9cdbba4b09881d1d87dfdc1ded" + || e.EventStreamId = "SkuFileUpload-1e0626cc418548bc8eb82808426430e2" || e.EventStreamId = "Inventory-FC000" // Too long || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some From 04c1f0002bd9cb53e6ba2bdaf6cb78a9bcb53ae1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 12:53:35 +0100 Subject: [PATCH 175/353] Add Gorge mode impl --- .../Equinox.Projection/Engine.fs | 59 +++++++++++++++---- equinox-sync/Sync/EventStoreSource.fs | 35 +++++++---- equinox-sync/Sync/Program.fs | 44 ++++++++++---- 3 files changed, 105 insertions(+), 33 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index b07fc8b1a..8dc9dc9de 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -217,6 +217,9 @@ type ProgressWriter<'Res when 'Res: equality>() = [] type SeriesMessage = | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq + | MoveToChunk of chunk: int + | AddStriped of chunk: int * epoch: int64 * markCompleted: Async * items: StreamItem seq + | EndOfChunk of chunk: int | Added of streams: int * events: int /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` | ProgressResult of Choice @@ -246,7 +249,8 @@ type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) member __.Handle : SeriesMessage -> unit = function - | Add _ -> () // Enqueuing of an event is not interesting - we assume it'll get processed and mapped to an `Added` in the same cycle + | Add _ | AddStriped _ | EndOfChunk _ | MoveToChunk _ -> + () // Enqueuing of an event is not interesting - we assume it'll get processed and mapped to an `Added` in the same cycle | ProgressResult (Choice1Of2 epoch) -> incr progCommits comittedEpoch <- Some epoch @@ -267,6 +271,14 @@ type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = dumpStats (available,maxDop) streams.Dump log +[] +type Push = + | Batch of epoch: int64 * markCompleted: Async * items: StreamItem seq + | Stream of span: StreamSpan + | SetActiveChunk of chunk: int + | ChunkBatch of chunk: int * epoch: int64 * markCompleted: Async * items: StreamItem seq + | EndOfChunk of chunk: int + /// Holds batches away from Core processing to limit in-flight processsing type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, maxSubmissions, statsInterval : TimeSpan, ?pumpDelayMs) = let cts = new CancellationTokenSource() @@ -275,22 +287,41 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, let write = new Sem(maxSubmissions) let streams = TrancheStreamBuffer() let pending = Queue<_>() + let readAhead = ResizeArray<_>() let mutable validatedPos = None let progressWriter = ProgressWriter<_>() let stats = TrancheStats(log, maxQueued, statsInterval) let pumpDelayMs = defaultArg pumpDelayMs 5 member private __.Pump() = async { + let mutable activeChunk = None + let ingestItems epoch checkpoint items = + let items = Array.ofSeq items + streams.Merge items + let markCompleted () = + write.Release() + read.Release() + validatedPos <- Some (epoch,checkpoint) + work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) + markCompleted, items let handle = function | Add (epoch, checkpoint, items) -> - let items = Array.ofSeq items - streams.Merge items - let markCompleted () = - write.Release() - read.Release() - validatedPos <- Some (epoch,checkpoint) - work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) + let markCompleted,items = ingestItems epoch checkpoint items pending.Enqueue((markCompleted,items)) + | AddStriped (chunk, epoch, checkpoint, items) -> + let markCompleted,items = ingestItems epoch checkpoint items + match activeChunk with + | Some c when c = chunk -> pending.Enqueue((markCompleted,items)) + | _ -> readAhead.Add((chunk,(markCompleted,items))) + | MoveToChunk newActiveChunk -> + activeChunk <- Some newActiveChunk + let isForActiveChunk (chunk,_) = activeChunk = Some chunk + readAhead |> Seq.where isForActiveChunk |> Seq.iter (snd >> pending.Enqueue) + readAhead.RemoveAll(fun (chunk,item) -> isForActiveChunk (chunk, item)) |> ignore + | EndOfChunk chunk -> + if activeChunk = Some chunk then + work.Enqueue <| MoveToChunk (chunk + 1) + // These events are for stats purposes | Added _ | ProgressResult _ -> () use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) @@ -319,9 +350,17 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, do! Async.Sleep pumpDelayMs } /// Awaits space in `read` to limit reading ahead - yields present state of Read and Write phases - member __.Submit(epoch, markBatchCompleted, events) = async { + member __.Submit(epoch, markBatchCompleted, events) = __.Submit <| Push.Batch (epoch, markBatchCompleted, events) + + /// Awaits space in `read` to limit reading ahead - yields present state of Read and Write phases + member __.Submit(content) = async { do! read.Await() - work.Enqueue <| Add (epoch, markBatchCompleted, events) + match content with + | Push.Batch (epoch, markBatchCompleted, events) -> work.Enqueue <| Add (epoch, markBatchCompleted, events) + | Push.Stream _items -> failwith "TODO" + | Push.ChunkBatch (chunk, epoch, markBatchCompleted, events) -> work.Enqueue <| AddStriped (chunk, epoch, markBatchCompleted, events) + | Push.SetActiveChunk chunk -> work.Enqueue <| MoveToChunk chunk + | Push.EndOfChunk chunk -> work.Enqueue <| EndOfChunk chunk return read.State } static member Start<'R>(log, ingester, maxRead, maxWrite, statsInterval) = diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 979c4516c..4040dd649 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -2,6 +2,7 @@ open Equinox.Store // AwaitTaskCorrect open Equinox.Projection +open Equinox.Projection.Engine open EventStore.ClientAPI open System open Serilog // NB Needs to shadow ILogger @@ -116,7 +117,7 @@ let establishMax (conn : IEventStoreConnection) = async { Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 return Option.get max } -let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : State.StreamSpan -> unit) = +let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : State.StreamSpan -> Async) = let rec fetchFrom pos limit = async { let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect @@ -124,7 +125,7 @@ let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int [| for x in currentSlice.Events -> let e = x.Event Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] - postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } + do! postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } match limit with | None when currentSlice.IsEndOfStream -> return () | None -> return! fetchFrom currentSlice.NextEventNumber None @@ -148,9 +149,11 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n3}s {cur}/{max}", range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, (let e = postSw.Elapsed in e.TotalSeconds), cur, max) - if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then return currentSlice.IsEndOfStream else - sw.Restart() // restart the clock as we hand off back to the Reader - return! aux () } + if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then + return currentSlice.IsEndOfStream + else + sw.Restart() // restart the clock as we hand off back to the Reader + return! aux () } async { try let! eof = aux () return if eof then Eof else EndOfTranche @@ -162,6 +165,12 @@ type [] Work = | Tranche of range: Range * batchSize : int | Tail of pos: Position * max : Position * interval: TimeSpan * batchSize : int +[] +type ReadItem = + | Batch of pos: Position * items: StreamItem seq + | EndOfTranche of chunk: int + | StreamSpan of span: State.StreamSpan + type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) @@ -179,13 +188,14 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = work.Enqueue <| Work.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() - member __.Process(conn, tryMapEvent, postItem, postBatch, work) = async { + member __.Process(conn, tryMapEvent, post : ReadItem -> Async, work) = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize + let postSpan = ReadItem.StreamSpan >> post >> Async.Ignore match work with | StreamPrefix (name,pos,len,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) - try let! t,() = pullStream (conn, batchSize) (name, pos, Some len) postItem |> Stopwatch.Time + try let! t,() = pullStream (conn, batchSize) (name, pos, Some len) postSpan |> Stopwatch.Time Log.Information("completed stream prefix in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) with e -> let bs = adjust batchSize @@ -195,7 +205,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Stream (name,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) Log.Warning("Reading stream; batch size {bs}", batchSize) - try let! t,() = pullStream (conn, batchSize) (name,0L,None) postItem |> Stopwatch.Time + try let! t,() = pullStream (conn, batchSize) (name,0L,None) postSpan |> Stopwatch.Time Log.Information("completed stream in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) with e -> let bs = adjust batchSize @@ -203,17 +213,21 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = __.AddStream(name, bs) return false | Tranche (range, batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",chunk range.Current) + let chunk = chunk range.Current |> int + use _ = Serilog.Context.LogContext.PushProperty("Tranche", chunk) Log.Warning("Commencing tranche, batch size {bs}", batchSize) + let postBatch pos items = post (ReadItem.Batch (pos, items)) let! t, res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with | PullResult.EndOfTranche -> Log.Warning("completed tranche in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired() + let! _ = post (ReadItem.EndOfTranche chunk) return false | PullResult.Eof -> Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired(true) + let! _ = post (ReadItem.EndOfTranche chunk) return true | PullResult.Exn e -> let bs = adjust batchSize @@ -234,6 +248,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let slicesStats, stats = SliceStatsBuffer(), OverallStats() use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let progressSw = Stopwatch.StartNew() + let postBatch pos items = post (ReadItem.Batch (pos, items)) while true do let currentPos = range.Current if progressSw.ElapsedMilliseconds > progressIntervalMs then @@ -241,7 +256,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = count, currentPos.CommitPosition, chunk currentPos) progressSw.Restart() count <- count + 1 - let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch + let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch do! awaitInterval match res with | PullResult.EndOfTranche | PullResult.Eof -> () diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index d9e9a78c3..7f78dd9e7 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -310,16 +310,17 @@ module EventStoreSource = let dop = new SemaphoreSlim(maxDop) let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) member __.HasCapacity = work.QueueCount < dop.CurrentCount - member __.AddTail(startPos, max, interval) = work.AddTail(startPos, max, interval) + // TODO stuff member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) - member __.Pump(postItem, postBatch) = async { + member __.Pump(post, startPos, max, tailInterval) = async { let! ct = Async.CancellationToken + work.AddTail(startPos, max, tailInterval) while not ct.IsCancellationRequested do work.OverallStats.DumpIfIntervalExpired() let! _ = dop.Await() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, tryMapEvent, postItem, postBatch, task) in () + try let! _ = work.Process(conn, tryMapEvent, post, task) in () finally dop.Release() |> ignore } return () } match work.TryDequeue() with @@ -333,7 +334,7 @@ module EventStoreSource = let dop = new SemaphoreSlim(maxDop) let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) - member __.Pump(postBatch, startPos, max) = async { + member __.Pump(post, startPos, max) = async { let! ct = Async.CancellationToken let mutable remainder = let nextPos = EventStoreSource.posFromChunkAfter startPos @@ -345,8 +346,7 @@ module EventStoreSource = let! _ = dop.Await() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - let postItem : StreamSpan -> unit = fun _ -> failwith "NA" - try let! eof = work.Process(conn, tryMapEvent, postItem, postBatch, task) in () + try let! eof = work.Process(conn, tryMapEvent, post, task) in () if eof then remainder <- None finally dop.Release() |> ignore } return () } @@ -401,17 +401,31 @@ module EventStoreSource = return startPos } let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) - let postBatch (pos : EventStore.ClientAPI.Position) xs = - let cp = pos.CommitPosition - trancheEngine.Submit(cp, checkpoints.Commit cp, xs) if spec.gorge then let readers = StripedReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) - do! readers.Pump(postBatch, startPos, max) + let post = function + | EventStoreSource.ReadItem.Batch (pos, xs) -> + let cp = pos.CommitPosition + let chunk = EventStoreSource.chunk pos + trancheEngine.Submit <| Push.ChunkBatch(int chunk, cp, checkpoints.Commit cp, xs) + | EventStoreSource.ReadItem.EndOfTranche chunk -> + trancheEngine.Submit <| Push.EndOfChunk chunk + | EventStoreSource.ReadItem.StreamSpan _ as x -> + failwithf "%A not supported when gorging" x + let startChunk = EventStoreSource.chunk startPos |> int + let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) + do! readers.Pump(post, startPos, max) else - let postStreamSpan : StreamSpan -> unit = fun _ -> failwith "TODO" // coordinator.Submit // TODO StreeamReaders config + let post = function + | EventStoreSource.ReadItem.Batch (pos, xs) -> + let cp = pos.CommitPosition + trancheEngine.Submit(cp, checkpoints.Commit cp, xs) + | EventStoreSource.ReadItem.StreamSpan span -> + trancheEngine.Submit <| Push.Stream span + | EventStoreSource.ReadItem.EndOfTranche _ as x -> + failwithf "%A not supported" x let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) - readers.AddTail(startPos, max, spec.tailInterval) - do! readers.Pump(postStreamSpan, postBatch) } + do! readers.Pump(post, startPos, max, spec.tailInterval) } #else module CosmosSource = open Microsoft.Azure.Documents @@ -596,6 +610,10 @@ let main argv = || e.EventStreamId = "PurchaseOrder-5791" // item too large || e.EventStreamId = "SkuFileUpload-99682b9cdbba4b09881d1d87dfdc1ded" || e.EventStreamId = "SkuFileUpload-1e0626cc418548bc8eb82808426430e2" + || e.EventStreamId = "SkuFileUpload-6b4f566d90194263a2700c0ad1bc54dd" + || e.EventStreamId = "SkuFileUpload-5926b2d7512c4f859540f7f20e35242b" + || e.EventStreamId = "SkuFileUpload-9ac536b61fed4b44853a1f5e2c127d50" + || e.EventStreamId = "SkuFileUpload-b501837ce7e6416db80ca0c48a4b3f7a" || e.EventStreamId = "Inventory-FC000" // Too long || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some From 079073ceb4e3e3e124ee8c2d4446b863e7abdae2 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 13:06:47 +0100 Subject: [PATCH 176/353] Fix stripe count off by one count --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 7f78dd9e7..088e363ca 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -402,7 +402,7 @@ module EventStoreSource = let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) if spec.gorge then - let readers = StripedReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) + let readers = StripedReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) let post = function | EventStoreSource.ReadItem.Batch (pos, xs) -> let cp = pos.CommitPosition From 6a67db2150807659df209fbef46da96982ad0a9c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 13:12:34 +0100 Subject: [PATCH 177/353] Log gorge mode --- equinox-sync/Sync/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 088e363ca..c923ec056 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -395,9 +395,9 @@ module EventStoreSource = do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) return Overridding, r, spec.checkpointInterval } - log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m", + log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m gorge: {gorge}", startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, - float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes) + float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes, spec.gorge) return startPos } let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) From 9d6c6873d294898d111da36fb26768f4613610d4 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 14:36:24 +0100 Subject: [PATCH 178/353] Add chunk buffer management + logging --- .../Equinox.Projection/Engine.fs | 45 +++++++++++++++---- 1 file changed, 36 insertions(+), 9 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 8dc9dc9de..4c3229cd9 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -287,7 +287,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, let write = new Sem(maxSubmissions) let streams = TrancheStreamBuffer() let pending = Queue<_>() - let readAhead = ResizeArray<_>() + let readingAhead, ready = Dictionary>(), Dictionary>() let mutable validatedPos = None let progressWriter = ProgressWriter<_>() let stats = TrancheStats(log, maxQueued, statsInterval) @@ -304,23 +304,50 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, validatedPos <- Some (epoch,checkpoint) work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) markCompleted, items + let tryRemove key (dict: Dictionary<_,_>) = + match ready.TryGetValue key with + | true, value -> + dict.Remove key |> ignore + Some value + | false, _ -> None let handle = function | Add (epoch, checkpoint, items) -> let markCompleted,items = ingestItems epoch checkpoint items pending.Enqueue((markCompleted,items)) | AddStriped (chunk, epoch, checkpoint, items) -> - let markCompleted,items = ingestItems epoch checkpoint items + let batchInfo = ingestItems epoch checkpoint items match activeChunk with - | Some c when c = chunk -> pending.Enqueue((markCompleted,items)) - | _ -> readAhead.Add((chunk,(markCompleted,items))) + | Some c when c = chunk -> pending.Enqueue batchInfo + | _ -> + match readingAhead.TryGetValue chunk with + | false, _ -> readingAhead.[chunk] <- ResizeArray(Seq.singleton batchInfo) + | true,current -> current.Add(batchInfo) | MoveToChunk newActiveChunk -> activeChunk <- Some newActiveChunk - let isForActiveChunk (chunk,_) = activeChunk = Some chunk - readAhead |> Seq.where isForActiveChunk |> Seq.iter (snd >> pending.Enqueue) - readAhead.RemoveAll(fun (chunk,item) -> isForActiveChunk (chunk, item)) |> ignore + let buffered = + match ready |> tryRemove newActiveChunk with + | Some completedChunkBatches -> + completedChunkBatches |> Seq.iter pending.Enqueue + work.Enqueue <| MoveToChunk (newActiveChunk + 1) + completedChunkBatches.Count + | None -> + match readingAhead |> tryRemove newActiveChunk with + | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count + | None -> 0 + log.Information("Moving to chunk {activeChunk}, releasing {buffered} buffered items", newActiveChunk, buffered) | EndOfChunk chunk -> - if activeChunk = Some chunk then - work.Enqueue <| MoveToChunk (chunk + 1) + match activeChunk with + | Some ac when ac = chunk -> + log.Information("Completed reading active chunk {activeChunk}, moving to next", ac) + work.Enqueue <| MoveToChunk (ac + 1) + | _ -> + match readingAhead |> tryRemove chunk with + | Some batchesRead -> + ready.[chunk] <- batchesRead + log.Information("Completed reading {chunkNo}, marking {buffered} buffered items ready", chunk, batchesRead.Count) + | None -> + ready.[chunk] <- ResizeArray() + log.Information("Completed reading {chunkNo}, leaving empty batch", chunk) // These events are for stats purposes | Added _ | ProgressResult _ -> () use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) From 46c07400b1381ffd6f9009515a055b0e96da3ae2 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 14:46:11 +0100 Subject: [PATCH 179/353] Fix chunk numbering --- equinox-sync/Sync/EventStoreSource.fs | 15 +++++++-------- equinox-sync/Sync/Program.fs | 6 ++++-- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 4040dd649..17c71993f 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -162,7 +162,7 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn type [] Work = | Stream of name: string * batchSize: int | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int - | Tranche of range: Range * batchSize : int + | Tranche of chunk: int * range: Range * batchSize : int | Tail of pos: Position * max : Position * interval: TimeSpan * batchSize : int [] @@ -180,10 +180,10 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = work.Enqueue <| Work.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) - member __.AddTranche(range, ?batchSizeOverride) = - work.Enqueue <| Work.Tranche (range, defaultArg batchSizeOverride batchSize) - member __.AddTranche(pos, nextPos, max, ?batchSizeOverride) = - __.AddTranche(Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) + member __.AddTranche(chunk, range, ?batchSizeOverride) = + work.Enqueue <| Work.Tranche (chunk, range, defaultArg batchSizeOverride batchSize) + member __.AddTranche(chunk, pos, nextPos, max, ?batchSizeOverride) = + __.AddTranche(chunk, Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) member __.AddTail(pos, max, interval, ?batchSizeOverride) = work.Enqueue <| Work.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = @@ -212,8 +212,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) __.AddStream(name, bs) return false - | Tranche (range, batchSize) -> - let chunk = chunk range.Current |> int + | Tranche (chunk, range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche", chunk) Log.Warning("Commencing tranche, batch size {bs}", batchSize) let postBatch pos items = post (ReadItem.Batch (pos, items)) @@ -233,7 +232,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let bs = adjust batchSize Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) __.OverallStats.DumpIfIntervalExpired() - __.AddTranche(range, bs) + __.AddTranche(chunk, range, bs) return false | Tail (pos, max, interval, batchSize) -> let mutable count, batchSize, range = 0, batchSize, Range(pos, None, max) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index c923ec056..e2d6dc044 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -338,7 +338,8 @@ module EventStoreSource = let! ct = Async.CancellationToken let mutable remainder = let nextPos = EventStoreSource.posFromChunkAfter startPos - work.AddTranche(startPos, nextPos, max) + let startChunk = EventStoreSource.chunk startPos |> int + work.AddTranche(startChunk, startPos, nextPos, max) Some nextPos let mutable finished = false while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do @@ -354,7 +355,8 @@ module EventStoreSource = | Some pos -> let nextPos = EventStoreSource.posFromChunkAfter pos remainder <- Some nextPos - do! forkRunRelease <| EventStoreSource.Work.Tranche (EventStoreSource.Range(pos, Some nextPos, max), batchSize) + let chunkNumber = EventStoreSource.chunk pos |> int + do! forkRunRelease <| EventStoreSource.Work.Tranche (chunkNumber, EventStoreSource.Range(pos, Some nextPos, max), batchSize) | None -> if finished then do! Async.Sleep 1000 else Log.Error("No further ingestion work to commence") From f02df11d148cf658e6ddbff074e0e179e023a7be Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 14:54:40 +0100 Subject: [PATCH 180/353] Fix queue management --- equinox-sync/Sync/Program.fs | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index e2d6dc044..0deb62f80 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -351,16 +351,20 @@ module EventStoreSource = if eof then remainder <- None finally dop.Release() |> ignore } return () } - match remainder with - | Some pos -> - let nextPos = EventStoreSource.posFromChunkAfter pos - remainder <- Some nextPos - let chunkNumber = EventStoreSource.chunk pos |> int - do! forkRunRelease <| EventStoreSource.Work.Tranche (chunkNumber, EventStoreSource.Range(pos, Some nextPos, max), batchSize) - | None -> - if finished then do! Async.Sleep 1000 - else Log.Error("No further ingestion work to commence") - finished <- true } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + match remainder with + | Some pos -> + let nextPos = EventStoreSource.posFromChunkAfter pos + remainder <- Some nextPos + let chunkNumber = EventStoreSource.chunk pos |> int + do! forkRunRelease <| EventStoreSource.Work.Tranche (chunkNumber, EventStoreSource.Range(pos, Some nextPos, max), batchSize) + | None -> + if finished then do! Async.Sleep 1000 + else Log.Error("No further ingestion work to commence") + finished <- true } type StartMode = Starting | Resuming | Overridding From 9a625e0c624e50609e05f50af907933c393d78a5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 15:05:02 +0100 Subject: [PATCH 181/353] Log mode info --- equinox-sync/Sync/Program.fs | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 0deb62f80..55c6befcd 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -160,8 +160,8 @@ module CmdParser = | None, None, None, _ -> StartPos.StartOrCheckpoint Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) - Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}] with {stripes} concurrent readers reading up to {maxPendingBatches} uncommitted batches ahead", - x.MinBatchSize, x.StartingBatchSize, x.Stripes, x.MaxPendingBatches) + Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}], reading up to {maxPendingBatches} uncommitted batches ahead", + x.MinBatchSize, x.StartingBatchSize, x.MaxPendingBatches) Log.Information("Max batches to process concurrently: {maxProcessing}", x.MaxProcessing) { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart @@ -401,9 +401,9 @@ module EventStoreSource = do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) return Overridding, r, spec.checkpointInterval } - log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) tailing every {interval}s, checkpointing every {checkpointFreq}m gorge: {gorge}", - startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, - float startPos.CommitPosition/float max.CommitPosition, spec.tailInterval.TotalSeconds, checkpointFreq.TotalMinutes, spec.gorge) + log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) checkpointing every {checkpointFreq:n1}m", + startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, float startPos.CommitPosition/float max.CommitPosition, + checkpointFreq.TotalMinutes) return startPos } let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) @@ -420,6 +420,7 @@ module EventStoreSource = failwithf "%A not supported when gorging" x let startChunk = EventStoreSource.chunk startPos |> int let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) + log.Information("Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) do! readers.Pump(post, startPos, max) else let post = function @@ -431,6 +432,7 @@ module EventStoreSource = | EventStoreSource.ReadItem.EndOfTranche _ as x -> failwithf "%A not supported" x let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) + log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) do! readers.Pump(post, startPos, max, spec.tailInterval) } #else module CosmosSource = From 3eeb083f1d4494a7a60853aa708cd5b0e96c8093 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 15:17:31 +0100 Subject: [PATCH 182/353] Fix chunk numbering --- equinox-sync/Sync/EventStoreSource.fs | 9 +++++---- equinox-sync/Sync/Program.fs | 13 +++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 17c71993f..e70c12209 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -168,7 +168,8 @@ type [] Work = [] type ReadItem = | Batch of pos: Position * items: StreamItem seq - | EndOfTranche of chunk: int + | ChunkBatch of chunk: int * pos: Position * items: StreamItem seq + | EndOfChunk of chunk: int | StreamSpan of span: State.StreamSpan type ReadQueue(batchSize, minBatchSize, ?statsInterval) = @@ -215,18 +216,18 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | Tranche (chunk, range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche", chunk) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let postBatch pos items = post (ReadItem.Batch (pos, items)) + let postBatch pos items = post (ReadItem.ChunkBatch (chunk, pos, items)) let! t, res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with | PullResult.EndOfTranche -> Log.Warning("completed tranche in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired() - let! _ = post (ReadItem.EndOfTranche chunk) + let! _ = post (ReadItem.EndOfChunk chunk) return false | PullResult.Eof -> Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired(true) - let! _ = post (ReadItem.EndOfTranche chunk) + let! _ = post (ReadItem.EndOfChunk chunk) return true | PullResult.Exn e -> let bs = adjust batchSize diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 55c6befcd..5d142a03f 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -410,12 +410,12 @@ module EventStoreSource = if spec.gorge then let readers = StripedReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) let post = function - | EventStoreSource.ReadItem.Batch (pos, xs) -> + | EventStoreSource.ReadItem.ChunkBatch (chunk, pos, xs) -> let cp = pos.CommitPosition - let chunk = EventStoreSource.chunk pos - trancheEngine.Submit <| Push.ChunkBatch(int chunk, cp, checkpoints.Commit cp, xs) - | EventStoreSource.ReadItem.EndOfTranche chunk -> + trancheEngine.Submit <| Push.ChunkBatch(chunk, cp, checkpoints.Commit cp, xs) + | EventStoreSource.ReadItem.EndOfChunk chunk -> trancheEngine.Submit <| Push.EndOfChunk chunk + | EventStoreSource.ReadItem.Batch _ | EventStoreSource.ReadItem.StreamSpan _ as x -> failwithf "%A not supported when gorging" x let startChunk = EventStoreSource.chunk startPos |> int @@ -429,8 +429,9 @@ module EventStoreSource = trancheEngine.Submit(cp, checkpoints.Commit cp, xs) | EventStoreSource.ReadItem.StreamSpan span -> trancheEngine.Submit <| Push.Stream span - | EventStoreSource.ReadItem.EndOfTranche _ as x -> - failwithf "%A not supported" x + | EventStoreSource.ReadItem.ChunkBatch _ + | EventStoreSource.ReadItem.EndOfChunk _ as x -> + failwithf "%A not supported when tailing" x let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) do! readers.Pump(post, startPos, max, spec.tailInterval) } From 4f325c35ca00ea625667fd610ae462e1e795992d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 15:29:13 +0100 Subject: [PATCH 183/353] Doh --- equinox-projector/Equinox.Projection/Engine.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 4c3229cd9..c090fa4e5 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -305,7 +305,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) markCompleted, items let tryRemove key (dict: Dictionary<_,_>) = - match ready.TryGetValue key with + match dict.TryGetValue key with | true, value -> dict.Remove key |> ignore Some value From 13706f78e67dbb6ab77e28330de05596080ab908 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 15:56:00 +0100 Subject: [PATCH 184/353] Log buffering counts --- equinox-projector/Equinox.Projection/Engine.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index c090fa4e5..37be52d48 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -334,7 +334,8 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, match readingAhead |> tryRemove newActiveChunk with | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count | None -> 0 - log.Information("Moving to chunk {activeChunk}, releasing {buffered} buffered items", newActiveChunk, buffered) + log.Information("Moving to chunk {activeChunk}, releasing {buffered} buffered items, {ready} others ready, {ahead} reading ahead, ", + newActiveChunk, buffered, ready.Count, readingAhead.Count) | EndOfChunk chunk -> match activeChunk with | Some ac when ac = chunk -> From c3aa973e8af99020bc3c6a7a49bae9b6c7883d84 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 16:28:00 +0100 Subject: [PATCH 185/353] Add jitter --- equinox-projector/Equinox.Projection/Engine.fs | 2 +- equinox-sync/Sync/Program.fs | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 37be52d48..46d4325e5 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -334,7 +334,7 @@ type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, match readingAhead |> tryRemove newActiveChunk with | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count | None -> 0 - log.Information("Moving to chunk {activeChunk}, releasing {buffered} buffered items, {ready} others ready, {ahead} reading ahead, ", + log.Information("Moving to chunk {activeChunk}, releasing {buffered} buffered items, {ready} others ready, {ahead} reading ahead", newActiveChunk, buffered, ready.Count, readingAhead.Count) | EndOfChunk chunk -> match activeChunk with diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 5d142a03f..d08b05618 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -342,6 +342,7 @@ module EventStoreSource = work.AddTranche(startChunk, startPos, nextPos, max) Some nextPos let mutable finished = false + let r = new Random() while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do work.OverallStats.DumpIfIntervalExpired() let! _ = dop.Await() @@ -357,6 +358,11 @@ module EventStoreSource = | false, _ -> match remainder with | Some pos -> + // Start the readers interleaved + if dop.CurrentCount > 1 then + let jitter = r.Next(1000, 2000) + Log.Warning("Waiting {jitter}ms jitter to offset reader stripes", jitter) + do! Async.Sleep jitter let nextPos = EventStoreSource.posFromChunkAfter pos remainder <- Some nextPos let chunkNumber = EventStoreSource.chunk pos |> int From 9dad3de5a60783781203e6d86f31bb37251409d7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 16:30:37 +0100 Subject: [PATCH 186/353] Fix jitter --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index d08b05618..3c45db94d 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -359,7 +359,7 @@ module EventStoreSource = match remainder with | Some pos -> // Start the readers interleaved - if dop.CurrentCount > 1 then + if dop.CurrentCount <> 0 then let jitter = r.Next(1000, 2000) Log.Warning("Waiting {jitter}ms jitter to offset reader stripes", jitter) do! Async.Sleep jitter From 0cd52b0cf6c9a2810b94c92fb28fd845d8be68e1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 16:42:32 +0100 Subject: [PATCH 187/353] More jitter work --- equinox-sync/Sync/Program.fs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 3c45db94d..9613c406b 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -358,11 +358,10 @@ module EventStoreSource = | false, _ -> match remainder with | Some pos -> - // Start the readers interleaved - if dop.CurrentCount <> 0 then - let jitter = r.Next(1000, 2000) - Log.Warning("Waiting {jitter}ms jitter to offset reader stripes", jitter) - do! Async.Sleep jitter + let currentCount = dop.CurrentCount + let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) + Log.Warning("Waiting {jitter}ms jitter to offset reader stripes, {currentCount} slots open", jitter, currentCount) + do! Async.Sleep jitter let nextPos = EventStoreSource.posFromChunkAfter pos remainder <- Some nextPos let chunkNumber = EventStoreSource.chunk pos |> int From 4bb8d1cb941349037261ffd94d04eb390ac0a5b2 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 16:48:25 +0100 Subject: [PATCH 188/353] Add universal jitter --- equinox-sync/Sync/Program.fs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 9613c406b..97b50ef59 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -352,16 +352,16 @@ module EventStoreSource = if eof then remainder <- None finally dop.Release() |> ignore } return () } + let currentCount = dop.CurrentCount + let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) + Log.Warning("Waiting {jitter}ms jitter to offset reader stripes, {currentCount} slots open", jitter, currentCount) + do! Async.Sleep jitter match work.TryDequeue() with | true, task -> do! forkRunRelease task | false, _ -> match remainder with | Some pos -> - let currentCount = dop.CurrentCount - let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) - Log.Warning("Waiting {jitter}ms jitter to offset reader stripes, {currentCount} slots open", jitter, currentCount) - do! Async.Sleep jitter let nextPos = EventStoreSource.posFromChunkAfter pos remainder <- Some nextPos let chunkNumber = EventStoreSource.chunk pos |> int From d0dde026a5a76756eb671032df6a138fc512533f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 17:09:23 +0100 Subject: [PATCH 189/353] Connection per stripe --- equinox-sync/Sync/Program.fs | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 97b50ef59..859428072 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -330,7 +330,7 @@ module EventStoreSource = dop.Release() |> ignore do! Async.Sleep sleepIntervalMs } - type StripedReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = + type StripedReader(conns : _ array, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = let dop = new SemaphoreSlim(maxDop) let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) @@ -343,18 +343,21 @@ module EventStoreSource = Some nextPos let mutable finished = false let r = new Random() + let mutable robin = 0 while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do work.OverallStats.DumpIfIntervalExpired() let! _ = dop.Await() let forkRunRelease task = async { let! _ = Async.StartChild <| async { - try let! eof = work.Process(conn, tryMapEvent, post, task) in () + try let connIndex = Interlocked.Increment(&robin) % conns.Length + let conn = conns.[connIndex] + let! eof = work.Process(conn, tryMapEvent, post, task) in () if eof then remainder <- None finally dop.Release() |> ignore } return () } let currentCount = dop.CurrentCount let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) - Log.Warning("Waiting {jitter}ms jitter to offset reader stripes, {currentCount} slots open", jitter, currentCount) + Log.Warning("Waiting {jitter}ms to jitter reader stripes, {currentCount} slots open", jitter, currentCount) do! Async.Sleep jitter match work.TryDequeue() with | true, task -> @@ -380,9 +383,10 @@ module EventStoreSource = // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) // | None -> more <- false - let run (log : Serilog.ILogger) (conn, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { + let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) - let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn + let conn = connect () + let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn let! initialCheckpointState = checkpoints.Read let! max = maxInParallel let! startPos = async { @@ -413,7 +417,8 @@ module EventStoreSource = let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) if spec.gorge then - let readers = StripedReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) + let conns = [| yield conn; yield! Seq.init (spec.stripes-1) (ignore >> connect) |] + let readers = StripedReader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) let post = function | EventStoreSource.ReadItem.ChunkBatch (chunk, pos, xs) -> let cp = pos.CommitPosition @@ -608,7 +613,7 @@ let main argv = (leaseId, startFromHere, batchSize, lagFrequency) createSyncHandler #else - let esConnection = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) + let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let catFilter = args.Source.CategoryFilterFunction let spec = args.BuildFeedParams() let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = @@ -631,7 +636,7 @@ let main argv = || e.EventStreamId = "Inventory-FC000" // Too long || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some - EventStoreSource.run log (esConnection.ReadConnection, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream + EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream #endif |> Async.RunSynchronously 0 From 211a7fcfcbb0631c16f6758abccea028ea7dd34d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 17:13:09 +0100 Subject: [PATCH 190/353] Blacklist inventory --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 859428072..f17ef35da 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -633,7 +633,7 @@ let main argv = || e.EventStreamId = "SkuFileUpload-5926b2d7512c4f859540f7f20e35242b" || e.EventStreamId = "SkuFileUpload-9ac536b61fed4b44853a1f5e2c127d50" || e.EventStreamId = "SkuFileUpload-b501837ce7e6416db80ca0c48a4b3f7a" - || e.EventStreamId = "Inventory-FC000" // Too long + || e.EventStreamId.StartsWith "Inventory-" // Too long || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream From 95a208f52ef0b6a6d031f09720c667bdad95ec4a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 17:16:19 +0100 Subject: [PATCH 191/353] Blacklist InvesntoryCount --- equinox-sync/Sync/Program.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index f17ef35da..40476b3c3 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -634,6 +634,7 @@ let main argv = || e.EventStreamId = "SkuFileUpload-9ac536b61fed4b44853a1f5e2c127d50" || e.EventStreamId = "SkuFileUpload-b501837ce7e6416db80ca0c48a4b3f7a" || e.EventStreamId.StartsWith "Inventory-" // Too long + || e.EventStreamId.StartsWith "InventoryCount-" // No Longer used || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream From bfefbab882debd4de841fd4857cdd6f9619fe3e0 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 19:55:35 +0100 Subject: [PATCH 192/353] Tidy ES ingestion, remove Ingester --- .../equinox-projector-consumer.sln | 6 - equinox-sync/Ingest/Ingest.fsproj | 17 - equinox-sync/Ingest/Program.fs | 334 ------------------ equinox-sync/Sync/EventStoreSource.fs | 152 ++++++-- equinox-sync/Sync/Infrastructure.fs | 34 +- equinox-sync/Sync/Program.fs | 100 +----- 6 files changed, 142 insertions(+), 501 deletions(-) delete mode 100644 equinox-sync/Ingest/Ingest.fsproj delete mode 100644 equinox-sync/Ingest/Program.fs diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index 8800cd964..5ac584d60 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -18,8 +18,6 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Tests", EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "..\equinox-sync\Sync\Sync.fsproj", "{C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "..\equinox-sync\Ingest\Ingest.fsproj", "{260FCBA4-C948-4D4E-92E1-B679075F6890}" -EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Codec", "Equinox.Projection.Codec\Equinox.Projection.Codec.fsproj", "{AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}" EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos.ProjectionEx", "Equinox.Projection.Cosmos\Equinox.Cosmos.ProjectionEx.fsproj", "{2071A2C9-B5C8-4143-B437-6833666D0ACA}" @@ -50,10 +48,6 @@ Global {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.Build.0 = Debug|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.ActiveCfg = Release|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.Build.0 = Release|Any CPU - {260FCBA4-C948-4D4E-92E1-B679075F6890}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {260FCBA4-C948-4D4E-92E1-B679075F6890}.Debug|Any CPU.Build.0 = Debug|Any CPU - {260FCBA4-C948-4D4E-92E1-B679075F6890}.Release|Any CPU.ActiveCfg = Release|Any CPU - {260FCBA4-C948-4D4E-92E1-B679075F6890}.Release|Any CPU.Build.0 = Release|Any CPU {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Debug|Any CPU.Build.0 = Debug|Any CPU {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/equinox-sync/Ingest/Ingest.fsproj b/equinox-sync/Ingest/Ingest.fsproj deleted file mode 100644 index 9f8ae20d5..000000000 --- a/equinox-sync/Ingest/Ingest.fsproj +++ /dev/null @@ -1,17 +0,0 @@ - - - - Exe - netcoreapp2.1 - 5 - - - - - - - - - - - \ No newline at end of file diff --git a/equinox-sync/Ingest/Program.fs b/equinox-sync/Ingest/Program.fs deleted file mode 100644 index 77602542b..000000000 --- a/equinox-sync/Ingest/Program.fs +++ /dev/null @@ -1,334 +0,0 @@ -module IngestTemplate.Program - -open Serilog -open System -open System.Collections.Concurrent -open System.Diagnostics -open System.Threading - -open SyncTemplate -open EventStoreSource - -type StartPos = Position of int64 | Chunk of int | Percentage of float | StreamList of string list | Start -type ReaderSpec = { start: StartPos; stripes: int; batchSize: int; minBatchSize: int } - -module CmdParser = - open Argu - - exception MissingArg of string - let envBackstop msg key = - match Environment.GetEnvironmentVariable key with - | null -> raise <| MissingArg (sprintf "Please provide a %s, either as an argment or via the %s environment variable" msg key) - | x -> x - - module Cosmos = - open Equinox.Cosmos - [] - type Parameters = - | [] ConnectionMode of ConnectionMode - | [] Timeout of float - | [] Retries of int - | [] RetriesWaitTime of int - | [] Connection of string - | [] Database of string - | [] Collection of string - interface IArgParserTemplate with - member a.Usage = - match a with - | Connection _ -> "specify a connection string for a Cosmos account (default: envvar:EQUINOX_COSMOS_CONNECTION)." - | Database _ -> "specify a database name for Cosmos account (default: envvar:EQUINOX_COSMOS_DATABASE)." - | Collection _ -> "specify a collection name for Cosmos account (default: envvar:EQUINOX_COSMOS_COLLECTION)." - | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 1)." - | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" - | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." - type Arguments(a : ParseResults) = - member __.Mode = a.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) - member __.Discovery = Discovery.FromConnectionString __.Connection - member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" - member __.Database = match a.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" - member __.Collection = match a.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" - - member __.Timeout = a.GetResult(Timeout, 5.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(Retries, 1) - member __.MaxRetryWaitTime = a.GetResult(RetriesWaitTime, 5) - - /// Connect with the provided parameters and/or environment variables - member x.Connect - /// Connection/Client identifier for logging purposes - name : Async = - let (Discovery.UriAndKey (endpointUri,_masterKey)) as discovery = x.Discovery - Log.Information("CosmosDb {mode} {endpointUri} Database {database} Collection {collection}.", - x.Mode, endpointUri, x.Database, x.Collection) - Log.Information("CosmosDb timeout {timeout}s; Throttling retries {retries}, max wait {maxRetryWaitTime}s", - (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) - let c = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) - c.Connect(name, discovery) - - /// To establish a local node to run against: - /// 1. cinst eventstore-oss -y # where cinst is an invocation of the Chocolatey Package Installer on Windows - /// 2. & $env:ProgramData\chocolatey\bin\EventStore.ClusterNode.exe --gossip-on-single-node --discover-via-dns 0 --ext-http-port=30778 - module EventStore = - open Equinox.EventStore - type [] Parameters = - | [] VerboseStore - | [] Timeout of float - | [] Retries of int - | [] Host of string - | [] Port of int - | [] Username of string - | [] Password of string - | [] HeartbeatTimeout of float - | [] MaxItems of int - | [] Cosmos of ParseResults - interface IArgParserTemplate with - member a.Usage = - match a with - | VerboseStore -> "Include low level Store logging." - | Timeout _ -> "specify operation timeout in seconds (default: 20)." - | Retries _ -> "specify operation retries (default: 3)." - | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." - | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." - | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." - | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." - | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." - | MaxItems _ -> "maximum item count to request. Default: 4096" - | Cosmos _ -> "specify CosmosDb parameters" - type Arguments(a : ParseResults ) = - member val Cosmos = Cosmos.Arguments(a.GetResult Cosmos) - member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" - member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int - member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host - member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" - member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" - member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds - member __.Timeout = a.GetResult(Timeout,20.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(Retries,3) - member __.Connect(log: ILogger, storeLog : ILogger, connectionStrategy) = - let s (x : TimeSpan) = x.TotalSeconds - log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s __.Heartbeat, s __.Timeout, __.Retries) - let log = if storeLog.IsEnabled Serilog.Events.LogEventLevel.Debug then Logger.SerilogVerbose storeLog else Logger.SerilogNormal storeLog - let tags = ["M", Environment.MachineName; "I", Guid.NewGuid() |> string] - GesConnector(__.User,__.Password, __.Timeout, __.Retries, log, heartbeatTimeout=__.Heartbeat, tags=tags) - .Establish("IngestTemplate", __.Discovery, connectionStrategy) - - [] - type Parameters = - | [] BatchSize of int - | [] MinBatchSize of int - | [] Verbose - | [] VerboseConsole - | [] LocalSeq - | [] Position of int64 - | [] Chunk of int - | [] Percent of float - | [] Stripes of int - | [] Stream of string - | [] Es of ParseResults - interface IArgParserTemplate with - member a.Usage = - match a with - | BatchSize _ -> "maximum item count to request from feed. Default: 4096" - | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | Verbose -> "request Verbose Logging. Default: off" - | VerboseConsole -> "request Verbose Console Logging. Default: off" - | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | Position _ -> "EventStore $all Stream Position to commence from" - | Chunk _ -> "EventStore $all Chunk to commence from" - | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | Stripes _ -> "number of concurrent readers" - | Stream _ -> "specific stream(s) to read" - | Es _ -> "specify EventStore parameters" - and Arguments(args : ParseResults) = - member val EventStore = EventStore.Arguments(args.GetResult Es) - member __.Verbose = args.Contains Verbose - member __.VerboseConsole = args.Contains VerboseConsole - member __.MaybeSeqEndpoint = if args.Contains LocalSeq then Some "http://localhost:5341" else None - member __.StartingBatchSize = args.GetResult(BatchSize,4096) - member __.MinBatchSize = args.GetResult(MinBatchSize,512) - member __.Stripes = args.GetResult(Stripes,1) - member x.BuildFeedParams() : ReaderSpec = - let startPos = - match args.TryGetResult Position, args.TryGetResult Chunk, args.TryGetResult Percent with - | Some p, _, _ -> StartPos.Position p - | _, Some c, _ -> StartPos.Chunk c - | _, _, Some p -> Percentage p - | None, None, None when args.GetResults Stream <> [] -> StreamList (args.GetResults Stream) - | None, None, None -> Start - Log.Information("Processing in batches of [{minBatchSize}..{batchSize}] with {stripes} stripes covering from {startPos}", - x.MinBatchSize, x.StartingBatchSize, x.Stripes, startPos) - { start = startPos; batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; stripes = x.Stripes } - - /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args - let parse argv : Arguments = - let programName = System.Reflection.Assembly.GetEntryAssembly().GetName().Name - let parser = ArgumentParser.Create(programName = programName) - parser.ParseCommandLine argv |> Arguments - -type Readers(conn, spec : ReaderSpec, tryMapEvent, postBatch, max : EventStore.ClientAPI.Position, ct : CancellationToken, ?statsInterval) = - let work = ReadQueue(spec.batchSize, spec.minBatchSize, ?statsInterval=statsInterval) - let posFromChunkAfter (pos: EventStore.ClientAPI.Position) = - let nextChunk = 1 + int (chunk pos) - posFromChunk nextChunk - let mutable remainder = - let startAt (startPos : EventStore.ClientAPI.Position) = - Log.Information("Start Position {pos} (chunk {chunk}, {pct:p1})", - startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition) - let nextPos = posFromChunkAfter startPos - work.AddTranche(startPos, nextPos, max) - Some nextPos - match spec.start with - | Start -> startAt <| EventStore.ClientAPI.Position.Start - | Position p -> startAt <| EventStore.ClientAPI.Position(p, 0L) - | Chunk c -> startAt <| posFromChunk c - | Percentage pct -> startAt <| posFromPercentage (pct, max) - | StreamList streams -> - for s in streams do - work.AddStream s - None - member __.Pump () = async { - let maxDop = spec.stripes - let dop = new SemaphoreSlim(maxDop) - let mutable finished = false - while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do - let! _ = dop.Await() - work.OverallStats.DumpIfIntervalExpired() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! eof = work.Process(conn, tryMapEvent, postBatch, (fun _pos -> Seq.iter postBatch), task) - if eof then remainder <- None - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - match remainder with - | Some pos -> - let nextPos = posFromChunkAfter pos - remainder <- Some nextPos - do! forkRunRelease <| Work.Tranche (Range(pos, Some nextPos, max), spec.batchSize) - | None -> - if finished then do! Async.Sleep 1000 - else Log.Warning("No further ingestion work to commence") - finished <- true } - -type Coordinator(log : Serilog.ILogger, writers : CosmosIngester.Writers, cancellationToken: CancellationToken, readerQueueLen, ?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 1.) in t.TotalMilliseconds |> int64 - let states = CosmosIngester.StreamStates() - let results = ConcurrentQueue<_>() - let work = new BlockingCollection<_>(ConcurrentQueue<_>(), readerQueueLen) - - member __.Add item = work.Add item - member __.HandleWriteResult = results.Enqueue - member __.Pump() = - let _ = writers.Result.Subscribe __.HandleWriteResult // codependent, wont worry about unsubcribing - let mutable bytesPended, bytesPendedAgg = 0L, 0L - let resultsHandled, ingestionsHandled, workPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let badCats = CosmosIngester.CatStats() - let progressTimer = Stopwatch.StartNew() - let writerResultLog = log.ForContext() - while not cancellationToken.IsCancellationRequested do - let mutable moreResults, rateLimited, timedOut = true, 0, 0 - while moreResults do - match results.TryDequeue() with - | true, res -> - incr resultsHandled - match states.HandleWriteResult res with - | (stream, _), CosmosIngester.TooLarge -> CosmosIngester.category stream |> badCats.Ingest - | (stream, _), CosmosIngester.Malformed -> CosmosIngester.category stream |> badCats.Ingest - | _, CosmosIngester.RateLimited -> rateLimited <- rateLimited + 1 - | _, CosmosIngester.TimedOut -> timedOut <- timedOut + 1 - | _, CosmosIngester.Ok -> res.WriteTo writerResultLog - | false, _ -> moreResults <- false - if rateLimited <> 0 || timedOut <> 0 then Log.Warning("Failures {rateLimited} Rate-limited, {timedOut} Timed out", rateLimited, timedOut) - let mutable t = Unchecked.defaultof<_> - let mutable toIngest = 4096 * 2 - while work.TryTake(&t) && toIngest > 0 do - incr ingestionsHandled - toIngest <- toIngest - 1 - states.Add t |> ignore - let mutable moreWork = true - while writers.HasCapacity && moreWork do - let pending = states.TryReady(writers.IsStreamBusy) - match pending with - | None -> moreWork <- false - | Some w -> - incr workPended - eventsPended := !eventsPended + w.span.events.Length - bytesPended <- bytesPended + int64 (Array.sumBy CosmosIngester.cosmosPayloadBytes w.span.events) - if progressTimer.ElapsedMilliseconds > intervalMs then - progressTimer.Restart() - Log.Information("Ingested {ingestions}", !ingestionsHandled) - bytesPendedAgg <- bytesPendedAgg + bytesPended - Log.Information("Writer Throughput {queued} reqs {events} events {mb:n}MB; Completed {completed} reqs; Egress {gb:n3}GB", - !workPended, !eventsPended, mb bytesPended, !resultsHandled, mb bytesPendedAgg / 1024.) - if badCats.Any then Log.Error("Malformed {badCats}", badCats.StatsDescending); badCats.Clear() - ingestionsHandled := 0; workPended := 0; eventsPended := 0; resultsHandled := 0; bytesPended <- 0L - states.Dump log - - static member Run log conn (spec : ReaderSpec, tryMapEvent) (ctx : Equinox.Cosmos.Core.CosmosContext) (writerCount, readerQueueLen) = async { - try let! ct = Async.CancellationToken - let! max = establishMax conn - let writers = CosmosIngester.Writers(CosmosIngester.Writer.write log ctx, writerCount) - let readers = Readers(conn, spec, tryMapEvent, writers.Enqueue, max, ct) - let instance = Coordinator(log, writers, ct, readerQueueLen) - let! _ = Async.StartChild <| writers.Pump() - let! _ = Async.StartChild <| readers.Pump() - let! _ = Async.StartChild(async { instance.Pump() }) in () - with e -> Log.Error(e,"Exiting") - do! Async.AwaitKeyboardInterrupt() } - -// Illustrates how to emit direct to the Console using Serilog -// Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog -module Logging = - let initialize verbose verboseConsole maybeSeqEndpoint = - Log.Logger <- - let ingesterLevel = if verboseConsole then Serilog.Events.LogEventLevel.Debug else Serilog.Events.LogEventLevel.Information - LoggerConfiguration() - .Destructure.FSharpTypes() - .Enrich.FromLogContext() - |> fun c -> if verbose then c.MinimumLevel.Debug() else c - |> fun c -> c.MinimumLevel.Override(typeof.FullName, ingesterLevel) - |> fun c -> let generalLevel = if verbose then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning - c.MinimumLevel.Override(typeof.FullName, generalLevel) - .MinimumLevel.Override(typeof.FullName, generalLevel) - .MinimumLevel.Override(typeof.FullName, generalLevel) - |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {Tranche} {Message:lj} {NewLine}{Exception}" - c.WriteTo.Console(ingesterLevel, theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) - |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) - |> fun c -> c.CreateLogger() - Log.ForContext() - -open Equinox.EventStore - -[] -let main argv = - try let args = CmdParser.parse argv - let log = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint - let source = args.EventStore.Connect(Log.Logger, Log.Logger, ConnectionStrategy.ClusterSingle NodePreference.Random) |> Async.RunSynchronously - let readerSpec = args.BuildFeedParams() - let writerCount, readerQueueLen = 128,4096*10*10 - let cosmos = args.EventStore.Cosmos // wierd nesting is due to me not finding a better way to express the semantics in Argu - let ctx = - let destination = cosmos.Connect "SyncTemplate.Ingester" |> Async.RunSynchronously - let colls = Equinox.Cosmos.CosmosCollections(cosmos.Database, cosmos.Collection) - Equinox.Cosmos.Core.CosmosContext(destination, colls, Log.ForContext()) - let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = - match x.Event with - | e when not e.IsJson - || e.EventStreamId.StartsWith("$") - || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) - || e.EventStreamId.EndsWith("_checkpoints") - || e.EventStreamId.EndsWith("_checkpoint") - || e.EventStreamId.StartsWith("marvel_bookmark_") - || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused - || e.EventStreamId = "ReloadBatchId" // does not start at 0 - || e.EventStreamId = "PurchaseOrder-5791" // Too large - || not (catFilter e.EventStreamId) -> None - | e -> EventStoreSource.tryToBatch e - Coordinator.Run log source.ReadConnection (readerSpec, tryMapEvent (fun _ -> true)) ctx (writerCount, readerQueueLen) |> Async.RunSynchronously - 0 - with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 - | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 - | e -> eprintfn "%s" e.Message; 1 \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index e70c12209..fd6c0e5e4 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -4,17 +4,18 @@ open Equinox.Store // AwaitTaskCorrect open Equinox.Projection open Equinox.Projection.Engine open EventStore.ClientAPI -open System open Serilog // NB Needs to shadow ILogger +open System +open System.Collections.Generic open System.Diagnostics open System.Threading -open System.Collections.Generic type EventStore.ClientAPI.RecordedEvent with member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = State.arrayBytes x.Data + State.arrayBytes x.Metadata let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 +let private mb x = float x / 1024. / 1024. let toIngestionItem (e : RecordedEvent) : Engine.StreamItem = let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata @@ -22,10 +23,9 @@ let toIngestionItem (e : RecordedEvent) : Engine.StreamItem = let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ { stream = e.EventStreamId; index = e.EventNumber; event = event} -let private mb x = float x / 1024. / 1024. - let category (streamName : string) = streamName.Split([|'-'|],2).[0] +/// Maintains ingestion stats (thread safe via lock free data structures so it can be used across multiple overlapping readers) type OverallStats(?statsInterval) = let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() @@ -43,6 +43,7 @@ type OverallStats(?statsInterval) = totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) progressStart.Restart() +/// Maintains stats for traversals of $all; Threadsafe [via naive locks] so can be used by multiple stripes reading concurrently type SliceStatsBuffer(?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 let recentCats, accStart = Dictionary(), Stopwatch.StartNew() @@ -74,6 +75,7 @@ type SliceStatsBuffer(?interval) = recentCats.Clear() accStart.Restart() +/// Defines a tranche of a traversal of a stream (or the store as a whole) type Range(start, sliceEnd : Position option, ?max : Position) = member val Current = start with get, set member __.TryNext(pos: Position) = @@ -91,6 +93,9 @@ type Range(start, sliceEnd : Position option, ?max : Position) = | p,m when p > m -> Double.NaN | p,m -> float p / float m +(* Logic for computation of chunk offsets; ES writes chunks whose index starts at a multiple of 256MB + to be able to address an arbitrary position as a percentage, we need to consider this aspect as only a valid Position can be supplied to the read call *) + // @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 let posFromChunk (chunk: int) = @@ -103,11 +108,13 @@ let posFromPercentage (pct,max : Position) = let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L +/// Read the current tail position; used to be able to compute and log progress of ingestion let fetchMax (conn : IEventStoreConnection) = async { let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect let max = lastItemBatch.FromPosition Log.Information("EventStore Tail Position: @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) return max } +/// `fetchMax` wrapped in a retry loop; Sync process is entirely reliant on establishing the max so we have a crude retry loop let establishMax (conn : IEventStoreConnection) = async { let mutable max = None while Option.isNone max do @@ -117,6 +124,9 @@ let establishMax (conn : IEventStoreConnection) = async { Log.Warning(e,"Could not establish max position") do! Async.Sleep 5000 return Option.get max } + +/// Walks a stream within the specified constraints; used to grab data when writing to a stream for which a prefix is missing +/// Can throw (in which case the caller is in charge of retrying, possibly with a smaller batch size) let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : State.StreamSpan -> Async) = let rec fetchFrom pos limit = async { let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize @@ -133,6 +143,8 @@ let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int | Some limit -> return! fetchFrom currentSlice.NextEventNumber (Some (limit - events.Length)) } fetchFrom pos limit +/// Walks the $all stream, yielding batches together with the associated Position info for the purposes of checkpointing +/// Can throw (in which case the caller is in charge of retrying, possibly with a smaller batch size) type [] PullResult = Exn of exn: exn | Eof | EndOfTranche let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) (range:Range, once) (tryMapEvent : ResolvedEvent -> Engine.StreamItem option) (postBatch : Position -> Engine.StreamItem[] -> Async) = @@ -159,39 +171,53 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn return if eof then Eof else EndOfTranche with e -> return Exn e } -type [] Work = - | Stream of name: string * batchSize: int - | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int - | Tranche of chunk: int * range: Range * batchSize : int +/// Specification for work to be performed by a reader thread +[] +type ReadRequest = + /// Tail from a given start position, at intervals of the specified timespan (no waiting if catching up) | Tail of pos: Position * max : Position * interval: TimeSpan * batchSize : int + /// Read a given segment of a stream (used when a stream needs to be rolled forward to lay down an event for which the preceding events are missing) + | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int + /// Read the entirity of a stream in blocks of the specified batchSize (TODO wire to commandline request) + | Stream of name: string * batchSize: int + /// Read a specific chunk (min-max range), posting batches tagged with that chunk number + | Chunk of chunk: int * range: Range * batchSize : int +/// Data with context resulting from a reader thread [] -type ReadItem = - | Batch of pos: Position * items: StreamItem seq +type ReadResult = + /// A batch read from a Chunk | ChunkBatch of chunk: int * pos: Position * items: StreamItem seq + /// Ingestion buffer requires an explicit end of chunk message before next chunk can commence processing | EndOfChunk of chunk: int + /// A Batch read from a Stream or StreamPrefix | StreamSpan of span: State.StreamSpan + /// A batch read by `Tail` + | Batch of pos: Position * items: StreamItem seq -type ReadQueue(batchSize, minBatchSize, ?statsInterval) = +/// Holds work queue, together with stats relating to the amount and/or categories of data being traversed +/// Processing is driven by external callers running multiple concurrent invocations of `Process` +type private ReadQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) member val SlicesStats = SliceStatsBuffer() member __.QueueCount = work.Count member __.AddStream(name, ?batchSizeOverride) = - work.Enqueue <| Work.Stream (name, defaultArg batchSizeOverride batchSize) + work.Enqueue <| ReadRequest.Stream (name, defaultArg batchSizeOverride batchSize) member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = - work.Enqueue <| Work.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) - member __.AddTranche(chunk, range, ?batchSizeOverride) = - work.Enqueue <| Work.Tranche (chunk, range, defaultArg batchSizeOverride batchSize) + work.Enqueue <| ReadRequest.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) + member private __.AddTranche(chunk, range, ?batchSizeOverride) = + work.Enqueue <| ReadRequest.Chunk (chunk, range, defaultArg batchSizeOverride batchSize) member __.AddTranche(chunk, pos, nextPos, max, ?batchSizeOverride) = __.AddTranche(chunk, Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) member __.AddTail(pos, max, interval, ?batchSizeOverride) = - work.Enqueue <| Work.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) + work.Enqueue <| ReadRequest.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) member __.TryDequeue () = work.TryDequeue() - member __.Process(conn, tryMapEvent, post : ReadItem -> Async, work) = async { + /// Invoked by Dispatcher to process a tranche of work; can have parallel invocations + member __.Process(conn, tryMapEvent, post : ReadResult -> Async, work) = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize - let postSpan = ReadItem.StreamSpan >> post >> Async.Ignore + let postSpan = ReadResult.StreamSpan >> post >> Async.Ignore match work with | StreamPrefix (name,pos,len,batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) @@ -213,21 +239,21 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) __.AddStream(name, bs) return false - | Tranche (chunk, range, batchSize) -> + | Chunk (chunk, range, batchSize) -> use _ = Serilog.Context.LogContext.PushProperty("Tranche", chunk) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let postBatch pos items = post (ReadItem.ChunkBatch (chunk, pos, items)) + let postBatch pos items = post (ReadResult.ChunkBatch (chunk, pos, items)) let! t, res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with | PullResult.EndOfTranche -> Log.Warning("completed tranche in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired() - let! _ = post (ReadItem.EndOfChunk chunk) + let! _ = post (ReadResult.EndOfChunk chunk) return false | PullResult.Eof -> Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) __.OverallStats.DumpIfIntervalExpired(true) - let! _ = post (ReadItem.EndOfChunk chunk) + let! _ = post (ReadResult.EndOfChunk chunk) return true | PullResult.Exn e -> let bs = adjust batchSize @@ -248,7 +274,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let slicesStats, stats = SliceStatsBuffer(), OverallStats() use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let progressSw = Stopwatch.StartNew() - let postBatch pos items = post (ReadItem.Batch (pos, items)) + let postBatch pos items = post (ReadResult.Batch (pos, items)) while true do let currentPos = range.Current if progressSw.ElapsedMilliseconds > progressIntervalMs then @@ -264,4 +290,82 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = batchSize <- adjust batchSize Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) stats.DumpIfIntervalExpired() - return true } \ No newline at end of file + return true } + +/// Handles Tailing mode - a single reader thread together with a (limited) set of concurrent of stream-catchup readers +type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxCatchupReaders, ?statsInterval) = + // to avoid busy waiting in main message pummp loop + let sleepIntervalMs = 100 + let dop = new SemaphoreSlim(1 + maxCatchupReaders) + let work = ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) + member __.HasCapacity = work.QueueCount < dop.CurrentCount + // TODO reinstate usage + member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) + + /// Single invcation will run until Cancelled, spawning child threads as necessary + member __.Pump(post, startPos, max, tailInterval) = async { + let! ct = Async.CancellationToken + work.AddTail(startPos, max, tailInterval) + while not ct.IsCancellationRequested do + work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let! _ = work.Process(conn, tryMapEvent, post, task) in () + finally dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + dop.Release() |> ignore + do! Async.Sleep sleepIntervalMs } + +/// Handles bulk ingestion - a specified number of concurrent reader threads round-robin over a supplied set of connections, taking the next available 256MB +/// chunk from the tail upon completion of the specified `Range` delimiting the chunk +type StripedReader(conns : _ array, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = + let dop = new SemaphoreSlim(maxDop) + let work = ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) + + /// Single invocation; spawns child threads within defined limits; exits when End of Store has been reached + member __.Pump(post, startPos, max) = async { + let! ct = Async.CancellationToken + let mutable remainder = + let nextPos = posFromChunkAfter startPos + let startChunk = chunk startPos |> int + work.AddTranche(startChunk, startPos, nextPos, max) + Some nextPos + let mutable finished = false + let r = new Random() + let mutable robin = 0 + while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do + work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let connIndex = Interlocked.Increment(&robin) % conns.Length + let conn = conns.[connIndex] + let! eof = work.Process(conn, tryMapEvent, post, task) in () + if eof then remainder <- None + finally dop.Release() |> ignore } + return () } + let currentCount = dop.CurrentCount + // Jitter is most relevant when processing commences - any commencement of a chunk can trigger significant page faults on server + // which we want to attempt to limit the effects of + let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) + Log.Warning("Waiting {jitter}ms to jitter reader stripes, {currentCount} further reader stripes awaiting start", jitter, currentCount) + do! Async.Sleep jitter + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + match remainder with + | Some pos -> + let nextPos = posFromChunkAfter pos + remainder <- Some nextPos + let chunkNumber = chunk pos |> int + do! forkRunRelease <| ReadRequest.Chunk (chunkNumber, Range(pos, Some nextPos, max), batchSize) + | None -> + if finished then do! Async.Sleep 1000 + else Log.Error("No further ingestion work to commence") + finished <- true } \ No newline at end of file diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 3a00ff679..9387ba6e6 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -1,13 +1,12 @@ [] module SyncTemplate.Infrastructure -open Equinox.Store // AwaitTaskCorrect open System open System.Threading open System.Threading.Tasks -//#nowarn "21" // re AwaitKeyboardInterrupt -//#nowarn "40" // re AwaitKeyboardInterrupt +#nowarn "21" // re AwaitKeyboardInterrupt +#nowarn "40" // re AwaitKeyboardInterrupt type Async with static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) @@ -17,31 +16,4 @@ type Async with let isDisposed = ref 0 let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) - -//module Queue = -// let tryDequeue (x : System.Collections.Generic.Queue<'T>) = -//#if NET461 -// if x.Count = 0 then None -// else x.Dequeue() |> Some -//#else -// match x.TryDequeue() with -// | false, _ -> None -// | true, res -> Some res -//#endif - -type SemaphoreSlim with - /// F# friendly semaphore await function - member semaphore.Await(?timeout : TimeSpan) = async { - let! ct = Async.CancellationToken - let timeout = defaultArg timeout Timeout.InfiniteTimeSpan - let task = semaphore.WaitAsync(timeout, ct) - return! Async.AwaitTaskCorrect task - } - -// /// Throttling wrapper which waits asynchronously until the semaphore has available capacity -// member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { -// let! _ = semaphore.Await() -// try return! workflow -// finally semaphore.Release() |> ignore -// } \ No newline at end of file + in ()) \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 40476b3c3..bd899641a 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -3,10 +3,10 @@ open Equinox.Cosmos //#if !eventStore open Equinox.Cosmos.Projection -open Equinox.Cosmos.Projection.Ingestion //#else open Equinox.EventStore //#endif +open Equinox.Cosmos.Projection.Ingestion open Equinox.Projection.Engine open Equinox.Projection.State //#if !eventStore @@ -16,7 +16,6 @@ open Serilog open System //#if !eventStore open System.Collections.Generic -//#else open System.Diagnostics open System.Threading //#endif @@ -305,84 +304,7 @@ module CmdParser = #if !cosmos module EventStoreSource = - type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = - let sleepIntervalMs = 100 - let dop = new SemaphoreSlim(maxDop) - let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) - member __.HasCapacity = work.QueueCount < dop.CurrentCount - // TODO stuff - member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) - member __.Pump(post, startPos, max, tailInterval) = async { - let! ct = Async.CancellationToken - work.AddTail(startPos, max, tailInterval) - while not ct.IsCancellationRequested do - work.OverallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, tryMapEvent, post, task) in () - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - dop.Release() |> ignore - do! Async.Sleep sleepIntervalMs } - - type StripedReader(conns : _ array, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = - let dop = new SemaphoreSlim(maxDop) - let work = EventStoreSource.ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) - - member __.Pump(post, startPos, max) = async { - let! ct = Async.CancellationToken - let mutable remainder = - let nextPos = EventStoreSource.posFromChunkAfter startPos - let startChunk = EventStoreSource.chunk startPos |> int - work.AddTranche(startChunk, startPos, nextPos, max) - Some nextPos - let mutable finished = false - let r = new Random() - let mutable robin = 0 - while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do - work.OverallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let connIndex = Interlocked.Increment(&robin) % conns.Length - let conn = conns.[connIndex] - let! eof = work.Process(conn, tryMapEvent, post, task) in () - if eof then remainder <- None - finally dop.Release() |> ignore } - return () } - let currentCount = dop.CurrentCount - let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) - Log.Warning("Waiting {jitter}ms to jitter reader stripes, {currentCount} slots open", jitter, currentCount) - do! Async.Sleep jitter - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - match remainder with - | Some pos -> - let nextPos = EventStoreSource.posFromChunkAfter pos - remainder <- Some nextPos - let chunkNumber = EventStoreSource.chunk pos |> int - do! forkRunRelease <| EventStoreSource.Work.Tranche (chunkNumber, EventStoreSource.Range(pos, Some nextPos, max), batchSize) - | None -> - if finished then do! Async.Sleep 1000 - else Log.Error("No further ingestion work to commence") - finished <- true } - type StartMode = Starting | Resuming | Overridding - - //// 4. Enqueue streams with gaps if there is capacity (not overloading, to avoid redundant work) - //let mutable more = true - //while more && readers.HasCapacity do - // match buffer.TryGap() with - // | Some (stream,pos,len) -> readers.AddStreamPrefix(stream,pos,len) - // | None -> more <- false - let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let conn = connect () @@ -418,15 +340,15 @@ module EventStoreSource = let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) if spec.gorge then let conns = [| yield conn; yield! Seq.init (spec.stripes-1) (ignore >> connect) |] - let readers = StripedReader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) + let readers = EventStoreSource.StripedReader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) let post = function - | EventStoreSource.ReadItem.ChunkBatch (chunk, pos, xs) -> + | EventStoreSource.ReadResult.ChunkBatch (chunk, pos, xs) -> let cp = pos.CommitPosition trancheEngine.Submit <| Push.ChunkBatch(chunk, cp, checkpoints.Commit cp, xs) - | EventStoreSource.ReadItem.EndOfChunk chunk -> + | EventStoreSource.ReadResult.EndOfChunk chunk -> trancheEngine.Submit <| Push.EndOfChunk chunk - | EventStoreSource.ReadItem.Batch _ - | EventStoreSource.ReadItem.StreamSpan _ as x -> + | EventStoreSource.ReadResult.Batch _ + | EventStoreSource.ReadResult.StreamSpan _ as x -> failwithf "%A not supported when gorging" x let startChunk = EventStoreSource.chunk startPos |> int let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) @@ -434,15 +356,15 @@ module EventStoreSource = do! readers.Pump(post, startPos, max) else let post = function - | EventStoreSource.ReadItem.Batch (pos, xs) -> + | EventStoreSource.ReadResult.Batch (pos, xs) -> let cp = pos.CommitPosition trancheEngine.Submit(cp, checkpoints.Commit cp, xs) - | EventStoreSource.ReadItem.StreamSpan span -> + | EventStoreSource.ReadResult.StreamSpan span -> trancheEngine.Submit <| Push.Stream span - | EventStoreSource.ReadItem.ChunkBatch _ - | EventStoreSource.ReadItem.EndOfChunk _ as x -> + | EventStoreSource.ReadResult.ChunkBatch _ + | EventStoreSource.ReadResult.EndOfChunk _ as x -> failwithf "%A not supported when tailing" x - let readers = TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes + 1) + let readers = EventStoreSource.TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) do! readers.Pump(post, startPos, max, spec.tailInterval) } #else From 1e4dc94c8bcd70116ee8d888287161187fdb321d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 20:02:55 +0100 Subject: [PATCH 193/353] Reorg Engine components --- .../Equinox.Projection/Engine.fs | 37 +++++++++++++++++ equinox-projector/Equinox.Projection/State.fs | 41 +------------------ 2 files changed, 38 insertions(+), 40 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 46d4325e5..21338b50a 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -5,6 +5,7 @@ open Serilog open System open System.Collections.Concurrent open System.Collections.Generic +open System.Diagnostics open System.Threading /// Item from a reader as supplied to the projector/ingestor loop for aggregation @@ -22,6 +23,13 @@ type ProjectionMessage<'R> = /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> +let expiredMs ms = + let timer = Stopwatch.StartNew() + fun () -> + let due = timer.ElapsedMilliseconds > ms + if due then timer.Restart() + due + /// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 @@ -52,6 +60,29 @@ type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () +/// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint +type Dispatcher<'R>(maxDop) = + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let result = Event<'R>() + let dop = new SemaphoreSlim(maxDop) + let dispatch work = async { + let! res = work + result.Trigger res + dop.Release() |> ignore } + [] member __.Result = result.Publish + member __.AvailableCapacity = + let available = dop.CurrentCount + available,maxDop + member __.TryAdd(item,?timeout) = async { + let! got = dop.Await(?timeout=timeout) + if got then + work.Add(item) + return got } + member __.Pump () = async { + let! ct = Async.CancellationToken + for item in work.GetConsumingEnumerable ct do + Async.Start(dispatch item) } + /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order /// a) does not itself perform any reading activities /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) @@ -190,6 +221,12 @@ type TrancheStreamBuffer() = if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) +let every ms f = + let timer = Stopwatch.StartNew() + fun () -> + if timer.ElapsedMilliseconds > ms then + f () + timer.Restart() /// Manages writing of progress /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) /// - retries until success or a new item is posted diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 6c2c97680..e9746ad0e 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -2,22 +2,6 @@ open Serilog open System.Collections.Generic -open System.Diagnostics -open System.Threading -open System.Collections.Concurrent - -let every ms f = - let timer = Stopwatch.StartNew() - fun () -> - if timer.ElapsedMilliseconds > ms then - f () - timer.Restart() -let expiredMs ms = - let timer = Stopwatch.StartNew() - fun () -> - let due = timer.ElapsedMilliseconds > ms - if due then timer.Restart() - due let arrayBytes (x:byte[]) = if x = null then 0 else x.Length let mb x = float x / 1024. / 1024. @@ -235,27 +219,4 @@ type ProgressState<'Pos>() = else let item = pending.Dequeue() in item.markCompleted() aux (completed + 1) - aux 0 - -/// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint -type Dispatcher<'R>(maxDop) = - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) - let result = Event<'R>() - let dop = new SemaphoreSlim(maxDop) - let dispatch work = async { - let! res = work - result.Trigger res - dop.Release() |> ignore } - [] member __.Result = result.Publish - member __.AvailableCapacity = - let available = dop.CurrentCount - available,maxDop - member __.TryAdd(item,?timeout) = async { - let! got = dop.Await(?timeout=timeout) - if got then - work.Add(item) - return got } - member __.Pump () = async { - let! ct = Async.CancellationToken - for item in work.GetConsumingEnumerable ct do - Async.Start(dispatch item) } \ No newline at end of file + aux 0 \ No newline at end of file From 7404730b814205f1984ae076cd02005e3f789fd8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 1 May 2019 23:45:26 +0100 Subject: [PATCH 194/353] Transition from Gorging to Tailing at end --- equinox-sync/Sync/EventStoreSource.fs | 159 ++++++++++++++++++++------ equinox-sync/Sync/Program.fs | 101 ++-------------- 2 files changed, 130 insertions(+), 130 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index fd6c0e5e4..ef2c014fb 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -1,5 +1,6 @@ module SyncTemplate.EventStoreSource +open Equinox.Cosmos.Projection.Ingestion open Equinox.Store // AwaitTaskCorrect open Equinox.Projection open Equinox.Projection.Engine @@ -197,10 +198,11 @@ type ReadResult = /// Holds work queue, together with stats relating to the amount and/or categories of data being traversed /// Processing is driven by external callers running multiple concurrent invocations of `Process` -type private ReadQueue(batchSize, minBatchSize, ?statsInterval) = +type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() member val OverallStats = OverallStats(?statsInterval=statsInterval) member val SlicesStats = SliceStatsBuffer() + member __.DefaultBatchSize = batchSize member __.QueueCount = work.Count member __.AddStream(name, ?batchSizeOverride) = work.Enqueue <| ReadRequest.Stream (name, defaultArg batchSizeOverride batchSize) @@ -292,40 +294,10 @@ type private ReadQueue(batchSize, minBatchSize, ?statsInterval) = stats.DumpIfIntervalExpired() return true } -/// Handles Tailing mode - a single reader thread together with a (limited) set of concurrent of stream-catchup readers -type TailAndPrefixesReader(conn, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxCatchupReaders, ?statsInterval) = - // to avoid busy waiting in main message pummp loop - let sleepIntervalMs = 100 - let dop = new SemaphoreSlim(1 + maxCatchupReaders) - let work = ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) - member __.HasCapacity = work.QueueCount < dop.CurrentCount - // TODO reinstate usage - member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) - - /// Single invcation will run until Cancelled, spawning child threads as necessary - member __.Pump(post, startPos, max, tailInterval) = async { - let! ct = Async.CancellationToken - work.AddTail(startPos, max, tailInterval) - while not ct.IsCancellationRequested do - work.OverallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, tryMapEvent, post, task) in () - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - dop.Release() |> ignore - do! Async.Sleep sleepIntervalMs } - /// Handles bulk ingestion - a specified number of concurrent reader threads round-robin over a supplied set of connections, taking the next available 256MB /// chunk from the tail upon completion of the specified `Range` delimiting the chunk -type StripedReader(conns : _ array, batchSize, minBatchSize, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop, ?statsInterval) = +type StripedReader(conns : _ array, work: ReadQueue, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop) = let dop = new SemaphoreSlim(maxDop) - let work = ReadQueue(batchSize, minBatchSize, ?statsInterval=statsInterval) /// Single invocation; spawns child threads within defined limits; exits when End of Store has been reached member __.Pump(post, startPos, max) = async { @@ -364,8 +336,123 @@ type StripedReader(conns : _ array, batchSize, minBatchSize, tryMapEvent: EventS let nextPos = posFromChunkAfter pos remainder <- Some nextPos let chunkNumber = chunk pos |> int - do! forkRunRelease <| ReadRequest.Chunk (chunkNumber, Range(pos, Some nextPos, max), batchSize) + do! forkRunRelease <| ReadRequest.Chunk (chunkNumber, Range(pos, Some nextPos, max), work.DefaultBatchSize) | None -> - if finished then do! Async.Sleep 1000 - else Log.Error("No further ingestion work to commence") - finished <- true } \ No newline at end of file + Log.Warning("No further ingestion work to commence, transitioning to tailing...") + finished <- true } + +/// Handles Tailing mode - a single reader thread together with a (limited) set of concurrent of stream-catchup readers +type TailAndPrefixesReader(conn, work: ReadQueue, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxCatchupReaders) = + // to avoid busy waiting in main message pummp loop + let sleepIntervalMs = 100 + let dop = new SemaphoreSlim(1 + maxCatchupReaders) + member __.HasCapacity = work.QueueCount < dop.CurrentCount + // TODO reinstate usage + member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) + + /// Single invcation will run until Cancelled, spawning child threads as necessary + member __.Pump(post, startPos, max, tailInterval) = async { + let! ct = Async.CancellationToken + work.AddTail(startPos, max, tailInterval) + while not ct.IsCancellationRequested do + work.OverallStats.DumpIfIntervalExpired() + let! _ = dop.Await() + let forkRunRelease task = async { + let! _ = Async.StartChild <| async { + try let! _ = work.Process(conn, tryMapEvent, post, task) in () + finally dop.Release() |> ignore } + return () } + match work.TryDequeue() with + | true, task -> + do! forkRunRelease task + | false, _ -> + dop.Release() |> ignore + do! Async.Sleep sleepIntervalMs } + +type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint + +type ReaderSpec = + { /// Identifier for this projection and it's state + groupName: string + /// Indicates user has specified that they wish to restart from the indicated position as opposed to resuming from the checkpoint position + forceRestart: bool + /// Start position from which forward reading is to commence // Assuming no stored position + start: StartPos + checkpointInterval: TimeSpan + /// Delay when reading yields an empty batch + tailInterval: TimeSpan + gorge: bool + /// Maximum number of striped readers to permit + stripes: int + /// Initial batch size to use when commencing reading + batchSize: int + /// Smallest batch size to degrade to in the presence of failures + minBatchSize: int } + +type StartMode = Starting | Resuming | Overridding + +let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { + let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) + let conn = connect () + let! maxInParallel = Async.StartChild <| establishMax conn + let! initialCheckpointState = checkpoints.Read + let! max = maxInParallel + let! startPos = async { + let mkPos x = EventStore.ClientAPI.Position(x, 0L) + let requestedStartPos = + match spec.start with + | Absolute p -> mkPos p + | Chunk c -> posFromChunk c + | Percentage pct -> posFromPercentage (pct, max) + | TailOrCheckpoint -> max + | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start + let! startMode, startPos, checkpointFreq = async { + match initialCheckpointState, requestedStartPos with + | Checkpoint.Folds.NotStarted, r -> + if spec.forceRestart then invalidOp "Cannot specify --forceRestart when no progress yet committed" + do! checkpoints.Start(spec.checkpointInterval, r.CommitPosition) + return Starting, r, spec.checkpointInterval + | Checkpoint.Folds.Running s, _ when not spec.forceRestart -> + return Resuming, mkPos s.state.pos, TimeSpan.FromSeconds(float s.config.checkpointFreqS) + | Checkpoint.Folds.Running _, r -> + do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) + return Overridding, r, spec.checkpointInterval + } + log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) checkpointing every {checkpointFreq:n1}m", + startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition, + checkpointFreq.TotalMinutes) + return startPos } + let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) + let queue = ReadQueue(spec.batchSize, spec.minBatchSize) + if spec.gorge then + let extraConns = Seq.init (spec.stripes-1) (ignore >> connect) + let conns = [| yield conn; yield! extraConns |] + let post = function + | ReadResult.ChunkBatch (chunk, pos, xs) -> + let cp = pos.CommitPosition + trancheEngine.Submit <| Push.ChunkBatch(chunk, cp, checkpoints.Commit cp, xs) + | ReadResult.EndOfChunk chunk -> + trancheEngine.Submit <| Push.EndOfChunk chunk + | ReadResult.Batch _ + | ReadResult.StreamSpan _ as x -> + failwithf "%A not supported when gorging" x + let startChunk = chunk startPos |> int + let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) + log.Information("Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) + let gorgingReader = StripedReader(conns, queue, tryMapEvent, spec.stripes) + do! gorgingReader.Pump(post, startPos, max) + for x in extraConns do x.Close() + // After doing the gorging, we switch to normal tailing (which avoids the app exiting too) + let post = function + | ReadResult.Batch (pos, xs) -> + let cp = pos.CommitPosition + trancheEngine.Submit(cp, checkpoints.Commit cp, xs) + | ReadResult.StreamSpan span -> + trancheEngine.Submit <| Push.Stream span + | ReadResult.ChunkBatch _ + | ReadResult.EndOfChunk _ as x -> + failwithf "%A not supported when tailing" x + let readers = TailAndPrefixesReader(conn, queue, tryMapEvent, spec.stripes) + log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) + do! readers.Pump(post, startPos, max, spec.tailInterval) } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index bd899641a..507246f49 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -20,28 +20,6 @@ open System.Diagnostics open System.Threading //#endif -//#if eventStore -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint - -type ReaderSpec = - { /// Identifier for this projection and it's state - groupName: string - /// Indicates user has specified that they wish to restart from the indicated position as opposed to resuming from the checkpoint position - forceRestart: bool - /// Start position from which forward reading is to commence // Assuming no stored position - start: StartPos - checkpointInterval: TimeSpan - /// Delay when reading yields an empty batch - tailInterval: TimeSpan - gorge: bool - /// Maximum number of striped readers to permit - stripes: int - /// Initial batch size to use when commencing reading - batchSize: int - /// Smallest batch size to degrade to in the presence of failures - minBatchSize: int } -//#endif - module CmdParser = open Argu @@ -149,14 +127,14 @@ module CmdParser = x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) disco, db, x.LeaseId, x.StartFromHere, x.BatchSize, x.LagFrequency #else - member x.BuildFeedParams() : ReaderSpec = + member x.BuildFeedParams() : EventStoreSource.ReaderSpec = let startPos = match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains FromTail with - | Some p, _, _, _ -> Absolute p - | _, Some c, _, _ -> StartPos.Chunk c - | _, _, Some p, _ -> Percentage p - | None, None, None, true -> StartPos.TailOrCheckpoint - | None, None, None, _ -> StartPos.StartOrCheckpoint + | Some p, _, _, _ -> EventStoreSource.Absolute p + | _, Some c, _, _ -> EventStoreSource.StartPos.Chunk c + | _, _, Some p, _ -> EventStoreSource.Percentage p + | None, None, None, true -> EventStoreSource.StartPos.TailOrCheckpoint + | None, None, None, _ -> EventStoreSource.StartPos.StartOrCheckpoint Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}], reading up to {maxPendingBatches} uncommitted batches ahead", @@ -302,72 +280,7 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -#if !cosmos -module EventStoreSource = - type StartMode = Starting | Resuming | Overridding - let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { - let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) - let conn = connect () - let! maxInParallel = Async.StartChild <| EventStoreSource.establishMax conn - let! initialCheckpointState = checkpoints.Read - let! max = maxInParallel - let! startPos = async { - let mkPos x = EventStore.ClientAPI.Position(x, 0L) - let requestedStartPos = - match spec.start with - | Absolute p -> mkPos p - | Chunk c -> EventStoreSource.posFromChunk c - | Percentage pct -> EventStoreSource.posFromPercentage (pct, max) - | TailOrCheckpoint -> max - | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start - let! startMode, startPos, checkpointFreq = async { - match initialCheckpointState, requestedStartPos with - | Checkpoint.Folds.NotStarted, r -> - if spec.forceRestart then raise <| CmdParser.InvalidArguments ("Cannot specify --forceRestart when no progress yet committed") - do! checkpoints.Start(spec.checkpointInterval, r.CommitPosition) - return Starting, r, spec.checkpointInterval - | Checkpoint.Folds.Running s, _ when not spec.forceRestart -> - return Resuming, mkPos s.state.pos, TimeSpan.FromSeconds(float s.config.checkpointFreqS) - | Checkpoint.Folds.Running _, r -> - do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) - return Overridding, r, spec.checkpointInterval - } - log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) checkpointing every {checkpointFreq:n1}m", - startMode, spec.groupName, startPos.CommitPosition, EventStoreSource.chunk startPos, float startPos.CommitPosition/float max.CommitPosition, - checkpointFreq.TotalMinutes) - return startPos } - let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) - let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) - if spec.gorge then - let conns = [| yield conn; yield! Seq.init (spec.stripes-1) (ignore >> connect) |] - let readers = EventStoreSource.StripedReader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) - let post = function - | EventStoreSource.ReadResult.ChunkBatch (chunk, pos, xs) -> - let cp = pos.CommitPosition - trancheEngine.Submit <| Push.ChunkBatch(chunk, cp, checkpoints.Commit cp, xs) - | EventStoreSource.ReadResult.EndOfChunk chunk -> - trancheEngine.Submit <| Push.EndOfChunk chunk - | EventStoreSource.ReadResult.Batch _ - | EventStoreSource.ReadResult.StreamSpan _ as x -> - failwithf "%A not supported when gorging" x - let startChunk = EventStoreSource.chunk startPos |> int - let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) - log.Information("Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) - do! readers.Pump(post, startPos, max) - else - let post = function - | EventStoreSource.ReadResult.Batch (pos, xs) -> - let cp = pos.CommitPosition - trancheEngine.Submit(cp, checkpoints.Commit cp, xs) - | EventStoreSource.ReadResult.StreamSpan span -> - trancheEngine.Submit <| Push.Stream span - | EventStoreSource.ReadResult.ChunkBatch _ - | EventStoreSource.ReadResult.EndOfChunk _ as x -> - failwithf "%A not supported when tailing" x - let readers = EventStoreSource.TailAndPrefixesReader(conn, spec.batchSize, spec.minBatchSize, tryMapEvent, spec.stripes) - log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) - do! readers.Pump(post, startPos, max, spec.tailInterval) } -#else +#if cosmos module CosmosSource = open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing From 75b8d7fcf5a53dbea80bdc408f02c3eaa54bc752 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 2 May 2019 00:43:37 +0100 Subject: [PATCH 195/353] Split out CosmosSource --- equinox-sync/Sync/CosmosSource.fs | 104 +++++++++++++++++++++ equinox-sync/Sync/EventStoreSource.fs | 8 +- equinox-sync/Sync/Program.fs | 124 +------------------------- equinox-sync/Sync/Sync.fsproj | 4 +- 4 files changed, 114 insertions(+), 126 deletions(-) create mode 100644 equinox-sync/Sync/CosmosSource.fs diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs new file mode 100644 index 000000000..74c1d7857 --- /dev/null +++ b/equinox-sync/Sync/CosmosSource.fs @@ -0,0 +1,104 @@ +module SyncTemplate.CosmosSource + +open Equinox.Cosmos.Core +open Equinox.Cosmos.Projection +open Equinox.Projection.Engine +open Equinox.Projection.State +open Equinox.Store // AwaitTaskCorrect +open Microsoft.Azure.Documents +open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing +open Serilog +open System +open System.Collections.Generic + +let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: CosmosContext, maxWriters) (transform : Document -> StreamItem seq) = + let ingestionEngine = Equinox.Cosmos.Projection.Ingestion.startIngestionEngine (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let mutable trancheEngine = Unchecked.defaultof<_> + let init rangeLog = + trancheEngine <- Equinox.Projection.Engine.TrancheEngine.Start (rangeLog, ingestionEngine, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) + let ingest epoch checkpoint docs = + let events = docs |> Seq.collect transform |> Array.ofSeq + trancheEngine.Submit(epoch, checkpoint, events) + let dispose () = trancheEngine.Stop () + let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok + let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 + // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved + let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } + let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time + log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", + epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., + let e = pt.Elapsed in e.TotalSeconds, cur, max) + sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + } + ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) + +let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) + createRangeProjector = async { + let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { + Log.Information("Lags {@rangeLags} (Range, Docs count)", remainingWork) + return! Async.Sleep interval } + let maybeLogLag = lagReportFreq |> Option.map logLag + let! _feedEventHost = + ChangeFeedProcessor.Start + ( Log.Logger, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = forceSkip, + cfBatchSize = batchSize, createObserver = createRangeProjector, ?reportLagAndAwaitNextEstimation = maybeLogLag) + do! Async.AwaitKeyboardInterrupt() } + +//#if marveleqx +[] +module EventV0Parser = + open Newtonsoft.Json + + /// A single Domain Event as Written by internal Equinox versions + type [] + EventV0 = + { /// DocDb-mandated Partition Key, must be maintained within the document + s: string // "{streamName}" + + /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) + c: DateTimeOffset // ISO 8601 + + /// The Case (Event Type); used to drive deserialization + t: string // required + + /// 'i' value for the Event + i: int64 // {index} + + /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb + [)>] + d: byte[] } + + type Document with + member document.Cast<'T>() = + let tmp = new Document() + tmp.SetPropertyValue("content", document) + tmp.GetPropertyValue<'T>("content") + type IEvent = + inherit Equinox.Codec.Core.IIndexedEvent + abstract member Stream : string + /// We assume all Documents represent Events laid out as above + let parse (d : Document) = + let x = d.Cast() + { new IEvent with + member __.Index = x.i + member __.IsUnfold = false + member __.EventType = x.t + member __.Data = x.d + member __.Meta = null + member __.Timestamp = x.c + member __.Stream = x.s } + +let transformV0 catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { + let parsed = EventV0Parser.parse v0SchemaDocument + let streamName = (*if parsed.Stream.Contains '-' then parsed.Stream else "Prefixed-"+*)parsed.Stream + if catFilter (category streamName) then + yield { stream = streamName; index = parsed.Index; event = parsed } } +//#else +let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq = seq { + for e in DocumentParser.enumEvents changeFeedDocument do + if catFilter (category e.Stream) then + // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level + yield { stream = e.Stream; index = e.Index; event = e } } +//#endif \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index ef2c014fb..b92e99931 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -424,7 +424,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro return startPos } let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) - let queue = ReadQueue(spec.batchSize, spec.minBatchSize) + let readerQueue = ReadQueue(spec.batchSize, spec.minBatchSize) if spec.gorge then let extraConns = Seq.init (spec.stripes-1) (ignore >> connect) let conns = [| yield conn; yield! extraConns |] @@ -440,7 +440,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro let startChunk = chunk startPos |> int let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) log.Information("Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) - let gorgingReader = StripedReader(conns, queue, tryMapEvent, spec.stripes) + let gorgingReader = StripedReader(conns, readerQueue, tryMapEvent, spec.stripes) do! gorgingReader.Pump(post, startPos, max) for x in extraConns do x.Close() // After doing the gorging, we switch to normal tailing (which avoids the app exiting too) @@ -453,6 +453,6 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro | ReadResult.ChunkBatch _ | ReadResult.EndOfChunk _ as x -> failwithf "%A not supported when tailing" x - let readers = TailAndPrefixesReader(conn, queue, tryMapEvent, spec.stripes) + let tailAndCatchupReaders = TailAndPrefixesReader(conn, readerQueue, tryMapEvent, spec.stripes) log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) - do! readers.Pump(post, startPos, max, spec.tailInterval) } \ No newline at end of file + do! tailAndCatchupReaders.Pump(post, startPos, max, spec.tailInterval) } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 507246f49..591048aa3 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -3,22 +3,11 @@ open Equinox.Cosmos //#if !eventStore open Equinox.Cosmos.Projection -//#else -open Equinox.EventStore -//#endif +//##endif open Equinox.Cosmos.Projection.Ingestion -open Equinox.Projection.Engine open Equinox.Projection.State -//#if !eventStore -open Equinox.Store -//#endif open Serilog open System -//#if !eventStore -open System.Collections.Generic -open System.Diagnostics -open System.Threading -//#endif module CmdParser = open Argu @@ -280,105 +269,6 @@ module CmdParser = let parser = ArgumentParser.Create(programName = programName) parser.ParseCommandLine argv |> Arguments -#if cosmos -module CosmosSource = - open Microsoft.Azure.Documents - open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing - - let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Core.CosmosContext, maxWriters) (transform : Microsoft.Azure.Documents.Document -> StreamItem seq) = - let ingestionEngine = Equinox.Cosmos.Projection.Ingestion.startIngestionEngine (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) - let maxUnconfirmedBatches = 10 - let mutable trancheEngine = Unchecked.defaultof<_> - let init rangeLog = - trancheEngine <- Equinox.Projection.Engine.TrancheEngine.Start (rangeLog, ingestionEngine, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) - let ingest epoch checkpoint docs = - let events = docs |> Seq.collect transform |> Array.ofSeq - trancheEngine.Submit(epoch, checkpoint, events) - let dispose () = trancheEngine.Stop () - let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok - let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved - let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } - let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", - epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., - let e = pt.Elapsed in e.TotalSeconds, cur, max) - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - } - ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) - - let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) - createRangeProjector = async { - let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - Log.Information("Lags {@rangeLags} (Range, Docs count)", remainingWork) - return! Async.Sleep interval } - let maybeLogLag = lagReportFreq |> Option.map logLag - let! _feedEventHost = - ChangeFeedProcessor.Start - ( Log.Logger, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = forceSkip, - cfBatchSize = batchSize, createObserver = createRangeProjector, ?reportLagAndAwaitNextEstimation = maybeLogLag) - do! Async.AwaitKeyboardInterrupt() } - - //#if marveleqx - [] - module EventV0Parser = - open Newtonsoft.Json - - /// A single Domain Event as Written by internal Equinox versions - type [] - EventV0 = - { /// DocDb-mandated Partition Key, must be maintained within the document - s: string // "{streamName}" - - /// Creation datetime (as opposed to system-defined _lastUpdated which is touched by triggers, replication etc.) - c: DateTimeOffset // ISO 8601 - - /// The Case (Event Type); used to drive deserialization - t: string // required - - /// 'i' value for the Event - i: int64 // {index} - - /// Event body, as UTF-8 encoded json ready to be injected into the Json being rendered for DocDb - [)>] - d: byte[] } - - type Document with - member document.Cast<'T>() = - let tmp = new Document() - tmp.SetPropertyValue("content", document) - tmp.GetPropertyValue<'T>("content") - type IEvent = - inherit Equinox.Codec.Core.IIndexedEvent - abstract member Stream : string - /// We assume all Documents represent Events laid out as above - let parse (d : Document) = - let x = d.Cast() - { new IEvent with - member __.Index = x.i - member __.IsUnfold = false - member __.EventType = x.t - member __.Data = x.d - member __.Meta = null - member __.Timestamp = x.c - member __.Stream = x.s } - - let transformV0 catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { - let parsed = EventV0Parser.parse v0SchemaDocument - let streamName = (*if parsed.Stream.Contains '-' then parsed.Stream else "Prefixed-"+*)parsed.Stream - if catFilter (category streamName) then - yield { stream = streamName; index = parsed.Index; event = parsed } } - //#else - let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq = seq { - for e in DocumentParser.enumEvents changeFeedDocument do - if catFilter (category e.Stream) then - // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level - yield { stream = e.Stream; index = e.Index; event = e } } - //#endif -#endif - // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog module Logging = @@ -459,17 +349,11 @@ let main argv = || e.EventStreamId.StartsWith "marvel_bookmark" || e.EventStreamId.EndsWith "_checkpoints" || e.EventStreamId.EndsWith "_checkpoint" - || e.EventStreamId.StartsWith("InventoryLog") // 5GB, causes lopsided partitions, unused - || e.EventStreamId = "ReloadBatchId" // does not start at 0 - || e.EventStreamId = "PurchaseOrder-5791" // item too large - || e.EventStreamId = "SkuFileUpload-99682b9cdbba4b09881d1d87dfdc1ded" - || e.EventStreamId = "SkuFileUpload-1e0626cc418548bc8eb82808426430e2" - || e.EventStreamId = "SkuFileUpload-6b4f566d90194263a2700c0ad1bc54dd" - || e.EventStreamId = "SkuFileUpload-5926b2d7512c4f859540f7f20e35242b" - || e.EventStreamId = "SkuFileUpload-9ac536b61fed4b44853a1f5e2c127d50" - || e.EventStreamId = "SkuFileUpload-b501837ce7e6416db80ca0c48a4b3f7a" || e.EventStreamId.StartsWith "Inventory-" // Too long || e.EventStreamId.StartsWith "InventoryCount-" // No Longer used + || e.EventStreamId.StartsWith "InventoryLog" // 5GB, causes lopsided partitions, unused + //|| e.EventStreamId = "ReloadBatchId" // does not start at 0 + //|| e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 9717a0c72..1a770ddca 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,13 +4,13 @@ Exe netcoreapp2.1 5 - cosmos_ + + - From e701d15b961aaec46272e0eb2d70644748f024d3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 2 May 2019 00:48:37 +0100 Subject: [PATCH 196/353] Fix cosmos switching args --- equinox-projector/Equinox.Projection/Engine.fs | 1 + equinox-sync/Sync/Program.fs | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs index 21338b50a..3a160d1d4 100644 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ b/equinox-projector/Equinox.Projection/Engine.fs @@ -227,6 +227,7 @@ let every ms f = if timer.ElapsedMilliseconds > ms then f () timer.Restart() + /// Manages writing of progress /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) /// - retries until success or a new item is posted diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 591048aa3..133ce1614 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -1,9 +1,11 @@ module SyncTemplate.Program open Equinox.Cosmos -//#if !eventStore +#if cosmos open Equinox.Cosmos.Projection -//##endif +#else +open Equinox.EventStore +#endif open Equinox.Cosmos.Projection.Ingestion open Equinox.Projection.State open Serilog From 5fd7791afc0526f38fc3bf2157820fa389aad6ba Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 2 May 2019 01:07:37 +0100 Subject: [PATCH 197/353] Tag logs --- equinox-projector/Projector/Infrastructure.fs | 9 +-------- equinox-sync/Sync/EventStoreSource.fs | 4 ++-- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/equinox-projector/Projector/Infrastructure.fs b/equinox-projector/Projector/Infrastructure.fs index 858bfce82..5fdef5dcb 100644 --- a/equinox-projector/Projector/Infrastructure.fs +++ b/equinox-projector/Projector/Infrastructure.fs @@ -36,11 +36,4 @@ type SemaphoreSlim with let timeout = defaultArg timeout Timeout.InfiniteTimeSpan let task = semaphore.WaitAsync(timeout, ct) return! Async.AwaitTaskCorrect task - } - - ///// Throttling wrapper that waits asynchronously until the semaphore has available capacity - //member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { - // let! _ = semaphore.Await() - // try return! workflow - // finally semaphore.Release() |> ignore - //} \ No newline at end of file + } \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index b92e99931..38b8ee7bb 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -422,8 +422,8 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let ingestionEngine = startIngestionEngine (log, maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) - let trancheEngine = TrancheEngine.Start (log, ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) + let ingestionEngine = startIngestionEngine (log.ForContext("Tranche","Cosmos"), maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let trancheEngine = TrancheEngine.Start (log.ForContext("Tranche","Tranches"), ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) let readerQueue = ReadQueue(spec.batchSize, spec.minBatchSize) if spec.gorge then let extraConns = Seq.init (spec.stripes-1) (ignore >> connect) From ac1837dde15ab117f1b0bc7bb7a0ca32e9d69914 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 May 2019 20:22:01 +0100 Subject: [PATCH 198/353] Rewrite :allthethings: --- .../{Ingestion.fs => CosmosIngester.fs} | 71 ++- .../Equinox.Cosmos.ProjectionEx.fsproj | 2 +- .../Equinox.Projection.Tests/ProgressTests.fs | 38 +- .../Equinox.Projection/Engine.fs | 438 ---------------- .../Equinox.Projection.fsproj | 2 +- .../Equinox.Projection/Infrastructure.fs | 1 - .../Equinox.Projection/Projection.fs | 471 ++++++++++++++++++ equinox-projector/Equinox.Projection/State.fs | 81 +-- equinox-projector/Projector/Program.fs | 12 +- equinox-sync/Sync/CosmosSource.fs | 74 ++- equinox-sync/Sync/EventStoreSource.fs | 312 +++++------- equinox-sync/Sync/Program.fs | 12 +- 12 files changed, 700 insertions(+), 814 deletions(-) rename equinox-projector/Equinox.Projection.Cosmos/{Ingestion.fs => CosmosIngester.fs} (72%) delete mode 100644 equinox-projector/Equinox.Projection/Engine.fs create mode 100644 equinox-projector/Equinox.Projection/Projection.fs diff --git a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs similarity index 72% rename from equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs rename to equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 69ab0c02b..9be03f028 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Ingestion.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -1,18 +1,15 @@ -module Equinox.Cosmos.Projection.Ingestion +module Equinox.Cosmos.Projection.CosmosIngester open Equinox.Cosmos.Core open Equinox.Cosmos.Store -open Equinox.Projection.Engine +open Equinox.Projection.Scheduling open Equinox.Projection.State open Serilog -open System.Threading - -let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 -let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 - -type [] ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Other +[] module Writer = + type [] ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Other + type [] Result = | Ok of updatedPos: int64 | Duplicate of updatedPos: int64 @@ -64,7 +61,7 @@ module Writer = | ResultKind.RateLimited | ResultKind.TimedOut | ResultKind.Other -> false | ResultKind.TooLarge | ResultKind.Malformed -> true -type CosmosStats(log : ILogger, statsInterval) = +type Stats(log : ILogger, statsInterval) = inherit Stats<(int*int)*Writer.Result>(log, statsInterval) let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 @@ -86,9 +83,9 @@ type CosmosStats(log : ILogger, statsInterval) = override __.Handle message = base.Handle message match message with - | ProjectionMessage.Add (_,_) - | ProjectionMessage.AddActive _ - | ProjectionMessage.Added _ -> () + | Add (_,_) + | AddActive _ + | Added _ -> () | Result (_stream, Choice1Of2 ((es,bs),r)) -> events <- events + es bytes <- bytes + int64 bs @@ -105,7 +102,9 @@ type CosmosStats(log : ILogger, statsInterval) = | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed | ResultKind.TimedOut -> incr timedOut -let startIngestionEngine (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, statsInterval) = +let start (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, statsInterval) = + let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 + let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 let writerResultLog = log.ForContext() let trim (writePos : int64 option, batch : StreamSpan) = let mutable bytesBudget = cosmosPayloadLimit @@ -114,39 +113,25 @@ let startIngestionEngine (log : Serilog.ILogger, maxPendingBatches, cosmosContex bytesBudget <- bytesBudget - cosmosPayloadBytes y count <- count + 1 // Reduce the item count when we don't yet know the write position in order to efficiently discover the redundancy where data is already present - count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) + count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) // always send at least one event in order to surface the problem and have the stream mark malformed { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } let project batch = async { let trimmed = trim batch try let! res = Writer.write log cosmosContext trimmed - let ctx = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes - return trimmed.stream, Choice1Of2 (ctx,res) - with e -> return trimmed.stream, Choice2Of2 e } - let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) res = + let stats = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes + return Choice1Of2 (stats,res) + with e -> return Choice2Of2 e } + let interpretProgress (streams: StreamStates) stream res = let applyResultToStreamState = function - | stream, (Choice1Of2 (ctx, Writer.Ok pos)) -> - Some ctx,streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (ctx, Writer.Duplicate pos)) -> - Some ctx,streams.InternalUpdate stream pos null - | stream, (Choice1Of2 (ctx, Writer.PartialDuplicate overage)) -> - Some ctx,streams.InternalUpdate stream overage.index [|overage|] - | stream, (Choice1Of2 (ctx, Writer.PrefixMissing (overage,pos))) -> - Some ctx,streams.InternalUpdate stream pos [|overage|] - | stream, (Choice2Of2 exn) -> + | Choice1Of2 (_stats, Writer.Ok pos) -> streams.InternalUpdate stream pos null + | Choice1Of2 (_stats, Writer.Duplicate pos) -> streams.InternalUpdate stream pos null + | Choice1Of2 (_stats, Writer.PartialDuplicate overage) -> streams.InternalUpdate stream overage.index [|overage|] + | Choice1Of2 (_stats, Writer.PrefixMissing (overage,pos)) -> streams.InternalUpdate stream pos [|overage|] + | Choice2Of2 exn -> let malformed = Writer.classify exn |> Writer.isMalformed - None,streams.SetMalformed(stream,malformed) - match res with - | ProjectionMessage.Result (s,r) -> - let _ctx,(stream,updatedState) = applyResultToStreamState (s,r) - match updatedState.write with - | Some wp -> - let closedBatches = progressState.MarkStreamProgress(stream, wp) - if closedBatches > 0 then - batches.Release(closedBatches) |> ignore - streams.MarkCompleted(stream,wp) - | None -> - streams.MarkFailed stream - Writer.logTo writerResultLog (s,r) - | _ -> () - let ingesterStats = CosmosStats(log, statsInterval) - ProjectionEngine<(int*int)*Writer.Result>.Start(ingesterStats, maxPendingBatches, maxWriters, project, handleResult) \ No newline at end of file + streams.SetMalformed(stream,malformed) + let _stream, { write = wp } = applyResultToStreamState res + Writer.logTo writerResultLog (stream,res) + wp + let projectionAndCosmosStats = Stats(log, statsInterval) + Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxPendingBatches, maxWriters, project, interpretProgress) \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj index 7d1b24529..4242a0a46 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj +++ b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj @@ -11,7 +11,7 @@ - + diff --git a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs index 06657c1b5..6a1c15eff 100644 --- a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs +++ b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs @@ -1,6 +1,6 @@ module ProgressTests -open Equinox.Projection.State +open Equinox.Projection open Swensen.Unquote open System.Collections.Generic open Xunit @@ -8,36 +8,32 @@ open Xunit let mkDictionary xs = Dictionary(dict xs) let [] ``Empty has zero streams pending or progress to write`` () = - let sut = ProgressState<_>() - let completed = sut.Validate(fun _ -> None) - 0 =! completed + let sut = Progress.State<_>() + let queue = sut.InScheduledOrder(fun _ -> 0) + test <@ Seq.isEmpty queue @> -let [] ``Can add multiple batches`` () = - let sut = ProgressState<_>() +let [] ``Can add multiple batches with overlapping streams`` () = + let sut = Progress.State<_>() let noBatchesComplete () = failwith "No bathes should complete" sut.AppendBatch(noBatchesComplete, mkDictionary ["a",1L; "b",2L]) sut.AppendBatch(noBatchesComplete, mkDictionary ["b",2L; "c",3L]) - let completed = sut.Validate(fun _ -> None) - 0 =! completed -let [] ``Marking Progress Removes batches and updates progress`` () = - let sut = ProgressState<_>() - let callbacks = ref 0 - let complete () = incr callbacks +let [] ``Marking Progress removes batches and triggers the callbacks`` () = + let sut = Progress.State<_>() + let mutable callbacks = 0 + let complete () = callbacks <- callbacks + 1 sut.AppendBatch(complete, mkDictionary ["a",1L; "b",2L]) sut.MarkStreamProgress("a",1L) |> ignore sut.MarkStreamProgress("b",1L) |> ignore - let completed = sut.Validate(fun _ -> None) - 0 =! completed - 1 =! !callbacks + 1 =! callbacks let [] ``Marking progress is not persistent`` () = - let sut = ProgressState<_>() - let callbacks = ref 0 - let complete () = incr callbacks + let sut = Progress.State<_>() + let mutable callbacks = 0 + let complete () = callbacks <- callbacks + 1 sut.AppendBatch(complete, mkDictionary ["a",1L]) sut.MarkStreamProgress("a",2L) |> ignore sut.AppendBatch(complete, mkDictionary ["a",1L; "b",2L]) - let completed = sut.Validate(fun _ -> None) - 0 =! completed - 1 =! !callbacks \ No newline at end of file + 1 =! callbacks + +// TODO: lots more coverage of newer functionality - the above were written very early into the exercise \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Engine.fs b/equinox-projector/Equinox.Projection/Engine.fs deleted file mode 100644 index 3a160d1d4..000000000 --- a/equinox-projector/Equinox.Projection/Engine.fs +++ /dev/null @@ -1,438 +0,0 @@ -module Equinox.Projection.Engine - -open Equinox.Projection.State -open Serilog -open System -open System.Collections.Concurrent -open System.Collections.Generic -open System.Diagnostics -open System.Threading - -/// Item from a reader as supplied to the projector/ingestor loop for aggregation -type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } - -/// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners -[] -type ProjectionMessage<'R> = - /// Enqueue a batch of items with supplied progress marking function - | Add of markCompleted: (unit -> unit) * items: StreamItem[] - /// Stats per submitted batch for stats listeners to aggregate - | Added of streams: int * skip: int * events: int - /// Submit new data pertaining to a stream that has commenced processing - | AddActive of KeyValuePair[] - /// Result of processing on stream - result (with basic stats) or the `exn` encountered - | Result of stream: string * outcome: Choice<'R,exn> - -let expiredMs ms = - let timer = Stopwatch.StartNew() - fun () -> - let due = timer.ElapsedMilliseconds > ms - if due then timer.Restart() - due - -/// Gathers stats pertaining to the core projection/ingestion activity -type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = - let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (available,maxDop) = - log.Information("Projection Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 - abstract member Handle : ProjectionMessage<'R> -> unit - default __.Handle msg = msg |> function - | Add _ | AddActive _ -> () - | Added (streams, skipped, events) -> - incr batchesPended - streamsPended := !streamsPended + streams - eventsPended := !eventsPended + events - eventsSkipped := !eventsSkipped + skipped - | Result (_stream, Choice1Of2 _) -> - incr resultCompleted - | Result (_stream, Choice2Of2 _) -> - incr resultExn - member __.TryDump((available,maxDop),streams : StreamStates) = - incr cycles - if statsDue () then - dumpStats (available,maxDop) - __.DumpExtraStats() - streams.Dump log - /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) - abstract DumpExtraStats : unit -> unit - default __.DumpExtraStats () = () - -/// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint -type Dispatcher<'R>(maxDop) = - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) - let result = Event<'R>() - let dop = new SemaphoreSlim(maxDop) - let dispatch work = async { - let! res = work - result.Trigger res - dop.Release() |> ignore } - [] member __.Result = result.Publish - member __.AvailableCapacity = - let available = dop.CurrentCount - available,maxDop - member __.TryAdd(item,?timeout) = async { - let! got = dop.Await(?timeout=timeout) - if got then - work.Add(item) - return got } - member __.Pump () = async { - let! ct = Async.CancellationToken - for item in work.GetConsumingEnumerable ct do - Async.Start(dispatch item) } - -/// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order -/// a) does not itself perform any reading activities -/// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) -/// c) submits work to the supplied Dispatcher (which it triggers pumping of) -/// d) periodically reports state (with hooks for ingestion engines to report same) -type ProjectionEngine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, handleResult) = - let sleepIntervalMs = 1 - let cts = new CancellationTokenSource() - let batches = new SemaphoreSlim(maxPendingBatches) - let work = ConcurrentQueue>() - let streams = StreamStates() - let progressState = ProgressState() - - member private __.Pump(stats : Stats<'R>) = async { - use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) - Async.Start(dispatcher.Pump(), cts.Token) - let validVsSkip (streamState : StreamState) (item : StreamItem) = - match streamState.write, item.index + 1L with - | Some cw, required when cw >= required -> 0, 1 - | _ -> 1, 0 - let handle x = - match x with - | Add (checkpoint, items) -> - let reqs = Dictionary() - let mutable count, skipCount = 0, 0 - for item in items do - let stream,streamState = streams.Add(item.stream,item.index,item.event) - match validVsSkip streamState item with - | 0, skip -> - skipCount <- skipCount + skip - | required, _ -> - count <- count + required - reqs.[stream] <- item.index+1L - progressState.AppendBatch(checkpoint,reqs) - work.Enqueue(Added (reqs.Count,skipCount,count)) - | AddActive events -> - for e in events do - streams.InternalMerge(e.Key,e.Value) - | Added _ -> - () - | Result _ as r -> - handleResult (streams, progressState, batches) r - - while not cts.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - // 2. Mark off any progress achieved (releasing memory and/or or unblocking reading of batches) - let completedBatches = progressState.Validate(streams.TryGetStreamWritePos) - if completedBatches > 0 then batches.Release(completedBatches) |> ignore - // 3. After that, top up provisioning of writers queue - let capacity,_ = dispatcher.AvailableCapacity - if capacity <> 0 then - let work = streams.Schedule(progressState.ScheduledOrder streams.QueueLength, capacity) - let xs = (Seq.ofArray work).GetEnumerator() - let mutable addsBeingAccepted = true - while xs.MoveNext() && addsBeingAccepted do - let! succeeded = dispatcher.TryAdd(project xs.Current) - addsBeingAccepted <- succeeded - // 4. Periodically emit status info - stats.TryDump(dispatcher.AvailableCapacity,streams) - // 5. Do a minimal sleep so we don't run completely hot when emprt - do! Async.Sleep sleepIntervalMs } - static member Start<'R>(stats, maxPendingBatches, processorDop, project, handleResult) = - let dispatcher = Dispatcher(processorDop) - let instance = new ProjectionEngine<'R>(maxPendingBatches, dispatcher, project, handleResult) - Async.Start <| instance.Pump(stats) - instance - - /// Attempt to feed in a batch (subject to there being capacity to do so) - member __.TrySubmit(markCompleted, events) = async { - let! got = batches.Await(TimeSpan.Zero) - if got then - work.Enqueue <| Add (markCompleted, events) - return got } - - member __.AddOpenStreamData(events) = - work.Enqueue <| AddActive events - - member __.AllStreams = streams.All - - member __.Stop() = - cts.Cancel() - -let startProjectionEngine (log, maxPendingBatches, processorDop, project : StreamSpan -> Async, statsInterval) = - let project (_maybeWritePos, batch) = async { - try let! count = project batch - return batch.stream, Choice1Of2 (batch.span.index + int64 count) - with e -> return batch.stream, Choice2Of2 e } - let handleResult (streams: StreamStates, progressState : ProgressState<_>, batches: SemaphoreSlim) = function - | Result (stream, Choice1Of2 index) -> - match progressState.MarkStreamProgress(stream,index) with 0 -> () | batchesCompleted -> batches.Release(batchesCompleted) |> ignore - streams.MarkCompleted(stream,index) - | Result (stream, Choice2Of2 _) -> - streams.MarkFailed stream - | _ -> () - let stats = Stats(log, statsInterval) - ProjectionEngine.Start(stats, maxPendingBatches, processorDop, project, handleResult) - -type Sem(max) = - let inner = new SemaphoreSlim(max) - member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore - member __.State = max-inner.CurrentCount,max - member __.Await() = inner.Await() |> Async.Ignore - member __.HasCapacity = inner.CurrentCount > 0 - member __.TryAwait(?timeout) = inner.Await(?timeout=timeout) - -type TrancheStreamBuffer() = - let states = Dictionary() - let merge stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - - member __.Merge(items : StreamItem seq) = - for item in items do - merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = Array.singleton item.event } |] } - - member __.Take(processingContains) = - let forward = [| for x in states do if processingContains x.Key then yield x |] - for x in forward do states.Remove x.Key |> ignore - forward - - member __.Dump(log : ILogger) = - let mutable waiting, waitingB = 0, 0L - let waitingCats, waitingStreams = CatStats(), CatStats() - for KeyValue (stream,state) in states do - let sz = int64 state.Size - waitingCats.Ingest(category stream) - waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) - waiting <- waiting + 1 - waitingB <- waitingB + sz - log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) - if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) - if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) - -let every ms f = - let timer = Stopwatch.StartNew() - fun () -> - if timer.ElapsedMilliseconds > ms then - f () - timer.Restart() - -/// Manages writing of progress -/// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) -/// - retries until success or a new item is posted -type ProgressWriter<'Res when 'Res: equality>() = - let pumpSleepMs = 100 - let due = expiredMs 5000L - let mutable committedEpoch = None - let mutable validatedPos = None - let result = Event>() - [] member __.Result = result.Publish - member __.Post(version,f) = - Volatile.Write(&validatedPos,Some (version,f)) - member __.CommittedEpoch = Volatile.Read(&committedEpoch) - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match Volatile.Read &validatedPos with - | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> - try do! f - Volatile.Write(&committedEpoch, Some v) - result.Trigger (Choice1Of2 v) - with e -> result.Trigger (Choice2Of2 e) - | _ -> do! Async.Sleep pumpSleepMs } - -[] -type SeriesMessage = - | Add of epoch: int64 * markCompleted: Async * items: StreamItem seq - | MoveToChunk of chunk: int - | AddStriped of chunk: int * epoch: int64 * markCompleted: Async * items: StreamItem seq - | EndOfChunk of chunk: int - | Added of streams: int * events: int - /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` - | ProgressResult of Choice - -type TrancheStats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = - let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None - let progCommitFails, progCommits = ref 0, ref 0 - let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (available,maxDop) = - log.Information("Tranche Cycles {cycles} Active {active}/{writers} Batches {batches} ({streams:n0}s {events:n0}e)", - !cycles, available, maxDop, !batchesPended, !streamsPended, !eventsPended) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 - if !progCommitFails <> 0 || !progCommits <> 0 then - match comittedEpoch with - | None -> - log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) - | Some committed when !progCommitFails <> 0 -> - log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) - | Some committed -> - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) - progCommits := 0; progCommitFails := 0 - else - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - member __.Handle : SeriesMessage -> unit = function - | Add _ | AddStriped _ | EndOfChunk _ | MoveToChunk _ -> - () // Enqueuing of an event is not interesting - we assume it'll get processed and mapped to an `Added` in the same cycle - | ProgressResult (Choice1Of2 epoch) -> - incr progCommits - comittedEpoch <- Some epoch - | ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progCommitFails - | Added (streams,events) -> - incr batchesPended - streamsPended := !streamsPended + streams - eventsPended := !eventsPended + events - member __.HandleValidated(epoch, pendingBatches) = - validatedEpoch <- epoch - pendingBatchCount <- pendingBatches - member __.HandleCommitted epoch = - comittedEpoch <- epoch - member __.TryDump((available,maxDop),streams : TrancheStreamBuffer) = - incr cycles - if statsDue () then - dumpStats (available,maxDop) - streams.Dump log - -[] -type Push = - | Batch of epoch: int64 * markCompleted: Async * items: StreamItem seq - | Stream of span: StreamSpan - | SetActiveChunk of chunk: int - | ChunkBatch of chunk: int * epoch: int64 * markCompleted: Async * items: StreamItem seq - | EndOfChunk of chunk: int - -/// Holds batches away from Core processing to limit in-flight processsing -type TrancheEngine<'R>(log : ILogger, ingester: ProjectionEngine<'R>, maxQueued, maxSubmissions, statsInterval : TimeSpan, ?pumpDelayMs) = - let cts = new CancellationTokenSource() - let work = ConcurrentQueue() - let read = new Sem(maxQueued) - let write = new Sem(maxSubmissions) - let streams = TrancheStreamBuffer() - let pending = Queue<_>() - let readingAhead, ready = Dictionary>(), Dictionary>() - let mutable validatedPos = None - let progressWriter = ProgressWriter<_>() - let stats = TrancheStats(log, maxQueued, statsInterval) - let pumpDelayMs = defaultArg pumpDelayMs 5 - - member private __.Pump() = async { - let mutable activeChunk = None - let ingestItems epoch checkpoint items = - let items = Array.ofSeq items - streams.Merge items - let markCompleted () = - write.Release() - read.Release() - validatedPos <- Some (epoch,checkpoint) - work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) - markCompleted, items - let tryRemove key (dict: Dictionary<_,_>) = - match dict.TryGetValue key with - | true, value -> - dict.Remove key |> ignore - Some value - | false, _ -> None - let handle = function - | Add (epoch, checkpoint, items) -> - let markCompleted,items = ingestItems epoch checkpoint items - pending.Enqueue((markCompleted,items)) - | AddStriped (chunk, epoch, checkpoint, items) -> - let batchInfo = ingestItems epoch checkpoint items - match activeChunk with - | Some c when c = chunk -> pending.Enqueue batchInfo - | _ -> - match readingAhead.TryGetValue chunk with - | false, _ -> readingAhead.[chunk] <- ResizeArray(Seq.singleton batchInfo) - | true,current -> current.Add(batchInfo) - | MoveToChunk newActiveChunk -> - activeChunk <- Some newActiveChunk - let buffered = - match ready |> tryRemove newActiveChunk with - | Some completedChunkBatches -> - completedChunkBatches |> Seq.iter pending.Enqueue - work.Enqueue <| MoveToChunk (newActiveChunk + 1) - completedChunkBatches.Count - | None -> - match readingAhead |> tryRemove newActiveChunk with - | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count - | None -> 0 - log.Information("Moving to chunk {activeChunk}, releasing {buffered} buffered items, {ready} others ready, {ahead} reading ahead", - newActiveChunk, buffered, ready.Count, readingAhead.Count) - | EndOfChunk chunk -> - match activeChunk with - | Some ac when ac = chunk -> - log.Information("Completed reading active chunk {activeChunk}, moving to next", ac) - work.Enqueue <| MoveToChunk (ac + 1) - | _ -> - match readingAhead |> tryRemove chunk with - | Some batchesRead -> - ready.[chunk] <- batchesRead - log.Information("Completed reading {chunkNo}, marking {buffered} buffered items ready", chunk, batchesRead.Count) - | None -> - ready.[chunk] <- ResizeArray() - log.Information("Completed reading {chunkNo}, leaving empty batch", chunk) - // These events are for stats purposes - | Added _ | ProgressResult _ -> () - use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) - Async.Start(progressWriter.Pump(), cts.Token) - while not cts.IsCancellationRequested do - work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - let mutable ingesterAccepting = true - // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && write.HasCapacity && ingesterAccepting do - let markCompleted, events = pending.Peek() - let! submitted = ingester.TrySubmit(markCompleted, events) - if submitted then - pending.Dequeue() |> ignore - // mark off a write as being in progress - do! write.Await() - else - ingesterAccepting <- false - // 2. Update any progress into the stats - stats.HandleValidated(Option.map fst validatedPos, fst read.State) - validatedPos |> Option.iter progressWriter.Post - stats.HandleCommitted progressWriter.CommittedEpoch - // 3. Forward content for any active streams into processor immediately - let relevantBufferedStreams = streams.Take(ingester.AllStreams.Contains) - ingester.AddOpenStreamData(relevantBufferedStreams) - // 4. Periodically emit status info - stats.TryDump(write.State,streams) - do! Async.Sleep pumpDelayMs } - - /// Awaits space in `read` to limit reading ahead - yields present state of Read and Write phases - member __.Submit(epoch, markBatchCompleted, events) = __.Submit <| Push.Batch (epoch, markBatchCompleted, events) - - /// Awaits space in `read` to limit reading ahead - yields present state of Read and Write phases - member __.Submit(content) = async { - do! read.Await() - match content with - | Push.Batch (epoch, markBatchCompleted, events) -> work.Enqueue <| Add (epoch, markBatchCompleted, events) - | Push.Stream _items -> failwith "TODO" - | Push.ChunkBatch (chunk, epoch, markBatchCompleted, events) -> work.Enqueue <| AddStriped (chunk, epoch, markBatchCompleted, events) - | Push.SetActiveChunk chunk -> work.Enqueue <| MoveToChunk chunk - | Push.EndOfChunk chunk -> work.Enqueue <| EndOfChunk chunk - return read.State } - - static member Start<'R>(log, ingester, maxRead, maxWrite, statsInterval) = - let instance = new TrancheEngine<'R>(log, ingester, maxRead, maxWrite, statsInterval) - Async.Start <| instance.Pump() - instance - - member __.Stop() = - cts.Cancel() \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj index 3035b7d4b..cf5c893ef 100644 --- a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj +++ b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj @@ -12,7 +12,7 @@ - + diff --git a/equinox-projector/Equinox.Projection/Infrastructure.fs b/equinox-projector/Equinox.Projection/Infrastructure.fs index 15a93cb57..cb82b676a 100644 --- a/equinox-projector/Equinox.Projection/Infrastructure.fs +++ b/equinox-projector/Equinox.Projection/Infrastructure.fs @@ -15,7 +15,6 @@ module ConcurrentQueue = aux () type Async with - static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) /// Asynchronously awaits the next keyboard interrupt event static member AwaitKeyboardInterrupt () : Async = Async.FromContinuations(fun (sc,_,_) -> diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs new file mode 100644 index 000000000..e0692ea73 --- /dev/null +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -0,0 +1,471 @@ +namespace Equinox.Projection + +open Equinox.Projection.State +open Serilog +open System +open System.Collections.Concurrent +open System.Collections.Generic +open System.Diagnostics +open System.Threading + +/// Item from a reader as supplied to the `IIngester` +type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } + +/// Core interface for projection system, representing the complete contract a feed consumer uses to deliver batches of work for projection +type IIngester = + /// Passes a (lazy) batch of items into the Ingestion Engine; the batch will be materialized out of band and submitted to the Projection engine for scheduling + /// Admission is Async in order that the Projector and Ingester can together contrive to force backpressure on the producer of the batches + /// Returns dynamic position in the queue at time of posting, together with current max number of items permissible + abstract member Submit: progressEpoch: int64 * markCompleted: Async * items: StreamItem seq -> Async + /// Requests immediate cancellation of + abstract member Stop: unit -> unit + +[] +module private Helpers = + let expiredMs ms = + let timer = Stopwatch.StartNew() + fun () -> + let due = timer.ElapsedMilliseconds > ms + if due then timer.Restart() + due + type Sem(max) = + let inner = new SemaphoreSlim(max) + member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore + member __.State = max-inner.CurrentCount,max + member __.Await() = inner.Await() |> Async.Ignore + member __.HasCapacity = inner.CurrentCount > 0 + +module Progress = + type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } + + type State<'Pos>() = + let pending = Queue<_>() + member __.AppendBatch(markCompleted, reqs : Dictionary) = + pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } + member __.MarkStreamProgress(stream, index) = + for x in pending do + match x.streamToRequiredIndex.TryGetValue stream with + | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore + | _, _ -> () + while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do + let item = pending.Dequeue() + item.markCompleted() + member __.InScheduledOrder getStreamQueueLength = + let raw = seq { + let streams = HashSet() + let mutable batch = 0 + for x in pending do + batch <- batch + 1 + for s in x.streamToRequiredIndex.Keys do + if streams.Add s then + yield s,(batch,getStreamQueueLength s) } + raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst + + /// Manages writing of progress + /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) + /// - retries until success or a new item is posted + type Writer<'Res when 'Res: equality>() = + let pumpSleepMs = 100 + let due = expiredMs 5000L + let mutable committedEpoch = None + let mutable validatedPos = None + let result = Event>() + [] member __.Result = result.Publish + member __.Post(version,f) = + Volatile.Write(&validatedPos,Some (version,f)) + member __.CommittedEpoch = Volatile.Read(&committedEpoch) + member __.Pump() = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + match Volatile.Read &validatedPos with + | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> + try do! f + Volatile.Write(&committedEpoch, Some v) + result.Trigger (Choice1Of2 v) + with e -> result.Trigger (Choice2Of2 e) + | _ -> do! Async.Sleep pumpSleepMs } + +module Scheduling = + + /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners + [] + type InternalMessage<'R> = + /// Enqueue a batch of items with supplied progress marking function + | Add of markCompleted: (unit -> unit) * items: StreamItem[] + /// Stats per submitted batch for stats listeners to aggregate + | Added of streams: int * skip: int * events: int + /// Submit new data pertaining to a stream that has commenced processing + | AddActive of KeyValuePair[] + /// Result of processing on stream - result (with basic stats) or the `exn` encountered + | Result of stream: string * outcome: Choice<'R,exn> + + /// Gathers stats pertaining to the core projection/ingestion activity + type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = + let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + let dumpStats (available,maxDop) = + log.Information("Projection Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + abstract member Handle : InternalMessage<'R> -> unit + default __.Handle msg = msg |> function + | Add _ | AddActive _ -> () + | Added (streams, skipped, events) -> + incr batchesPended + streamsPended := !streamsPended + streams + eventsPended := !eventsPended + events + eventsSkipped := !eventsSkipped + skipped + | Result (_stream, Choice1Of2 _) -> + incr resultCompleted + | Result (_stream, Choice2Of2 _) -> + incr resultExn + member __.TryDump((available,maxDop),streams : StreamStates) = + incr cycles + if statsDue () then + dumpStats (available,maxDop) + __.DumpExtraStats() + streams.Dump log + /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) + abstract DumpExtraStats : unit -> unit + default __.DumpExtraStats () = () + + /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint + type Dispatcher<'R>(maxDop) = + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let result = Event<'R>() + let dop = new SemaphoreSlim(maxDop) + let dispatch work = async { + let! res = work + result.Trigger res + dop.Release() |> ignore } + [] member __.Result = result.Publish + member __.AvailableCapacity = + let available = dop.CurrentCount + available,maxDop + member __.TryAdd(item,?timeout) = async { + let! got = dop.Await(?timeout=timeout) + if got then + work.Add(item) + return got } + member __.Pump () = async { + let! ct = Async.CancellationToken + for item in work.GetConsumingEnumerable ct do + Async.Start(dispatch item) } + + /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order + /// a) does not itself perform any reading activities + /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) + /// c) submits work to the supplied Dispatcher (which it triggers pumping of) + /// d) periodically reports state (with hooks for ingestion engines to report same) + type Engine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = + let sleepIntervalMs = 1 + let cts = new CancellationTokenSource() + let batches = new SemaphoreSlim(maxPendingBatches) + let work = ConcurrentQueue>() + let streams = StreamStates() + let progressState = Progress.State() + + member private __.Pump(stats : Stats<'R>) = async { + use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) + Async.Start(dispatcher.Pump(), cts.Token) + let validVsSkip (streamState : StreamState) (item : StreamItem) = + match streamState.write, item.index + 1L with + | Some cw, required when cw >= required -> 0, 1 + | _ -> 1, 0 + let handle x = + match x with + | Add (checkpoint, items) -> + let reqs = Dictionary() + let mutable count, skipCount = 0, 0 + for item in items do + let stream,streamState = streams.Add(item.stream,item.index,item.event) + match validVsSkip streamState item with + | 0, skip -> + skipCount <- skipCount + skip + | required, _ -> + count <- count + required + reqs.[stream] <- item.index+1L + progressState.AppendBatch(checkpoint,reqs) + work.Enqueue(Added (reqs.Count,skipCount,count)) + | AddActive events -> + for e in events do + streams.InternalMerge(e.Key,e.Value) + | Added _ -> + () + | Result (stream,r) -> + match interpretProgress streams stream r with + | Some index -> + progressState.MarkStreamProgress(stream,index) + streams.MarkCompleted(stream,index) + | None -> + streams.MarkFailed stream + + while not cts.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + let mutable idle = true + work |> ConcurrentQueue.drain (fun x -> + handle x + stats.Handle x + idle <- false) + // 2. top up provisioning of writers queue + let capacity,_ = dispatcher.AvailableCapacity + if capacity <> 0 then + idle <- false + let work = streams.Schedule(progressState.InScheduledOrder streams.QueueLength, capacity) + let xs = (Seq.ofArray work).GetEnumerator() + let mutable addsBeingAccepted = true + while xs.MoveNext() && addsBeingAccepted do + let (_,{stream = s} : StreamSpan) as item = xs.Current + let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) + addsBeingAccepted <- succeeded + // 3. Periodically emit status info + stats.TryDump(dispatcher.AvailableCapacity,streams) + // 4. Do a minimal sleep so we don't run completely hot when empty + if idle then do! Async.Sleep sleepIntervalMs } + static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = + let dispatcher = Dispatcher(processorDop) + let instance = new Engine<'R>(maxPendingBatches, dispatcher, project, interpretProgress) + Async.Start <| instance.Pump(stats) + instance + + /// Attempt to feed in a batch (subject to there being capacity to do so) + member __.TrySubmit(markCompleted, events) = async { + let! got = batches.Await(TimeSpan.Zero) + if got then + work.Enqueue <| Add (markCompleted, events) + return got } + + member __.AddOpenStreamData(events) = + work.Enqueue <| AddActive events + + member __.AllStreams = streams.All + + member __.Stop() = + cts.Cancel() + +type Projector = + static member Start(log, maxPendingBatches, maxActiveBatches, project : StreamSpan -> Async, ?statsInterval) = + let project (_maybeWritePos, batch) = async { + try let! count = project batch + return Choice1Of2 (batch.span.index + int64 count) + with e -> return Choice2Of2 e } + let interpretProgress _streams _stream = function + | Choice1Of2 index -> Some index + | Choice2Of2 _ -> None + let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + Scheduling.Engine.Start(stats, maxPendingBatches, maxActiveBatches, project, interpretProgress) + +module Ingestion = + + [] + type Message = + | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq + //| StreamSegment of span: StreamSpan + | EndOfSeries of seriesIndex: int + + type private Streams() = + let states = Dictionary() + let merge stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + + member __.Merge(items : StreamItem seq) = + for item in items do + merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = Array.singleton item.event } |] } + + member __.Take(processingContains) = + let forward = [| for x in states do if processingContains x.Key then yield x |] + for x in forward do states.Remove x.Key |> ignore + forward + + member __.Dump(log : ILogger) = + let mutable waiting, waitingB = 0, 0L + let waitingCats, waitingStreams = CatStats(), CatStats() + for KeyValue (stream,state) in states do + let sz = int64 state.Size + waitingCats.Ingest(category stream) + waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) + waiting <- waiting + 1 + waitingB <- waitingB + sz + log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) + if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) + if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) + + type private Stats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = + let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None + let progCommitFails, progCommits = ref 0, ref 0 + let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + let dumpStats (available,maxDop) = + log.Information("Holding Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", + !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 + if !progCommitFails <> 0 || !progCommits <> 0 then + match comittedEpoch with + | None -> + log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) + | Some committed when !progCommitFails <> 0 -> + log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) + | Some committed -> + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) + progCommits := 0; progCommitFails := 0 + else + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + member __.Handle : InternalMessage -> unit = function + | Batch _ | ActivateSeries _ | CloseSeries _-> () // stats are managed via Added internal message in same cycle + | ProgressResult (Choice1Of2 epoch) -> + incr progCommits + comittedEpoch <- Some epoch + | ProgressResult (Choice2Of2 (_exn : exn)) -> + incr progCommitFails + | Added (streams,events) -> + incr batchesPended + streamsPended := !streamsPended + streams + eventsPended := !eventsPended + events + member __.HandleValidated(epoch, pendingBatches) = + validatedEpoch <- epoch + pendingBatchCount <- pendingBatches + member __.HandleCommitted epoch = + comittedEpoch <- epoch + member __.TryDump((available,maxDop),streams : Streams) = + incr cycles + if statsDue () then + dumpStats (available,maxDop) + streams.Dump log + + and [] private InternalMessage = + | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq + /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` + | ProgressResult of Choice + /// Internal message for stats purposes + | Added of steams: int * events: int + | CloseSeries of seriesIndex: int + | ActivateSeries of seriesIndex: int + + let tryRemove key (dict: Dictionary<_,_>) = + match dict.TryGetValue key with + | true, value -> + dict.Remove key |> ignore + Some value + | false, _ -> None + + /// Holds batches away from Core processing to limit in-flight processing + type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxQueued, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = + let cts = new CancellationTokenSource() + let pumpDelayMs = defaultArg pumpDelayMs 5 + let work = ConcurrentQueue() + let readMax = new Sem(maxQueued) + let submissionsMax = new Sem(maxSubmissions) + let streams = Streams() + let stats = Stats(log, maxQueued, statsInterval) + let pending = Queue<_>() + let readingAhead, ready = Dictionary>(), Dictionary>() + let progressWriter = Progress.Writer<_>() + let mutable activeSeries = initialSeriesIndex + let mutable validatedPos = None + + member private __.Pump() = async { + let handle = function + | Batch (seriesId, epoch, checkpoint, items) -> + let batchInfo = + let items = Array.ofSeq items + streams.Merge items + let markCompleted () = + submissionsMax.Release() + readMax.Release() + validatedPos <- Some (epoch,checkpoint) + work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) + markCompleted, items + if activeSeries = seriesId then pending.Enqueue batchInfo + else + match readingAhead.TryGetValue seriesId with + | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) + | true,current -> current.Add(batchInfo) + | ActivateSeries newActiveSeries -> + activeSeries <- newActiveSeries + let buffered = + match ready |> tryRemove newActiveSeries with + | Some completedChunkBatches -> + completedChunkBatches |> Seq.iter pending.Enqueue + work.Enqueue <| ActivateSeries (newActiveSeries + 1) + completedChunkBatches.Count + | None -> + match readingAhead |> tryRemove newActiveSeries with + | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count + | None -> 0 + log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", + newActiveSeries, buffered, ready.Count, readingAhead.Count) + | CloseSeries seriesIndex -> + if activeSeries = seriesIndex then + log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) + work.Enqueue <| ActivateSeries (activeSeries + 1) + else + match readingAhead |> tryRemove seriesIndex with + | Some batchesRead -> + ready.[seriesIndex] <- batchesRead + log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) + | None -> + ready.[seriesIndex] <- ResizeArray() + log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) + // These events are for stats purposes + | Added _ | ProgressResult _ -> () + use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) + Async.Start(progressWriter.Pump(), cts.Token) + while not cts.IsCancellationRequested do + work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) + let mutable ingesterAccepting = true + // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted + while pending.Count <> 0 && submissionsMax.HasCapacity && ingesterAccepting do + let markCompleted, events = pending.Peek() + let! submitted = scheduler.TrySubmit(markCompleted, events) + if submitted then + pending.Dequeue() |> ignore + // mark off a write as being in progress + do! submissionsMax.Await() + else + ingesterAccepting <- false + // 2. Update any progress into the stats + stats.HandleValidated(Option.map fst validatedPos, fst readMax.State) + validatedPos |> Option.iter progressWriter.Post + stats.HandleCommitted progressWriter.CommittedEpoch + // 3. Forward content for any active streams into processor immediately + let relevantBufferedStreams = streams.Take(scheduler.AllStreams.Contains) + scheduler.AddOpenStreamData(relevantBufferedStreams) + // 4. Periodically emit status info + stats.TryDump(submissionsMax.State,streams) + do! Async.Sleep pumpDelayMs } + + /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes + static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval) = + let instance = new Engine<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval = statsInterval) + Async.Start <| instance.Pump() + instance + + /// Awaits space in `read` to limit reading ahead - yields (used,maximum) counts from Read Semaphore for logging purposes + member __.Submit(content : Message) = async { + do! readMax.Await() + match content with + | Message.Batch (seriesId, epoch, markBatchCompleted, events) -> work.Enqueue <| Batch (seriesId, epoch, markBatchCompleted, events) + | Message.EndOfSeries seriesId -> work.Enqueue <| CloseSeries seriesId + return readMax.State } + + /// As range assignments get revoked, a user is expected to `Stop `the active processing thread for the Ingester before releasing references to it + member __.Stop() = cts.Cancel() + +type Ingester = + /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing + static member Start<'R>(log, scheduler, maxRead, maxSubmissions, ?statsInterval) = + let singleSeriesIndex = 0 + let instance = Ingestion.Engine<'R>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + { new IIngester with + member __.Submit(epoch, markCompleted, items) : Async = + instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) + member __.Stop() = __.Stop() } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index e9746ad0e..fd9abf3a1 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -13,12 +13,6 @@ type [] StreamState = { isMalformed: bool; write: int64 option; qu member __.Size = if __.queue = null then 0 else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) - member __.TryGap() = - if __.queue = null then None - else - match __.write, Array.tryHead __.queue with - | Some w, Some { index = i } when i > w -> Some (w, i-w) - | _ -> None member __.IsReady = if __.queue = null || __.isMalformed then false else @@ -105,14 +99,16 @@ type StreamStates() = let x = xs.Current let state = states.[x] if not state.isMalformed && busy.Add x then - let q = state.queue - if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", x) - toSchedule.Add(state.write, { stream = x; span = q.[0] }) + toSchedule.Add(state.write, { stream = x; span = state.queue.[0] }) remaining <- remaining - 1 toSchedule.ToArray() let markNotBusy stream = busy.Remove stream |> ignore + // Result is intentionally a thread-safe persisent data structure + // This enables the (potentially multiple) Ingesters to determine streams (for which they potentially have successor events) that are in play + // Ingesters then supply these 'preview events' in advance of the processing being scheduled + // This enables the projection logic to roll future work into the current work in the interests of medium term throughput member __.All = streams member __.InternalMerge(stream, state) = update stream state |> ignore member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } @@ -122,15 +118,8 @@ type StreamStates() = updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] member __.SetMalformed(stream,isMalformed) = updateWritePos stream isMalformed None [| { index = 0L; events = null } |] - // DEPRECATED - will be removed - member __.TryGetStreamWritePos stream = - match states.TryGetValue stream with - | true, value -> value.write - | false, _ -> None member __.QueueLength(stream) = - let q = states.[stream].queue - if q = null then Log.Warning("Attempt to request scheduling for completed {stream} that has no items queued", stream) - q.[0].events.Length + states.[stream].queue.[0].events.Length member __.MarkCompleted(stream, index) = markNotBusy stream markCompleted stream index @@ -163,60 +152,4 @@ type StreamStates() = if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) - - // Used to trigger catch-up reading of streams which have events missing prior to the observed write position - //member __.TryGap() : (string*int64*int) option = - // let rec aux () = - // match gap |> Queue.tryDequeue with - // | None -> None - // | Some stream -> - - // match states.[stream].TryGap() with - // | Some (pos,count) -> Some (stream,pos,int count) - // | None -> aux () - // aux () - -type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } - -type ProgressState<'Pos>() = - let pending = Queue<_>() - member __.AppendBatch(markCompleted, reqs : Dictionary) = - pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } - member __.MarkStreamProgress(stream, index) = - for x in pending do - match x.streamToRequiredIndex.TryGetValue stream with - | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore - | _, _ -> () - let headIsComplete () = pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 - let mutable completed = 0 - while headIsComplete () do - let item = pending.Dequeue() in item.markCompleted() - completed <- completed + 1 - completed - member __.ScheduledOrder getStreamQueueLength = - let raw = seq { - let streams = HashSet() - let mutable batch = 0 - for x in pending do - batch <- batch + 1 - for s in x.streamToRequiredIndex.Keys do - if streams.Add s then - yield s,(batch,getStreamQueueLength s) } - raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst - member __.Validate tryGetStreamWritePos = - let rec aux completed = - if pending.Count = 0 then completed else - let batch = pending.Peek() - for KeyValue (stream, requiredIndex) in Array.ofSeq batch.streamToRequiredIndex do - match tryGetStreamWritePos stream with - | Some index when requiredIndex <= index -> - Log.Warning("Validation had to remove {stream} as required {req} has been met by {index}", stream, requiredIndex, index) - batch.streamToRequiredIndex.Remove stream |> ignore - | _ -> () - if batch.streamToRequiredIndex.Count <> 0 then - completed - else - let item = pending.Dequeue() in item.markCompleted() - aux (completed + 1) - aux 0 \ No newline at end of file + if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index c6f3db823..aa1d78820 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -11,7 +11,7 @@ open Equinox.Projection.Codec open Equinox.Store open Jet.ConfluentKafka.FSharp //#else -open Equinox.Projection.Engine +open Equinox.Projection //#endif open Microsoft.Azure.Documents.ChangeFeedProcessor open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing @@ -176,15 +176,15 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = ChangeFeedObserver.Create(log, projectBatch, dispose = disposeProducer) //#else let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) = - let projectionEngine = startProjectionEngine (log, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) + let projectionEngine = Projector.Start (log, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) fun () -> - let mutable trancheEngine = Unchecked.defaultof> - let init rangeLog = trancheEngine <- TrancheEngine.Start (rangeLog, projectionEngine, maxPendingBatches, processorDop, TimeSpan.FromMinutes 1.) + let mutable rangeIngester = Unchecked.defaultof + let init rangeLog = rangeIngester <- Ingester.Start (rangeLog, projectionEngine, maxPendingBatches, processorDop, TimeSpan.FromMinutes 1.) let ingest epoch checkpoint docs = let events = Seq.collect DocumentParser.enumEvents docs let items = seq { for x in events -> { stream = x.Stream; index = x.Index; event = x } } - trancheEngine.Submit(epoch, checkpoint, items) - let dispose () = trancheEngine.Stop() + rangeIngester.Submit(epoch, checkpoint, items) + let dispose () = rangeIngester.Stop() let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 74c1d7857..ac9efd4df 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -2,7 +2,7 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Projection -open Equinox.Projection.Engine +open Equinox.Projection open Equinox.Projection.State open Equinox.Store // AwaitTaskCorrect open Microsoft.Azure.Documents @@ -12,27 +12,25 @@ open System open System.Collections.Generic let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: CosmosContext, maxWriters) (transform : Document -> StreamItem seq) = - let ingestionEngine = Equinox.Cosmos.Projection.Ingestion.startIngestionEngine (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) - let mutable trancheEngine = Unchecked.defaultof<_> - let init rangeLog = - trancheEngine <- Equinox.Projection.Engine.TrancheEngine.Start (rangeLog, ingestionEngine, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) - let ingest epoch checkpoint docs = - let events = docs |> Seq.collect transform |> Array.ofSeq - trancheEngine.Submit(epoch, checkpoint, events) - let dispose () = trancheEngine.Stop () - let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok - let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved - let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } - let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", - epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., - let e = pt.Elapsed in e.TotalSeconds, cur, max) - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - } - ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) + let cosmosIngester = CosmosIngester.start (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + fun () -> + let mutable rangeIngester = Unchecked.defaultof<_> + let init rangeLog = rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) + let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq in rangeIngester.Submit(epoch, checkpoint, events) + let dispose () = rangeIngester.Stop () + let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok + let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 + // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved + let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } + let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time + log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", + epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., + let e = pt.Elapsed in e.TotalSeconds, cur, max) + sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + } + ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) createRangeProjector = async { @@ -75,30 +73,28 @@ module EventV0Parser = let tmp = new Document() tmp.SetPropertyValue("content", document) tmp.GetPropertyValue<'T>("content") - type IEvent = - inherit Equinox.Codec.Core.IIndexedEvent - abstract member Stream : string + + /// Maps fields in an Equinox V0 Event within an Eqinox.Cosmos Event (in a Batch or Tip) to the interface defined by the default Codec + let (|StandardCodecEvent|) (x: EventV0) = + { new Equinox.Codec.IEvent<_> with + member __.EventType = x.t + member __.Data = x.d + member __.Meta = null + member __.Timestamp = x.c } + /// We assume all Documents represent Events laid out as above - let parse (d : Document) = - let x = d.Cast() - { new IEvent with - member __.Index = x.i - member __.IsUnfold = false - member __.EventType = x.t - member __.Data = x.d - member __.Meta = null - member __.Timestamp = x.c - member __.Stream = x.s } + let parse (d : Document) : StreamItem = + let (StandardCodecEvent e) as x = d.Cast() + { stream = x.s; index = x.i; event = e } : Equinox.Projection.StreamItem let transformV0 catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { let parsed = EventV0Parser.parse v0SchemaDocument - let streamName = (*if parsed.Stream.Contains '-' then parsed.Stream else "Prefixed-"+*)parsed.Stream - if catFilter (category streamName) then - yield { stream = streamName; index = parsed.Index; event = parsed } } + let streamName = (*if parsed.Stream.Contains '-' then parsed.Stream else "Prefixed-"+*)parsed.stream + if catFilter (category streamName) then yield parsed } //#else let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq = seq { for e in DocumentParser.enumEvents changeFeedDocument do - if catFilter (category e.Stream) then + if catFilter (category e.Stream) then // TODO yield parsed // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level yield { stream = e.Stream; index = e.Index; event = e } } //#endif \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 38b8ee7bb..fbba2d6fa 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -1,15 +1,15 @@ module SyncTemplate.EventStoreSource -open Equinox.Cosmos.Projection.Ingestion +//open Equinox.Cosmos.Projection open Equinox.Store // AwaitTaskCorrect open Equinox.Projection -open Equinox.Projection.Engine open EventStore.ClientAPI open Serilog // NB Needs to shadow ILogger open System open System.Collections.Generic open System.Diagnostics open System.Threading +open Equinox.Cosmos.Projection type EventStore.ClientAPI.RecordedEvent with member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) @@ -18,7 +18,7 @@ let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = State.array let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 let private mb x = float x / 1024. / 1024. -let toIngestionItem (e : RecordedEvent) : Engine.StreamItem = +let toIngestionItem (e : RecordedEvent) : StreamItem = let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ @@ -146,9 +146,9 @@ let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int /// Walks the $all stream, yielding batches together with the associated Position info for the purposes of checkpointing /// Can throw (in which case the caller is in charge of retrying, possibly with a smaller batch size) -type [] PullResult = Exn of exn: exn | Eof | EndOfTranche +type [] PullResult = Exn of exn: exn | Eof of Position | EndOfTranche let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) - (range:Range, once) (tryMapEvent : ResolvedEvent -> Engine.StreamItem option) (postBatch : Position -> Engine.StreamItem[] -> Async) = + (range:Range, once) (tryMapEvent : ResolvedEvent -> StreamItem option) (postBatch : Position -> StreamItem[] -> Async) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let rec aux () = async { let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect @@ -163,107 +163,93 @@ let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, batchEvents, usedCats, usedStreams, batches.Length, (let e = postSw.Elapsed in e.TotalSeconds), cur, max) if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then - return currentSlice.IsEndOfStream + if currentSlice.IsEndOfStream then return Eof currentSlice.NextPosition + else return EndOfTranche else sw.Restart() // restart the clock as we hand off back to the Reader return! aux () } async { - try let! eof = aux () - return if eof then Eof else EndOfTranche + try return! aux () with e -> return Exn e } /// Specification for work to be performed by a reader thread [] -type ReadRequest = +type Req = /// Tail from a given start position, at intervals of the specified timespan (no waiting if catching up) - | Tail of pos: Position * max : Position * interval: TimeSpan * batchSize : int + | Tail of seriesId: int * startPos: Position * max : Position * interval: TimeSpan * batchSize : int /// Read a given segment of a stream (used when a stream needs to be rolled forward to lay down an event for which the preceding events are missing) - | StreamPrefix of name: string * pos: int64 * len: int * batchSize: int + //| StreamPrefix of name: string * pos: int64 * len: int * batchSize: int /// Read the entirity of a stream in blocks of the specified batchSize (TODO wire to commandline request) - | Stream of name: string * batchSize: int + //| Stream of name: string * batchSize: int /// Read a specific chunk (min-max range), posting batches tagged with that chunk number - | Chunk of chunk: int * range: Range * batchSize : int + | Chunk of seriesId: int * range: Range * batchSize : int /// Data with context resulting from a reader thread [] -type ReadResult = +type Res = /// A batch read from a Chunk - | ChunkBatch of chunk: int * pos: Position * items: StreamItem seq + | Batch of seriesId: int * pos: Position * items: StreamItem seq /// Ingestion buffer requires an explicit end of chunk message before next chunk can commence processing - | EndOfChunk of chunk: int + | EndOfChunk of seriesId: int /// A Batch read from a Stream or StreamPrefix - | StreamSpan of span: State.StreamSpan - /// A batch read by `Tail` - | Batch of pos: Position * items: StreamItem seq + //| StreamSpan of span: State.StreamSpan /// Holds work queue, together with stats relating to the amount and/or categories of data being traversed /// Processing is driven by external callers running multiple concurrent invocations of `Process` -type ReadQueue(batchSize, minBatchSize, ?statsInterval) = +type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Res -> Async, tailInterval, dop, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() - member val OverallStats = OverallStats(?statsInterval=statsInterval) - member val SlicesStats = SliceStatsBuffer() - member __.DefaultBatchSize = batchSize - member __.QueueCount = work.Count - member __.AddStream(name, ?batchSizeOverride) = - work.Enqueue <| ReadRequest.Stream (name, defaultArg batchSizeOverride batchSize) - member __.AddStreamPrefix(name, pos, len, ?batchSizeOverride) = - work.Enqueue <| ReadRequest.StreamPrefix (name, pos, len, defaultArg batchSizeOverride batchSize) - member private __.AddTranche(chunk, range, ?batchSizeOverride) = - work.Enqueue <| ReadRequest.Chunk (chunk, range, defaultArg batchSizeOverride batchSize) - member __.AddTranche(chunk, pos, nextPos, max, ?batchSizeOverride) = - __.AddTranche(chunk, Range (pos, Some nextPos, max), ?batchSizeOverride=batchSizeOverride) - member __.AddTail(pos, max, interval, ?batchSizeOverride) = - work.Enqueue <| ReadRequest.Tail (pos, max, interval, defaultArg batchSizeOverride batchSize) - member __.TryDequeue () = - work.TryDequeue() - /// Invoked by Dispatcher to process a tranche of work; can have parallel invocations - member __.Process(conn, tryMapEvent, post : ReadResult -> Async, work) = async { + let sleepIntervalMs = 100 + let overallStats = OverallStats(?statsInterval=statsInterval) + let slicesStats = SliceStatsBuffer() + let mutable eofSpottedInChunk = false + + /// Invoked by pump to process a tranche of work; can have parallel invocations + let exec conn req = async { let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize - let postSpan = ReadResult.StreamSpan >> post >> Async.Ignore - match work with - | StreamPrefix (name,pos,len,batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) - try let! t,() = pullStream (conn, batchSize) (name, pos, Some len) postSpan |> Stopwatch.Time - Log.Information("completed stream prefix in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) - with e -> - let bs = adjust batchSize - Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - __.AddStreamPrefix(name, pos, len, bs) - return false - | Stream (name,batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - Log.Warning("Reading stream; batch size {bs}", batchSize) - try let! t,() = pullStream (conn, batchSize) (name,0L,None) postSpan |> Stopwatch.Time - Log.Information("completed stream in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) - with e -> - let bs = adjust batchSize - Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - __.AddStream(name, bs) - return false - | Chunk (chunk, range, batchSize) -> - use _ = Serilog.Context.LogContext.PushProperty("Tranche", chunk) + //let postSpan = ReadResult.StreamSpan >> post >> Async.Ignore + match req with + //| StreamPrefix (name,pos,len,batchSize) -> + // use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) + // Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) + // try let! t,() = pullStream (conn, batchSize) (name, pos, Some len) postSpan |> Stopwatch.Time + // Log.Information("completed stream prefix in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) + // with e -> + // let bs = adjust batchSize + // Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + // __.AddStreamPrefix(name, pos, len, bs) + // return false + //| Stream (name,batchSize) -> + // use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) + // Log.Warning("Reading stream; batch size {bs}", batchSize) + // try let! t,() = pullStream (conn, batchSize) (name,0L,None) postSpan |> Stopwatch.Time + // Log.Information("completed stream in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) + // with e -> + // let bs = adjust batchSize + // Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) + // __.AddStream(name, bs) + // return false + | Chunk (series, range, batchSize) -> + let postBatch pos items = post (Res.Batch (series, pos, items)) + use _ = Serilog.Context.LogContext.PushProperty("Tranche", series) Log.Warning("Commencing tranche, batch size {bs}", batchSize) - let postBatch pos items = post (ReadResult.ChunkBatch (chunk, pos, items)) - let! t, res = pullAll (__.SlicesStats, __.OverallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time + let! t, res = pullAll (slicesStats, overallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with - | PullResult.EndOfTranche -> - Log.Warning("completed tranche in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) - __.OverallStats.DumpIfIntervalExpired() - let! _ = post (ReadResult.EndOfChunk chunk) - return false - | PullResult.Eof -> + | PullResult.Eof pos -> Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) - __.OverallStats.DumpIfIntervalExpired(true) - let! _ = post (ReadResult.EndOfChunk chunk) - return true + overallStats.DumpIfIntervalExpired(true) + let! _ = post (Res.EndOfChunk series) in () + work.Enqueue <| Req.Tail (series+1, pos, pos, tailInterval, defaultBatchSize) + eofSpottedInChunk <- true + | PullResult.EndOfTranche -> + Log.Information("completed tranche in {ms:n1}m", let e = t.Elapsed in e.TotalMinutes) + let! _ = post (Res.EndOfChunk series) in () | PullResult.Exn e -> - let bs = adjust batchSize - Log.Warning(e, "Could not read All, retrying with batch size {bs}", bs) - __.OverallStats.DumpIfIntervalExpired() - __.AddTranche(chunk, range, bs) - return false - | Tail (pos, max, interval, batchSize) -> + let abs = adjust batchSize + Log.Warning(e, "Could not read All, retrying with batch size {bs}", abs) + work.Enqueue <| Req.Chunk (series, range, abs) + | Tail (series, pos, max, interval, batchSize) -> + let postBatch pos items = post (Res.Batch (series, pos, items)) + use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let mutable count, batchSize, range = 0, batchSize, Range(pos, None, max) let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds @@ -274,9 +260,7 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = | _ -> () tailSw.Restart() } let slicesStats, stats = SliceStatsBuffer(), OverallStats() - use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") let progressSw = Stopwatch.StartNew() - let postBatch pos items = post (ReadResult.Batch (pos, items)) while true do let currentPos = range.Current if progressSw.ElapsedMilliseconds > progressIntervalMs then @@ -287,87 +271,60 @@ type ReadQueue(batchSize, minBatchSize, ?statsInterval) = let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch do! awaitInterval match res with - | PullResult.EndOfTranche | PullResult.Eof -> () + | PullResult.EndOfTranche | PullResult.Eof _ -> () | PullResult.Exn e -> batchSize <- adjust batchSize - Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) - stats.DumpIfIntervalExpired() - return true } - -/// Handles bulk ingestion - a specified number of concurrent reader threads round-robin over a supplied set of connections, taking the next available 256MB -/// chunk from the tail upon completion of the specified `Range` delimiting the chunk -type StripedReader(conns : _ array, work: ReadQueue, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxDop) = - let dop = new SemaphoreSlim(maxDop) + Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) } - /// Single invocation; spawns child threads within defined limits; exits when End of Store has been reached - member __.Pump(post, startPos, max) = async { - let! ct = Async.CancellationToken - let mutable remainder = - let nextPos = posFromChunkAfter startPos - let startChunk = chunk startPos |> int - work.AddTranche(startChunk, startPos, nextPos, max) - Some nextPos - let mutable finished = false - let r = new Random() + let pump (initialSeriesId, initialPos) max = async { let mutable robin = 0 - while not ct.IsCancellationRequested && not (finished && dop.CurrentCount <> maxDop) do - work.OverallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - let forkRunRelease task = async { + let selectConn () = + let connIndex = Interlocked.Increment(&robin) % conns.Length + conns.[connIndex] + let dop = new SemaphoreSlim(dop) + let forkRunRelease = + let r = new Random() + fun req -> async { // this is not called in parallel hence no need to lock `r` + let currentCount = dop.CurrentCount + // Jitter is most relevant when processing commences - any commencement of a chunk can trigger significant page faults on server + // which we want to attempt to limit the effects of + let jitterMs = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) + Log.Warning("Waiting {jitter}ms to jitter reader stripes, {currentCount} further reader stripes awaiting start", jitterMs, currentCount) + do! Async.Sleep jitterMs let! _ = Async.StartChild <| async { - try let connIndex = Interlocked.Increment(&robin) % conns.Length - let conn = conns.[connIndex] - let! eof = work.Process(conn, tryMapEvent, post, task) in () - if eof then remainder <- None - finally dop.Release() |> ignore } - return () } - let currentCount = dop.CurrentCount - // Jitter is most relevant when processing commences - any commencement of a chunk can trigger significant page faults on server - // which we want to attempt to limit the effects of - let jitter = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) - Log.Warning("Waiting {jitter}ms to jitter reader stripes, {currentCount} further reader stripes awaiting start", jitter, currentCount) - do! Async.Sleep jitter - match work.TryDequeue() with - | true, task -> - do! forkRunRelease task - | false, _ -> - match remainder with - | Some pos -> - let nextPos = posFromChunkAfter pos - remainder <- Some nextPos - let chunkNumber = chunk pos |> int - do! forkRunRelease <| ReadRequest.Chunk (chunkNumber, Range(pos, Some nextPos, max), work.DefaultBatchSize) - | None -> - Log.Warning("No further ingestion work to commence, transitioning to tailing...") - finished <- true } - -/// Handles Tailing mode - a single reader thread together with a (limited) set of concurrent of stream-catchup readers -type TailAndPrefixesReader(conn, work: ReadQueue, tryMapEvent: EventStore.ClientAPI.ResolvedEvent -> StreamItem option, maxCatchupReaders) = - // to avoid busy waiting in main message pummp loop - let sleepIntervalMs = 100 - let dop = new SemaphoreSlim(1 + maxCatchupReaders) - member __.HasCapacity = work.QueueCount < dop.CurrentCount - // TODO reinstate usage - member __.AddStreamPrefix(stream, pos, len) = work.AddStreamPrefix(stream, pos, len) - - /// Single invcation will run until Cancelled, spawning child threads as necessary - member __.Pump(post, startPos, max, tailInterval) = async { + try let conn = selectConn () + do! exec conn req + finally dop.Release() |> ignore } in () } + let mutable seriesId = initialSeriesId + let mutable remainder = + if conns.Length > 1 then + let nextPos = posFromChunkAfter initialPos + work.Enqueue <| Req.Chunk (initialSeriesId, new Range(initialPos, Some nextPos, max), defaultBatchSize) + Some nextPos + else + work.Enqueue <| Req.Tail (initialSeriesId, initialPos, max, tailInterval, defaultBatchSize) + None let! ct = Async.CancellationToken - work.AddTail(startPos, max, tailInterval) while not ct.IsCancellationRequested do - work.OverallStats.DumpIfIntervalExpired() + overallStats.DumpIfIntervalExpired() let! _ = dop.Await() - let forkRunRelease task = async { - let! _ = Async.StartChild <| async { - try let! _ = work.Process(conn, tryMapEvent, post, task) in () - finally dop.Release() |> ignore } - return () } - match work.TryDequeue() with - | true, task -> + match work.TryDequeue(), remainder with + | (true, task), _ -> do! forkRunRelease task - | false, _ -> + | (false, _), Some nextChunk when not eofSpottedInChunk -> + seriesId <- seriesId + 1 + let nextPos = posFromChunkAfter nextChunk + remainder <- Some nextPos + do! forkRunRelease <| Req.Chunk (seriesId, Range(nextChunk, Some nextPos, max), defaultBatchSize) + | (false, _), x -> + dop.Release() |> ignore + Log.Warning("No further ingestion work to commence, transitioning to tailing...") + remainder <- None + | (false, _), None -> dop.Release() |> ignore do! Async.Sleep sleepIntervalMs } + member __.Start initialPos max = async { + let _ = Async.StartChild (pump initialPos max) in () } type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint @@ -381,8 +338,12 @@ type ReaderSpec = checkpointInterval: TimeSpan /// Delay when reading yields an empty batch tailInterval: TimeSpan + /// Enable initial phase where interleaved reading stripes a 256MB chunk apart attain a balance between good reading speed and not killing the server gorge: bool /// Maximum number of striped readers to permit + /// - for gorging, this dictates how many concurrent readers there will be + /// - when tailing, this dictates how many stream readers will be used to perform catchup work on streams that are missing a prefix + /// (e.g. due to not starting from the start of the $all stream, and/or deleting data from the destination store) stripes: int /// Initial batch size to use when commencing reading batchSize: int @@ -422,37 +383,22 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let ingestionEngine = startIngestionEngine (log.ForContext("Tranche","Cosmos"), maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) - let trancheEngine = TrancheEngine.Start (log.ForContext("Tranche","Tranches"), ingestionEngine, maxReadAhead, maxProcessing, TimeSpan.FromMinutes 1.) - let readerQueue = ReadQueue(spec.batchSize, spec.minBatchSize) - if spec.gorge then - let extraConns = Seq.init (spec.stripes-1) (ignore >> connect) - let conns = [| yield conn; yield! extraConns |] - let post = function - | ReadResult.ChunkBatch (chunk, pos, xs) -> - let cp = pos.CommitPosition - trancheEngine.Submit <| Push.ChunkBatch(chunk, cp, checkpoints.Commit cp, xs) - | ReadResult.EndOfChunk chunk -> - trancheEngine.Submit <| Push.EndOfChunk chunk - | ReadResult.Batch _ - | ReadResult.StreamSpan _ as x -> - failwithf "%A not supported when gorging" x - let startChunk = chunk startPos |> int - let! _ = trancheEngine.Submit (Push.SetActiveChunk startChunk) - log.Information("Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) - let gorgingReader = StripedReader(conns, readerQueue, tryMapEvent, spec.stripes) - do! gorgingReader.Pump(post, startPos, max) - for x in extraConns do x.Close() - // After doing the gorging, we switch to normal tailing (which avoids the app exiting too) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","EqxCosmos"), maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let initialSeriesId, conns, dop = + log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) + if spec.gorge then + log.Information("Commencing Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) + let extraConns = Seq.init (spec.stripes-1) (ignore >> connect) + let conns = [| yield conn; yield! extraConns |] + chunk startPos |> int, conns, conns.Length + else + 0, [|conn|], spec.stripes+1 + let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","EventStore"), cosmosIngestionEngine, maxReadAhead, maxProcessing, initialSeriesId, TimeSpan.FromMinutes 1.) let post = function - | ReadResult.Batch (pos, xs) -> + | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId + | Res.Batch (seriesId, pos, xs) -> let cp = pos.CommitPosition - trancheEngine.Submit(cp, checkpoints.Commit cp, xs) - | ReadResult.StreamSpan span -> - trancheEngine.Submit <| Push.Stream span - | ReadResult.ChunkBatch _ - | ReadResult.EndOfChunk _ as x -> - failwithf "%A not supported when tailing" x - let tailAndCatchupReaders = TailAndPrefixesReader(conn, readerQueue, tryMapEvent, spec.stripes) - log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) - do! tailAndCatchupReaders.Pump(post, startPos, max, spec.tailInterval) } \ No newline at end of file + trancheEngine.Submit <| Ingestion.Message.Batch(seriesId, cp, checkpoints.Commit cp, xs) + let reader = Reader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, post, spec.tailInterval, dop) + do! reader.Start (initialSeriesId,startPos) max + do! Async.AwaitKeyboardInterrupt() } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 133ce1614..308b5836e 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -6,8 +6,6 @@ open Equinox.Cosmos.Projection #else open Equinox.EventStore #endif -open Equinox.Cosmos.Projection.Ingestion -open Equinox.Projection.State open Serilog open System @@ -284,10 +282,10 @@ module Logging = let cfpLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpLevel) |> fun c -> let ingesterLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information - c.MinimumLevel.Override(typeof.FullName, ingesterLevel) + c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning - c.MinimumLevel.Override(typeof.FullName, generalLevel) + c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, LogEventLevel.Information) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let configure (a : Configuration.LoggerSinkConfiguration) : unit = @@ -295,7 +293,7 @@ module Logging = l.WriteTo.Sink(Metrics.RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke - let isWriter = Filters.Matching.FromSource().Invoke + let isWriter = Filters.Matching.FromSource().Invoke let isCheckpointing = Filters.Matching.FromSource().Invoke (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x || isWriter x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) @@ -303,7 +301,7 @@ module Logging = c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() - Log.ForContext(), Log.ForContext() + Log.ForContext(), Log.ForContext() [] let main argv = @@ -355,7 +353,7 @@ let main argv = || e.EventStreamId.StartsWith "InventoryCount-" // No Longer used || e.EventStreamId.StartsWith "InventoryLog" // 5GB, causes lopsided partitions, unused //|| e.EventStreamId = "ReloadBatchId" // does not start at 0 - //|| e.EventStreamId = "PurchaseOrder-5791" // item too large + || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream From 185cafc8d7b2fb81fe68d85436d477fe0ad5334e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 May 2019 20:29:53 +0100 Subject: [PATCH 199/353] Tune batch size defaults --- equinox-sync/Sync/Program.fs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 308b5836e..b45b49312 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -79,9 +79,9 @@ module CmdParser = and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.Verbose = a.Contains Verbose - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,5000) - member __.MaxProcessing = a.GetResult(MaxProcessing,128) - member __.MaxWriters = a.GetResult(MaxWriters,512) + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,1000) + member __.MaxProcessing = a.GetResult(MaxProcessing,32) + member __.MaxWriters = a.GetResult(MaxWriters,1024) #if cosmos member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName From 184175c68b7d5e641a5300c416eb87a860bd25cb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 May 2019 20:37:11 +0100 Subject: [PATCH 200/353] Important ! --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index fbba2d6fa..e3121f4c8 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -324,7 +324,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re dop.Release() |> ignore do! Async.Sleep sleepIntervalMs } member __.Start initialPos max = async { - let _ = Async.StartChild (pump initialPos max) in () } + let! _ = Async.StartChild (pump initialPos max) in () } type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint From aca53c935e6eb6466d7cbdf8a25002d124a62e07 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 May 2019 21:04:38 +0100 Subject: [PATCH 201/353] Add missing Release --- equinox-projector/Equinox.Projection/Projection.fs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index e0692ea73..7b175099d 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -453,8 +453,12 @@ module Ingestion = member __.Submit(content : Message) = async { do! readMax.Await() match content with - | Message.Batch (seriesId, epoch, markBatchCompleted, events) -> work.Enqueue <| Batch (seriesId, epoch, markBatchCompleted, events) - | Message.EndOfSeries seriesId -> work.Enqueue <| CloseSeries seriesId + | Message.Batch (seriesId, epoch, markBatchCompleted, events) -> + work.Enqueue <| Batch (seriesId, epoch, markBatchCompleted, events) + // NB readMax.Release() is effected in the Batch handler's MarkCompleted() + | Message.EndOfSeries seriesId -> + work.Enqueue <| CloseSeries seriesId + readMax.Release() return readMax.State } /// As range assignments get revoked, a user is expected to `Stop `the active processing thread for the Ingester before releasing references to it From 1312802b11e77f0b89372612bd27fec241c1bb15 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 May 2019 22:31:17 +0100 Subject: [PATCH 202/353] Fix batch Releasing --- .../Equinox.Projection/Projection.fs | 55 +++++++++++-------- 1 file changed, 31 insertions(+), 24 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index 7b175099d..7713588d4 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -32,8 +32,12 @@ module private Helpers = let inner = new SemaphoreSlim(max) member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore member __.State = max-inner.CurrentCount,max + /// Wait infinitely to get the semaphore member __.Await() = inner.Await() |> Async.Ignore + /// Wait for the specified timeout to acquire (or return false instantly) + member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) member __.HasCapacity = inner.CurrentCount > 0 + member __.CurrentCapacity = inner.CurrentCount module Progress = type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } @@ -48,8 +52,8 @@ module Progress = | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore | _, _ -> () while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do - let item = pending.Dequeue() - item.markCompleted() + let batch = pending.Dequeue() + batch.markCompleted() member __.InScheduledOrder getStreamQueueLength = let raw = seq { let streams = HashSet() @@ -103,9 +107,9 @@ module Scheduling = type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (available,maxDop) = - log.Information("Projection Cycles {cycles} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, maxDop-available, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + let dumpStats capacity (used,maxDop) = + log.Information("Projection Cycles {cycles} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function @@ -119,10 +123,10 @@ module Scheduling = incr resultCompleted | Result (_stream, Choice2Of2 _) -> incr resultExn - member __.TryDump((available,maxDop),streams : StreamStates) = + member __.TryDump(capacity,(used,max),streams : StreamStates) = incr cycles if statsDue () then - dumpStats (available,maxDop) + dumpStats capacity (used,max) __.DumpExtraStats() streams.Dump log /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) @@ -133,17 +137,17 @@ module Scheduling = type Dispatcher<'R>(maxDop) = let work = new BlockingCollection<_>(ConcurrentQueue<_>()) let result = Event<'R>() - let dop = new SemaphoreSlim(maxDop) + let dop = new Sem(maxDop) let dispatch work = async { let! res = work result.Trigger res - dop.Release() |> ignore } + dop.Release() } [] member __.Result = result.Publish - member __.AvailableCapacity = - let available = dop.CurrentCount - available,maxDop + member __.HasCapacity = dop.HasCapacity + member __.CurrentCapacity = dop.CurrentCapacity + member __.State = dop.State member __.TryAdd(item,?timeout) = async { - let! got = dop.Await(?timeout=timeout) + let! got = dop.TryAwait(?timeout=timeout) if got then work.Add(item) return got } @@ -160,7 +164,7 @@ module Scheduling = type Engine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() - let batches = new SemaphoreSlim(maxPendingBatches) + let batches = Sem maxPendingBatches let work = ConcurrentQueue>() let streams = StreamStates() let progressState = Progress.State() @@ -174,7 +178,7 @@ module Scheduling = | _ -> 1, 0 let handle x = match x with - | Add (checkpoint, items) -> + | Add (releaseRead, items) -> let reqs = Dictionary() let mutable count, skipCount = 0, 0 for item in items do @@ -185,7 +189,10 @@ module Scheduling = | required, _ -> count <- count + required reqs.[stream] <- item.index+1L - progressState.AppendBatch(checkpoint,reqs) + let markCompleted () = + releaseRead() + batches.Release() + progressState.AppendBatch(markCompleted,reqs) work.Enqueue(Added (reqs.Count,skipCount,count)) | AddActive events -> for e in events do @@ -208,18 +215,18 @@ module Scheduling = stats.Handle x idle <- false) // 2. top up provisioning of writers queue - let capacity,_ = dispatcher.AvailableCapacity + let capacity = dispatcher.CurrentCapacity if capacity <> 0 then - idle <- false let work = streams.Schedule(progressState.InScheduledOrder streams.QueueLength, capacity) let xs = (Seq.ofArray work).GetEnumerator() let mutable addsBeingAccepted = true while xs.MoveNext() && addsBeingAccepted do let (_,{stream = s} : StreamSpan) as item = xs.Current let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) + idle <- idle && not succeeded // any add makes it not idle addsBeingAccepted <- succeeded // 3. Periodically emit status info - stats.TryDump(dispatcher.AvailableCapacity,streams) + stats.TryDump(capacity,dispatcher.State,streams) // 4. Do a minimal sleep so we don't run completely hot when empty if idle then do! Async.Sleep sleepIntervalMs } static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = @@ -230,7 +237,7 @@ module Scheduling = /// Attempt to feed in a batch (subject to there being capacity to do so) member __.TrySubmit(markCompleted, events) = async { - let! got = batches.Await(TimeSpan.Zero) + let! got = batches.TryAwait() if got then work.Enqueue <| Add (markCompleted, events) return got } @@ -421,17 +428,17 @@ module Ingestion = Async.Start(progressWriter.Pump(), cts.Token) while not cts.IsCancellationRequested do work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - let mutable ingesterAccepting = true + let mutable schedulerAccepting = true // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && submissionsMax.HasCapacity && ingesterAccepting do + while pending.Count <> 0 && submissionsMax.HasCapacity && schedulerAccepting do let markCompleted, events = pending.Peek() let! submitted = scheduler.TrySubmit(markCompleted, events) if submitted then pending.Dequeue() |> ignore - // mark off a write as being in progress + // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) do! submissionsMax.Await() else - ingesterAccepting <- false + schedulerAccepting <- false // 2. Update any progress into the stats stats.HandleValidated(Option.map fst validatedPos, fst readMax.State) validatedPos |> Option.iter progressWriter.Post From d241a49be80f551d1865eb430ffcd5d4953a1038 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 3 May 2019 23:04:16 +0100 Subject: [PATCH 203/353] Tmp --- .../Equinox.Projection/Projection.fs | 91 ++++++++++--------- 1 file changed, 47 insertions(+), 44 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index 7713588d4..074db6f27 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -379,51 +379,54 @@ module Ingestion = let mutable activeSeries = initialSeriesIndex let mutable validatedPos = None + let handle = function + | Batch (seriesId, epoch, checkpoint, items) -> + let batchInfo = + let items = Array.ofSeq items + streams.Merge items + let markCompleted () = + Log.Error("MC") + submissionsMax.Release() + readMax.Release() + validatedPos <- Some (epoch,checkpoint) + work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) + markCompleted, items + if activeSeries = seriesId then pending.Enqueue batchInfo + else + match readingAhead.TryGetValue seriesId with + | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) + | true,current -> current.Add(batchInfo) + | ActivateSeries newActiveSeries -> + activeSeries <- newActiveSeries + let buffered = + match ready |> tryRemove newActiveSeries with + | Some completedChunkBatches -> + completedChunkBatches |> Seq.iter pending.Enqueue + work.Enqueue <| ActivateSeries (newActiveSeries + 1) + completedChunkBatches.Count + | None -> + match readingAhead |> tryRemove newActiveSeries with + | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count + | None -> 0 + log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", + newActiveSeries, buffered, ready.Count, readingAhead.Count) + | CloseSeries seriesIndex -> + if activeSeries = seriesIndex then + log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) + work.Enqueue <| ActivateSeries (activeSeries + 1) + else + match readingAhead |> tryRemove seriesIndex with + | Some batchesRead -> + ready.[seriesIndex] <- batchesRead + log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) + | None -> + ready.[seriesIndex] <- ResizeArray() + log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) + // These events are for stats purposes + | Added _ + | ProgressResult _ -> () + member private __.Pump() = async { - let handle = function - | Batch (seriesId, epoch, checkpoint, items) -> - let batchInfo = - let items = Array.ofSeq items - streams.Merge items - let markCompleted () = - submissionsMax.Release() - readMax.Release() - validatedPos <- Some (epoch,checkpoint) - work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) - markCompleted, items - if activeSeries = seriesId then pending.Enqueue batchInfo - else - match readingAhead.TryGetValue seriesId with - | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) - | true,current -> current.Add(batchInfo) - | ActivateSeries newActiveSeries -> - activeSeries <- newActiveSeries - let buffered = - match ready |> tryRemove newActiveSeries with - | Some completedChunkBatches -> - completedChunkBatches |> Seq.iter pending.Enqueue - work.Enqueue <| ActivateSeries (newActiveSeries + 1) - completedChunkBatches.Count - | None -> - match readingAhead |> tryRemove newActiveSeries with - | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count - | None -> 0 - log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", - newActiveSeries, buffered, ready.Count, readingAhead.Count) - | CloseSeries seriesIndex -> - if activeSeries = seriesIndex then - log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) - work.Enqueue <| ActivateSeries (activeSeries + 1) - else - match readingAhead |> tryRemove seriesIndex with - | Some batchesRead -> - ready.[seriesIndex] <- batchesRead - log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) - | None -> - ready.[seriesIndex] <- ResizeArray() - log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) - // These events are for stats purposes - | Added _ | ProgressResult _ -> () use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) while not cts.IsCancellationRequested do From e568a744e8bc78caa867bd5749d91232a7105b40 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 00:06:50 +0100 Subject: [PATCH 204/353] Add unprefixed stats --- .../CosmosIngester.fs | 2 +- .../Equinox.Projection/Projection.fs | 1 - equinox-projector/Equinox.Projection/State.fs | 31 ++++++++++++------- 3 files changed, 21 insertions(+), 13 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 9be03f028..119edff94 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -104,7 +104,7 @@ type Stats(log : ILogger, statsInterval) = let start (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, statsInterval) = let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 - let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + 96 + let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (writePos : int64 option, batch : StreamSpan) = let mutable bytesBudget = cosmosPayloadLimit diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index 074db6f27..a3c53bbb5 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -385,7 +385,6 @@ module Ingestion = let items = Array.ofSeq items streams.Merge items let markCompleted () = - Log.Error("MC") submissionsMax.Release() readMax.Release() validatedPos <- Some (epoch,checkpoint) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index fd9abf3a1..6ee22d26a 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -4,6 +4,7 @@ open Serilog open System.Collections.Generic let arrayBytes (x:byte[]) = if x = null then 0 else x.Length +let inline eventSize (x : Equinox.Codec.IEvent<_>) = arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16 let mb x = float x / 1024. / 1024. let category (streamName : string) = streamName.Split([|'-'|],2).[0] @@ -12,7 +13,7 @@ type [] StreamSpan = { stream: string; span: Span } type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with member __.Size = if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy (fun x -> arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16) + else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy eventSize member __.IsReady = if __.queue = null || __.isMalformed then false else @@ -98,7 +99,7 @@ type StreamStates() = while xs.MoveNext() && remaining <> 0 do let x = xs.Current let state = states.[x] - if not state.isMalformed && busy.Add x then + if state.IsReady && busy.Add x then toSchedule.Add(state.write, { stream = x; span = state.queue.[0] }) remaining <- remaining - 1 toSchedule.ToArray() @@ -120,6 +121,8 @@ type StreamStates() = updateWritePos stream isMalformed None [| { index = 0L; events = null } |] member __.QueueLength(stream) = states.[stream].queue.[0].events.Length + member __.QueueLength(stream) = + states.[stream].queue.[0].events.Length member __.MarkCompleted(stream, index) = markNotBusy stream markCompleted stream index @@ -128,28 +131,34 @@ type StreamStates() = member __.Schedule(requestedOrder : string seq, capacity: int) : (int64 option * StreamSpan)[] = schedule requestedOrder capacity member __.Dump(log : ILogger) = - let mutable busyCount, busyB, ready, readyB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0 - let busyCats, readyCats, readyStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats() + let mutable busyCount, busyB, ready, readyB, unprefixed, unprefixedB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0 + let busyCats, readyCats, readyStreams, unprefixedStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats(), CatStats() + let kb sz = (sz + 512L) / 1024L for KeyValue (stream,state) in states do match int64 state.Size with | 0L -> synced <- synced + 1 - | sz when state.isMalformed -> - malformedStreams.Ingest(stream, mb sz |> int64) - malformed <- malformed + 1 - malformedB <- malformedB + sz | sz when busy.Contains stream -> busyCats.Ingest(category stream) busyCount <- busyCount + 1 busyB <- busyB + sz + | sz when state.isMalformed -> + malformedStreams.Ingest(stream, mb sz |> int64) + malformed <- malformed + 1 + malformedB <- malformedB + sz + | sz when not state.IsReady -> + unprefixedStreams.Ingest(stream, mb sz |> int64) + unprefixed <- unprefixed + 1 + unprefixedB <- unprefixedB + sz | sz -> readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) + readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) ready <- ready + 1 readyB <- readyB + sz - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", - synced, busyCount, mb busyB, ready, mb readyB, malformed, mb malformedB) + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Missing {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb:n1}MB", + synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + if unprefixedStreams.Any then log.Information("Missing Streams, KB {missingStreams}", Seq.truncate 5 unprefixedStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) \ No newline at end of file From b3458c847d90f2738a2e05eaaa9048b86617d0d6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 00:08:40 +0100 Subject: [PATCH 205/353] fix --- equinox-projector/Equinox.Projection/State.fs | 2 -- 1 file changed, 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 6ee22d26a..5a87248b8 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -121,8 +121,6 @@ type StreamStates() = updateWritePos stream isMalformed None [| { index = 0L; events = null } |] member __.QueueLength(stream) = states.[stream].queue.[0].events.Length - member __.QueueLength(stream) = - states.[stream].queue.[0].events.Length member __.MarkCompleted(stream, index) = markNotBusy stream markCompleted stream index From 449c4e55cefa10e7155e0b137139954733e68253 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 00:49:25 +0100 Subject: [PATCH 206/353] Fix locking; extend ordering --- .../Equinox.Projection/Projection.fs | 9 +++--- equinox-projector/Equinox.Projection/State.fs | 28 ++++++++----------- 2 files changed, 17 insertions(+), 20 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index a3c53bbb5..87fb91727 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -54,7 +54,7 @@ module Progress = while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do let batch = pending.Dequeue() batch.markCompleted() - member __.InScheduledOrder getStreamQueueLength = + member __.InScheduledOrder getStreamWeight = let raw = seq { let streams = HashSet() let mutable batch = 0 @@ -62,7 +62,7 @@ module Progress = batch <- batch + 1 for s in x.streamToRequiredIndex.Keys do if streams.Add s then - yield s,(batch,getStreamQueueLength s) } + yield s,(batch,getStreamWeight s) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst /// Manages writing of progress @@ -217,12 +217,13 @@ module Scheduling = // 2. top up provisioning of writers queue let capacity = dispatcher.CurrentCapacity if capacity <> 0 then - let work = streams.Schedule(progressState.InScheduledOrder streams.QueueLength, capacity) - let xs = (Seq.ofArray work).GetEnumerator() + let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight, capacity) + let xs = potential.GetEnumerator() let mutable addsBeingAccepted = true while xs.MoveNext() && addsBeingAccepted do let (_,{stream = s} : StreamSpan) as item = xs.Current let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) + if succeeded then streams.MarkBusy stream idle <- idle && not succeeded // any add makes it not idle addsBeingAccepted <- succeeded // 3. Periodically emit status info diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 5a87248b8..ead225df2 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -92,19 +92,13 @@ type StreamStates() = let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore let busy = HashSet() - let schedule (requestedOrder : string seq) (capacity: int) = - let toSchedule = ResizeArray<_>(capacity) - let xs = requestedOrder.GetEnumerator() - let mutable remaining = capacity - while xs.MoveNext() && remaining <> 0 do - let x = xs.Current + let pending (requestedOrder : string seq) = seq { + for x in requestedOrder do let state = states.[x] - if state.IsReady && busy.Add x then - toSchedule.Add(state.write, { stream = x; span = state.queue.[0] }) - remaining <- remaining - 1 - toSchedule.ToArray() - let markNotBusy stream = - busy.Remove stream |> ignore + if state.IsReady then + yield state.write, { stream = x; span = state.queue.[0] } } + let markBusy stream = busy.Add stream |> ignore + let markNotBusy stream = busy.Remove stream |> ignore // Result is intentionally a thread-safe persisent data structure // This enables the (potentially multiple) Ingesters to determine streams (for which they potentially have successor events) that are in play @@ -119,15 +113,17 @@ type StreamStates() = updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] member __.SetMalformed(stream,isMalformed) = updateWritePos stream isMalformed None [| { index = 0L; events = null } |] - member __.QueueLength(stream) = - states.[stream].queue.[0].events.Length + member __.QueueWeight(stream) = + states.[stream].queue.[0].events |> Seq.sumBy eventSize + member __.MarkBusy stream = + markBusy stream member __.MarkCompleted(stream, index) = markNotBusy stream markCompleted stream index member __.MarkFailed stream = markNotBusy stream - member __.Schedule(requestedOrder : string seq, capacity: int) : (int64 option * StreamSpan)[] = - schedule requestedOrder capacity + member __.Pending(byQueuedPriority : string seq) : (int64 option * StreamSpan) seq = + pending byQueuedPriority member __.Dump(log : ILogger) = let mutable busyCount, busyB, ready, readyB, unprefixed, unprefixedB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0 let busyCats, readyCats, readyStreams, unprefixedStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats(), CatStats() From d3fff060426cfe09e811d43ce4ab9d91fe2883d9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 00:51:06 +0100 Subject: [PATCH 207/353] Fix transition --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index e3121f4c8..63e291050 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -316,7 +316,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re let nextPos = posFromChunkAfter nextChunk remainder <- Some nextPos do! forkRunRelease <| Req.Chunk (seriesId, Range(nextChunk, Some nextPos, max), defaultBatchSize) - | (false, _), x -> + | (false, _), Some _ -> dop.Release() |> ignore Log.Warning("No further ingestion work to commence, transitioning to tailing...") remainder <- None From cc89893866d90add480b727f3cbc75a81dc9bd57 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 00:54:57 +0100 Subject: [PATCH 208/353] fix --- equinox-projector/Equinox.Projection/Projection.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index 87fb91727..14ac210e1 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -217,13 +217,13 @@ module Scheduling = // 2. top up provisioning of writers queue let capacity = dispatcher.CurrentCapacity if capacity <> 0 then - let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight, capacity) + let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight) let xs = potential.GetEnumerator() let mutable addsBeingAccepted = true while xs.MoveNext() && addsBeingAccepted do let (_,{stream = s} : StreamSpan) as item = xs.Current let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) - if succeeded then streams.MarkBusy stream + if succeeded then streams.MarkBusy s idle <- idle && not succeeded // any add makes it not idle addsBeingAccepted <- succeeded // 3. Periodically emit status info From 407c70a953650b4f7fbcecf16cbf8916f29949ad Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 01:28:47 +0100 Subject: [PATCH 209/353] Omit busy --- equinox-projector/Equinox.Projection/State.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index ead225df2..2f3c98250 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -95,7 +95,7 @@ type StreamStates() = let pending (requestedOrder : string seq) = seq { for x in requestedOrder do let state = states.[x] - if state.IsReady then + if state.IsReady && not (busy.Contains x) then yield state.write, { stream = x; span = state.queue.[0] } } let markBusy stream = busy.Add stream |> ignore let markNotBusy stream = busy.Remove stream |> ignore From 0ef85123cc1af89b2fd0e397a5539d18e64b205a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 01:46:49 +0100 Subject: [PATCH 210/353] Add filled metric --- .../Equinox.Projection/Projection.fs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index 14ac210e1..df73e64b9 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -105,11 +105,11 @@ module Scheduling = /// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = - let cycles, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let cycles, filled, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats capacity (used,maxDop) = - log.Information("Projection Cycles {cycles} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + log.Information("Projection Cycles {cycles} Filled {filled:P0} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + !cycles, float !filled/float !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function @@ -123,8 +123,9 @@ module Scheduling = incr resultCompleted | Result (_stream, Choice2Of2 _) -> incr resultExn - member __.TryDump(capacity,(used,max),streams : StreamStates) = + member __.TryDump(wasFull,capacity,(used,max),streams : StreamStates) = incr cycles + if wasFull then incr filled if statsDue () then dumpStats capacity (used,max) __.DumpExtraStats() @@ -216,10 +217,10 @@ module Scheduling = idle <- false) // 2. top up provisioning of writers queue let capacity = dispatcher.CurrentCapacity - if capacity <> 0 then + let mutable addsBeingAccepted = capacity <> 0 + if addsBeingAccepted then let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight) let xs = potential.GetEnumerator() - let mutable addsBeingAccepted = true while xs.MoveNext() && addsBeingAccepted do let (_,{stream = s} : StreamSpan) as item = xs.Current let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) @@ -227,7 +228,7 @@ module Scheduling = idle <- idle && not succeeded // any add makes it not idle addsBeingAccepted <- succeeded // 3. Periodically emit status info - stats.TryDump(capacity,dispatcher.State,streams) + stats.TryDump(not addsBeingAccepted,capacity,dispatcher.State,streams) // 4. Do a minimal sleep so we don't run completely hot when empty if idle then do! Async.Sleep sleepIntervalMs } static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = From 41bfddbab9a3f0721efb0ed97a4f06a2edddf64c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 01:54:21 +0100 Subject: [PATCH 211/353] Fix percentage --- equinox-projector/Equinox.Projection/Projection.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index df73e64b9..7284840f4 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -110,7 +110,7 @@ module Scheduling = let dumpStats capacity (used,maxDop) = log.Information("Projection Cycles {cycles} Filled {filled:P0} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", !cycles, float !filled/float !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + cycles := 0; filled := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function | Add _ | AddActive _ -> () From 65b2db469249e3ce74100bbd6d64de1406fa8cdb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 02:16:21 +0100 Subject: [PATCH 212/353] Trim first batch better --- .../Equinox.Projection.Cosmos/CosmosIngester.fs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs index 119edff94..285921220 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs +++ b/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs @@ -107,14 +107,17 @@ let start (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (writePos : int64 option, batch : StreamSpan) = - let mutable bytesBudget = cosmosPayloadLimit + // Reduce the item count when we don't yet know the write position in order to efficiently discover the redundancy where data is already present + // 100K budget for first page of an event makes validations cheaper while retaining general efficiency + let mutable bytesBudget, countBudget = match writePos with None -> 100 * 1024, 100 | Some _ -> cosmosPayloadLimit, 4096 let mutable count = 0 let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = bytesBudget <- bytesBudget - cosmosPayloadBytes y + countBudget <- countBudget - 1 count <- count + 1 - // Reduce the item count when we don't yet know the write position in order to efficiently discover the redundancy where data is already present - count <= (if Option.isNone writePos then 100 else 4096) && (bytesBudget >= 0 || count = 1) // always send at least one event in order to surface the problem and have the stream mark malformed - { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } + // always send at least one event in order to surface the problem and have the stream mark malformed + count = 1 || (countBudget >= 0 && bytesBudget >= 0) + { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } let project batch = async { let trimmed = trim batch try let! res = Writer.write log cosmosContext trimmed From e6cc9df30b018a5b76776b5e7cb5010a641a6391 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 02:41:22 +0100 Subject: [PATCH 213/353] Slipstream repeats as fillers --- equinox-projector/Equinox.Projection/State.fs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 2f3c98250..293b8be11 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -93,10 +93,14 @@ type StreamStates() = let busy = HashSet() let pending (requestedOrder : string seq) = seq { - for x in requestedOrder do - let state = states.[x] - if state.IsReady && not (busy.Contains x) then - yield state.write, { stream = x; span = state.queue.[0] } } + for s in requestedOrder do + let state = states.[s] + if state.IsReady && not (busy.Contains s) then + yield state.write, { stream = s; span = state.queue.[0] } + // [lazily] Slipstream in futher events that have been posted to streams which we've already visited + for KeyValue(s,v) in states do + if v.IsReady && not (busy.Contains s) then + yield v.write, { stream = s; span = v.queue.[0] } } let markBusy stream = busy.Add stream |> ignore let markNotBusy stream = busy.Remove stream |> ignore From 793768673f4c567411da5047836583e8885d080a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 03:04:01 +0100 Subject: [PATCH 214/353] Prevent too many tails --- equinox-sync/Sync/EventStoreSource.fs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 63e291050..ebd044307 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -201,7 +201,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re let sleepIntervalMs = 100 let overallStats = OverallStats(?statsInterval=statsInterval) let slicesStats = SliceStatsBuffer() - let mutable eofSpottedInChunk = false + let mutable eofSpottedInChunk = 0 /// Invoked by pump to process a tranche of work; can have parallel invocations let exec conn req = async { @@ -238,8 +238,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) overallStats.DumpIfIntervalExpired(true) let! _ = post (Res.EndOfChunk series) in () - work.Enqueue <| Req.Tail (series+1, pos, pos, tailInterval, defaultBatchSize) - eofSpottedInChunk <- true + if 1 = Interlocked.Increment &eofSpottedInChunk then work.Enqueue <| Req.Tail (series+1, pos, pos, tailInterval, defaultBatchSize) | PullResult.EndOfTranche -> Log.Information("completed tranche in {ms:n1}m", let e = t.Elapsed in e.TotalMinutes) let! _ = post (Res.EndOfChunk series) in () @@ -383,7 +382,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","EqxCosmos"), maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) if spec.gorge then @@ -393,7 +392,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro chunk startPos |> int, conns, conns.Length else 0, [|conn|], spec.stripes+1 - let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","EventStore"), cosmosIngestionEngine, maxReadAhead, maxProcessing, initialSeriesId, TimeSpan.FromMinutes 1.) + let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","ES"), cosmosIngestionEngine, maxReadAhead, maxProcessing, initialSeriesId, TimeSpan.FromMinutes 1.) let post = function | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId | Res.Batch (seriesId, pos, xs) -> From 52992b58851c963962aecb36eb4086fdb2eba5fe Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 03:05:20 +0100 Subject: [PATCH 215/353] fix --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index ebd044307..0ce4e46dd 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -310,7 +310,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re match work.TryDequeue(), remainder with | (true, task), _ -> do! forkRunRelease task - | (false, _), Some nextChunk when not eofSpottedInChunk -> + | (false, _), Some nextChunk when eofSpottedInChunk = 0 -> seriesId <- seriesId + 1 let nextPos = posFromChunkAfter nextChunk remainder <- Some nextPos From 6c2471cb70bf7edb3fa962167f489310864c54c7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 03:20:45 +0100 Subject: [PATCH 216/353] Add trimming --- equinox-projector/Equinox.Projection/Projection.fs | 9 ++++++--- equinox-projector/Equinox.Projection/State.fs | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs index 7284840f4..12803df0a 100644 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ b/equinox-projector/Equinox.Projection/Projection.fs @@ -44,17 +44,20 @@ module Progress = type State<'Pos>() = let pending = Queue<_>() + let trim () = + while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do + let batch = pending.Dequeue() + batch.markCompleted() member __.AppendBatch(markCompleted, reqs : Dictionary) = + trim () pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } member __.MarkStreamProgress(stream, index) = for x in pending do match x.streamToRequiredIndex.TryGetValue stream with | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore | _, _ -> () - while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do - let batch = pending.Dequeue() - batch.markCompleted() member __.InScheduledOrder getStreamWeight = + trim () let raw = seq { let streams = HashSet() let mutable batch = 0 diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index 293b8be11..e355b6465 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -153,7 +153,7 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) ready <- ready + 1 readyB <- readyB + sz - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Missing {waiting}/{waitingMb}MB Malformed {malformed}/{malformedMb:n1}MB", + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Missing {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) From 28474d0f2efc37224178367515833ebb481d4e6a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 4 May 2019 03:47:28 +0100 Subject: [PATCH 217/353] Relabel logs --- equinox-projector/Equinox.Projection/State.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs index e355b6465..d23e92caf 100644 --- a/equinox-projector/Equinox.Projection/State.fs +++ b/equinox-projector/Equinox.Projection/State.fs @@ -153,10 +153,10 @@ type StreamStates() = readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) ready <- ready + 1 readyB <- readyB + sz - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Missing {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - if unprefixedStreams.Any then log.Information("Missing Streams, KB {missingStreams}", Seq.truncate 5 unprefixedStreams.StatsDescending) + if unprefixedStreams.Any then log.Information("Waiting Streams, KB {missingStreams}", Seq.truncate 3 unprefixedStreams.StatsDescending) if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) \ No newline at end of file From 2a840df9bd59f75dba945c8faa96763d5b1547a7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 6 May 2019 08:07:15 +0100 Subject: [PATCH 218/353] Update to Equinox 2.0.0-preview6 --- equinox-projector/Consumer/Consumer.fsproj | 6 +- equinox-projector/Consumer/Infrastructure.fs | 39 +- .../CosmosInternalJson.fs | 57 -- .../Equinox.Projection.Codec.fsproj | 28 - .../Equinox.Projection.Codec/RenderedEvent.fs | 28 - .../Equinox.Cosmos.ProjectionEx.fsproj | 33 -- .../Equinox.Projection.Tests.fsproj | 30 -- .../FeedValidatorTests.fs | 70 --- .../Equinox.Projection.Tests/ProgressTests.fs | 39 -- .../StreamStateTests.fs | 78 --- .../Equinox.Projection.fsproj | 29 -- .../Equinox.Projection/FeedValidator.fs | 110 ---- .../Equinox.Projection/Infrastructure.fs | 44 -- .../Equinox.Projection/Projection.fs | 489 ------------------ equinox-projector/Equinox.Projection/State.fs | 162 ------ .../Projector.Tests/ProgressTests.fs | 41 -- .../Projector.Tests/Projector.Tests.fsproj | 26 - .../Projector.Tests/StreamStateTests.fs | 78 --- equinox-projector/Projector/Infrastructure.fs | 25 +- equinox-projector/Projector/Program.fs | 18 +- equinox-projector/Projector/Projector.fsproj | 10 +- .../equinox-projector-consumer.sln | 24 - .../Sync}/CosmosIngester.fs | 0 equinox-sync/Sync/CosmosSource.fs | 8 +- .../Sync}/Metrics.fs | 4 +- equinox-sync/Sync/Sync.fsproj | 10 +- 26 files changed, 32 insertions(+), 1454 deletions(-) delete mode 100644 equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs delete mode 100644 equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj delete mode 100644 equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs delete mode 100644 equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj delete mode 100644 equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj delete mode 100644 equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs delete mode 100644 equinox-projector/Equinox.Projection.Tests/ProgressTests.fs delete mode 100644 equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs delete mode 100644 equinox-projector/Equinox.Projection/Equinox.Projection.fsproj delete mode 100644 equinox-projector/Equinox.Projection/FeedValidator.fs delete mode 100644 equinox-projector/Equinox.Projection/Infrastructure.fs delete mode 100644 equinox-projector/Equinox.Projection/Projection.fs delete mode 100644 equinox-projector/Equinox.Projection/State.fs delete mode 100644 equinox-projector/Projector.Tests/ProgressTests.fs delete mode 100644 equinox-projector/Projector.Tests/Projector.Tests.fsproj delete mode 100644 equinox-projector/Projector.Tests/StreamStateTests.fs rename {equinox-projector/Equinox.Projection.Cosmos => equinox-sync/Sync}/CosmosIngester.fs (100%) rename {equinox-projector/Equinox.Projection.Cosmos => equinox-sync/Sync}/Metrics.fs (98%) diff --git a/equinox-projector/Consumer/Consumer.fsproj b/equinox-projector/Consumer/Consumer.fsproj index 739fb0a98..279d4741c 100644 --- a/equinox-projector/Consumer/Consumer.fsproj +++ b/equinox-projector/Consumer/Consumer.fsproj @@ -14,9 +14,9 @@ - - - + + + diff --git a/equinox-projector/Consumer/Infrastructure.fs b/equinox-projector/Consumer/Infrastructure.fs index a1524f12f..1a52ea50d 100644 --- a/equinox-projector/Consumer/Infrastructure.fs +++ b/equinox-projector/Consumer/Infrastructure.fs @@ -1,44 +1,9 @@ [] module private ProjectorTemplate.Consumer.Infrastructure -open System -open System.Threading -open System.Threading.Tasks - -type Async with - static member AwaitTaskCorrect (task : Task<'T>) : Async<'T> = - Async.FromContinuations <| fun (k,ek,_) -> - task.ContinueWith (fun (t:Task<'T>) -> - if t.IsFaulted then - let e = t.Exception - if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] - else ek e - elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) - elif t.IsCompleted then k t.Result - else ek(Exception "invalid Task state!")) - |> ignore - - static member AwaitTaskCorrect (task : Task) : Async = - Async.FromContinuations <| fun (k,ek,_) -> - task.ContinueWith (fun (t:Task) -> - if t.IsFaulted then - let e = t.Exception - if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] - else ek e - elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) - elif t.IsCompleted then k () - else ek(Exception "invalid Task state!")) - |> ignore - -type SemaphoreSlim with - /// F# friendly semaphore await function - member semaphore.Await(?timeout : TimeSpan) = async { - let! ct = Async.CancellationToken - let timeout = defaultArg timeout Timeout.InfiniteTimeSpan - let task = semaphore.WaitAsync(timeout, ct) - return! Async.AwaitTaskCorrect task - } +open Equinox.Projection // semaphore.Await +type System.Threading.SemaphoreSlim with /// Throttling wrapper which waits asynchronously until the semaphore has available capacity member semaphore.Throttle(workflow : Async<'T>) : Async<'T> = async { let! _ = semaphore.Await() diff --git a/equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs b/equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs deleted file mode 100644 index 1ed994ce6..000000000 --- a/equinox-projector/Equinox.Projection.Codec/CosmosInternalJson.fs +++ /dev/null @@ -1,57 +0,0 @@ -namespace Equinox.Cosmos.Internal.Json - -open Newtonsoft.Json.Linq -open Newtonsoft.Json - -/// Manages injecting prepared json into the data being submitted to DocDb as-is, on the basis we can trust it to be valid json as DocDb will need it to be -type VerbatimUtf8JsonConverter() = - inherit JsonConverter() - - static let enc = System.Text.Encoding.UTF8 - - override __.ReadJson(reader, _, _, _) = - let token = JToken.Load reader - if token.Type = JTokenType.Null then null - else token |> string |> enc.GetBytes |> box - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - - override __.WriteJson(writer, value, serializer) = - let array = value :?> byte[] - if array = null || array.Length = 0 then serializer.Serialize(writer, null) - else writer.WriteRawValue(enc.GetString(array)) - -open System.IO -open System.IO.Compression - -/// Manages zipping of the UTF-8 json bytes to make the index record minimal from the perspective of the writer stored proc -/// Only applied to snapshots in the Index -type Base64ZipUtf8JsonConverter() = - inherit JsonConverter() - let pickle (input : byte[]) : string = - if input = null then null else - - use output = new MemoryStream() - use compressor = new DeflateStream(output, CompressionLevel.Optimal) - compressor.Write(input,0,input.Length) - compressor.Close() - System.Convert.ToBase64String(output.ToArray()) - let unpickle str : byte[] = - if str = null then null else - - let compressedBytes = System.Convert.FromBase64String str - use input = new MemoryStream(compressedBytes) - use decompressor = new DeflateStream(input, CompressionMode.Decompress) - use output = new MemoryStream() - decompressor.CopyTo(output) - output.ToArray() - - override __.CanConvert(objectType) = - typeof.Equals(objectType) - override __.ReadJson(reader, _, _, serializer) = - //( if reader.TokenType = JsonToken.Null then null else - serializer.Deserialize(reader, typedefof) :?> string |> unpickle |> box - override __.WriteJson(writer, value, serializer) = - let pickled = value |> unbox |> pickle - serializer.Serialize(writer, pickled) \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj b/equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj deleted file mode 100644 index bd30eac24..000000000 --- a/equinox-projector/Equinox.Projection.Codec/Equinox.Projection.Codec.fsproj +++ /dev/null @@ -1,28 +0,0 @@ - - - - netstandard2.0 - 5 - false - true - true - true - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs b/equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs deleted file mode 100644 index 0629dd3f4..000000000 --- a/equinox-projector/Equinox.Projection.Codec/RenderedEvent.fs +++ /dev/null @@ -1,28 +0,0 @@ -namespace Equinox.Projection - -open Newtonsoft.Json -open System - -module Codec = - /// Default rendition of an event when being projected to Kafka - type [] RenderedEvent = - { /// Stream Name - s: string - - /// Index within stream - i: int64 - - /// Event Type associated with event data in `d` - c: string - - /// Timestamp of original write - t: DateTimeOffset // ISO 8601 - - /// Event body, as UTF-8 encoded json ready to be injected directly into the Json being rendered - [)>] - d: byte[] // required - - /// Optional metadata, as UTF-8 encoded json, ready to emit directly (entire field is not written if value is null) - [)>] - [] - m: byte[] } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj b/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj deleted file mode 100644 index 4242a0a46..000000000 --- a/equinox-projector/Equinox.Projection.Cosmos/Equinox.Cosmos.ProjectionEx.fsproj +++ /dev/null @@ -1,33 +0,0 @@ - - - - netstandard2.0 - 5 - false - true - true - true - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj b/equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj deleted file mode 100644 index 002dc015f..000000000 --- a/equinox-projector/Equinox.Projection.Tests/Equinox.Projection.Tests.fsproj +++ /dev/null @@ -1,30 +0,0 @@ - - - - netcoreapp2.1;net461 - 5 - false - - - - - - - - - - - - - - - - - - - all - runtime; build; native; contentfiles; analyzers - - - - diff --git a/equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs b/equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs deleted file mode 100644 index 7df7c8112..000000000 --- a/equinox-projector/Equinox.Projection.Tests/FeedValidatorTests.fs +++ /dev/null @@ -1,70 +0,0 @@ -module Equinox.Projection.FeedValidator.Tests - -open Equinox.Projection.Validation -open FsCheck.Xunit -open Swensen.Unquote -open Xunit - -let [] ``Properties`` state index = - let result,state' = StreamState.combine state index - match state with - | Some (Partial (_, pos)) when pos <= 1 -> () - | Some (Partial (min, pos)) when min > pos || min = 0 -> () - | None -> - match state' with - | All _ -> result =! New - | Partial _ -> result =! Ok - | Some (All x) -> - match state',result with - | All x' , Duplicate -> x' =! x - | All x', New -> x' =! x+1 - | All x', Gap -> x' =! x - | x -> failwithf "Unexpected %A" x - | Some (Partial (min, pos)) -> - match state',result with - | All 0,Duplicate when min=0 && index = 0 -> () - | All x,Duplicate when min=1 && pos = x && index = 0 -> () - | Partial (min', pos'), Duplicate -> min' =! max min' index; pos' =! pos - | Partial (min', pos'), Ok - | Partial (min', pos'), New -> min' =! min; pos' =! index - | x -> failwithf "Unexpected %A" x - -let [] ``Zero on unknown stream is New`` () = - let result,state = StreamState.combine None 0 - New =! result - All 0 =! state - -let [] ``Non-zero on unknown stream is Ok`` () = - let result,state = StreamState.combine None 1 - Ok =! result - Partial (1,1) =! state - -let [] ``valid successor is New`` () = - let result,state = StreamState.combine (All 0 |> Some) 1 - New =! result - All 1 =! state - -let [] ``single immediate repeat is flagged`` () = - let result,state = StreamState.combine (All 0 |> Some) 0 - Duplicate =! result - All 0 =! state - -let [] ``non-immediate repeat is flagged`` () = - let result,state = StreamState.combine (All 1 |> Some) 0 - Duplicate =! result - All 1 =! state - -let [] ``Gap is flagged`` () = - let result,state = StreamState.combine (All 1 |> Some) 3 - Gap =! result - All 1 =! state - -let [] ``Potential gaps are not flagged as such when we're processing a Partial`` () = - let result,state = StreamState.combine (Some (Partial (1,1))) 3 - New =! result - Partial (1,3) =! state - -let [] ``Earlier values widen the min when we're processing a Partial`` () = - let result,state = StreamState.combine (Some (Partial (2,3))) 1 - Duplicate =! result - Partial (1,3) =! state \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs b/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs deleted file mode 100644 index 6a1c15eff..000000000 --- a/equinox-projector/Equinox.Projection.Tests/ProgressTests.fs +++ /dev/null @@ -1,39 +0,0 @@ -module ProgressTests - -open Equinox.Projection -open Swensen.Unquote -open System.Collections.Generic -open Xunit - -let mkDictionary xs = Dictionary(dict xs) - -let [] ``Empty has zero streams pending or progress to write`` () = - let sut = Progress.State<_>() - let queue = sut.InScheduledOrder(fun _ -> 0) - test <@ Seq.isEmpty queue @> - -let [] ``Can add multiple batches with overlapping streams`` () = - let sut = Progress.State<_>() - let noBatchesComplete () = failwith "No bathes should complete" - sut.AppendBatch(noBatchesComplete, mkDictionary ["a",1L; "b",2L]) - sut.AppendBatch(noBatchesComplete, mkDictionary ["b",2L; "c",3L]) - -let [] ``Marking Progress removes batches and triggers the callbacks`` () = - let sut = Progress.State<_>() - let mutable callbacks = 0 - let complete () = callbacks <- callbacks + 1 - sut.AppendBatch(complete, mkDictionary ["a",1L; "b",2L]) - sut.MarkStreamProgress("a",1L) |> ignore - sut.MarkStreamProgress("b",1L) |> ignore - 1 =! callbacks - -let [] ``Marking progress is not persistent`` () = - let sut = Progress.State<_>() - let mutable callbacks = 0 - let complete () = callbacks <- callbacks + 1 - sut.AppendBatch(complete, mkDictionary ["a",1L]) - sut.MarkStreamProgress("a",2L) |> ignore - sut.AppendBatch(complete, mkDictionary ["a",1L; "b",2L]) - 1 =! callbacks - -// TODO: lots more coverage of newer functionality - the above were written very early into the exercise \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs b/equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs deleted file mode 100644 index 5431c7df7..000000000 --- a/equinox-projector/Equinox.Projection.Tests/StreamStateTests.fs +++ /dev/null @@ -1,78 +0,0 @@ -module CosmosIngesterTests - -open Equinox.Projection.State -open Swensen.Unquote -open Xunit - -let canonicalTime = System.DateTimeOffset.UtcNow -let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } -let mergeSpans = StreamState.Span.merge -let trimSpans = StreamState.Span.trim - -let [] ``nothing`` () = - let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] - r =! null - -let [] ``synced`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] - r =! null - -let [] ``no overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] - r =! [| mk 0L 1; mk 2L 2 |] - -let [] ``overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 0L 2 |] - -let [] ``remove nulls`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 1L 1 |] - -let [] ``adjacent`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 0L 3 |] - -let [] ``adjacent to min`` () = - let r = List.map (trimSpans 2L) [ mk 0L 1; mk 1L 2 ] - r =! [ mk 2L 0; mk 2L 1 ] - -let [] ``adjacent to min merge`` () = - let r = mergeSpans 2L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 2L 1 |] - -let [] ``adjacent to min no overlap`` () = - let r = mergeSpans 2L [ mk 0L 1; mk 2L 1 ] - r =! [| mk 2L 1|] - -let [] ``adjacent trim`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2 ] - r =! [ mk 1L 1; mk 2L 2 ] - -let [] ``adjacent trim merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] - r =! [| mk 1L 3 |] - -let [] ``adjacent trim append`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [ mk 1L 1; mk 2L 2; mk 5L 1 ] - -let [] ``adjacent trim append merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``mixed adjacent trim append`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 5L 1; mk 2L 2] - r =! [ mk 1L 1; mk 5L 1; mk 2L 2 ] - -let [] ``mixed adjacent trim append merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``fail`` () = - let r = mergeSpans 11614L [ {index=11614L; events=null}; mk 11614L 1 ] - r =! [| mk 11614L 1 |] - -let [] ``fail 2`` () = - let r = mergeSpans 11613L [ mk 11614L 1; {index=11614L; events=null} ] - r =! [| mk 11614L 1 |] diff --git a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj b/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj deleted file mode 100644 index cf5c893ef..000000000 --- a/equinox-projector/Equinox.Projection/Equinox.Projection.fsproj +++ /dev/null @@ -1,29 +0,0 @@ - - - - netstandard2.0 - 5 - false - true - true - true - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/FeedValidator.fs b/equinox-projector/Equinox.Projection/FeedValidator.fs deleted file mode 100644 index bdb176125..000000000 --- a/equinox-projector/Equinox.Projection/FeedValidator.fs +++ /dev/null @@ -1,110 +0,0 @@ -module Equinox.Projection.Validation - -open System.Collections.Generic - -module private Option = - let defaultValue def option = defaultArg option def - -/// Represents the categorisation of an item being ingested -type IngestResult = - /// The item is a correct item at the tail of the known sequence - | New - /// Consistent as far as we know (but this Validator has not seen the head) - | Ok - /// The item represents a duplicate of an earlier item - | Duplicate - /// The item is beyond the tail of the stream and likely represets a gap - | Gap - -/// Represents the present state of a given stream -type StreamState = - /// We've observed the stream from the start - | All of pos: int - /// We've not observed the stream from the start - | Partial of min: int * pos: int - -module StreamState = - let combine (state : StreamState option) index : IngestResult*StreamState = - match state, index with - | None, 0 -> New, All 0 - | None, x -> Ok, Partial (x,x) - | Some (All x), i when i <= x -> Duplicate, All x - | Some (All x), i when i = x + 1 -> New, All i - | Some (All x), _ -> Gap, All x - | Some (Partial (min=1; pos=pos)), 0 -> Duplicate, All pos - | Some (Partial (min=min; pos=x)), i when i <= min -> Duplicate, Partial (i, x) - | Some (Partial (min=min; pos=x)), i when i = x + 1 -> Ok, Partial (min, i) - | Some (Partial (min=min)), i -> New, Partial (min, i) - - -type FeedStats = { complete: int; partial: int } - -/// Maintains the state of a set of streams being ingested into a processor for consistency checking purposes -/// - to determine whether an incoming event on a stream should be considered a duplicate and hence not processed -/// - to allow inconsistencies to be logged -type FeedValidator() = - let streamStates = System.Collections.Generic.Dictionary() - - /// Thread safe operation to a) classify b) track change implied by a new message as encountered - member __.Ingest(stream, index) : IngestResult * StreamState = - lock streamStates <| fun () -> - let state = - match streamStates.TryGetValue stream with - | true, state -> Some state - | false, _ -> None - let (res, state') = StreamState.combine state index - streamStates.[stream] <- state' - res, state' - - /// Determine count of streams being tracked - member __.Stats = - lock streamStates <| fun () -> - let raw = streamStates |> Seq.countBy (fun x -> match x.Value with All _ -> true | Partial _ -> false) |> Seq.toList - { complete = raw |> List.tryPick (function (true,c) -> Some c | (false,_) -> None) |> Option.defaultValue 0 - partial = raw |> List.tryPick (function (false,c) -> Some c | (true,_) -> None) |> Option.defaultValue 0 } - -type [] StreamSummary = { mutable fresh : int; mutable ok : int; mutable dup : int; mutable gap : int; mutable complete: bool } - -type BatchStats = { fresh : int; ok : int; dup : int; gap : int; categories : int; streams : BatchStreamStats } with - member s.TotalStreams = let s = s.streams in s.complete + s.incomplete - -and BatchStreamStats = { complete: int; incomplete: int } - -/// Used to establish aggregate stats for a batch of inputs for logging purposes -/// The Ingested inputs are passed to the supplied validator in order to classify them -type BatchValidator(validator : FeedValidator) = - let streams = System.Collections.Generic.Dictionary() - let streamSummary (streamName : string) = - match streams.TryGetValue streamName with - | true, acc -> acc - | false, _ -> let t = { fresh = 0; ok = 0; dup = 0; gap = 0; complete = false } in streams.[streamName] <- t; t - - /// Collate into Feed Validator and Batch stats - member __.TryIngest(stream, index) : IngestResult = - let res, state = validator.Ingest(stream, index) - let streamStats = streamSummary stream - match state with - | All _ -> streamStats.complete <- true - | Partial _ -> streamStats.complete <- false - match res with - | New -> streamStats.fresh <- streamStats.fresh + 1 - | Ok -> streamStats.ok <- streamStats.ok + 1 - | Duplicate -> streamStats.dup <- streamStats.dup + 1 - | Gap -> streamStats.gap <- streamStats.gap + 1 - res - - member __.Enum() : IEnumerable> = upcast streams - - member __.Stats : BatchStats = - let mutable fresh, ok, dup, gap, complete, incomplete = 0, 0, 0, 0, 0, 0 - let cats = HashSet() - for KeyValue(k,v) in streams do - fresh <- fresh + v.fresh - ok <- ok + v.ok - dup <- dup + v.dup - gap <- gap + v.gap - match k.IndexOf('-') with - | -1 -> () - | i -> cats.Add(k.Substring(0, i)) |> ignore - if v.complete then complete <- complete + 1 else incomplete <- incomplete + 1 - { fresh = fresh; ok = ok; dup = dup; gap = gap; categories = cats.Count; streams = { complete = complete; incomplete = incomplete } } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Infrastructure.fs b/equinox-projector/Equinox.Projection/Infrastructure.fs deleted file mode 100644 index cb82b676a..000000000 --- a/equinox-projector/Equinox.Projection/Infrastructure.fs +++ /dev/null @@ -1,44 +0,0 @@ -[] -module Equinox.Projection.Infrastructure - -open System -open System.Collections.Concurrent -open System.Threading -open System.Threading.Tasks - -module ConcurrentQueue = - let drain handle (xs : ConcurrentQueue<_>) = - let rec aux () = - match xs.TryDequeue() with - | false, _ -> () - | true, x -> handle x; aux () - aux () - -type Async with - /// Asynchronously awaits the next keyboard interrupt event - static member AwaitKeyboardInterrupt () : Async = - Async.FromContinuations(fun (sc,_,_) -> - let isDisposed = ref 0 - let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore - and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) - static member AwaitTaskCorrect (task : Task<'T>) : Async<'T> = - Async.FromContinuations <| fun (k,ek,_) -> - task.ContinueWith (fun (t:Task<'T>) -> - if t.IsFaulted then - let e = t.Exception - if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] - else ek e - elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) - elif t.IsCompleted then k t.Result - else ek(Exception "invalid Task state!")) - |> ignore - -type SemaphoreSlim with - /// F# friendly semaphore await function - member semaphore.Await(?timeout : TimeSpan) = async { - let! ct = Async.CancellationToken - let timeout = defaultArg timeout Timeout.InfiniteTimeSpan - let task = semaphore.WaitAsync(timeout, ct) - return! Async.AwaitTaskCorrect task - } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/Projection.fs b/equinox-projector/Equinox.Projection/Projection.fs deleted file mode 100644 index 12803df0a..000000000 --- a/equinox-projector/Equinox.Projection/Projection.fs +++ /dev/null @@ -1,489 +0,0 @@ -namespace Equinox.Projection - -open Equinox.Projection.State -open Serilog -open System -open System.Collections.Concurrent -open System.Collections.Generic -open System.Diagnostics -open System.Threading - -/// Item from a reader as supplied to the `IIngester` -type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } - -/// Core interface for projection system, representing the complete contract a feed consumer uses to deliver batches of work for projection -type IIngester = - /// Passes a (lazy) batch of items into the Ingestion Engine; the batch will be materialized out of band and submitted to the Projection engine for scheduling - /// Admission is Async in order that the Projector and Ingester can together contrive to force backpressure on the producer of the batches - /// Returns dynamic position in the queue at time of posting, together with current max number of items permissible - abstract member Submit: progressEpoch: int64 * markCompleted: Async * items: StreamItem seq -> Async - /// Requests immediate cancellation of - abstract member Stop: unit -> unit - -[] -module private Helpers = - let expiredMs ms = - let timer = Stopwatch.StartNew() - fun () -> - let due = timer.ElapsedMilliseconds > ms - if due then timer.Restart() - due - type Sem(max) = - let inner = new SemaphoreSlim(max) - member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore - member __.State = max-inner.CurrentCount,max - /// Wait infinitely to get the semaphore - member __.Await() = inner.Await() |> Async.Ignore - /// Wait for the specified timeout to acquire (or return false instantly) - member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) - member __.HasCapacity = inner.CurrentCount > 0 - member __.CurrentCapacity = inner.CurrentCount - -module Progress = - type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } - - type State<'Pos>() = - let pending = Queue<_>() - let trim () = - while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do - let batch = pending.Dequeue() - batch.markCompleted() - member __.AppendBatch(markCompleted, reqs : Dictionary) = - trim () - pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } - member __.MarkStreamProgress(stream, index) = - for x in pending do - match x.streamToRequiredIndex.TryGetValue stream with - | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore - | _, _ -> () - member __.InScheduledOrder getStreamWeight = - trim () - let raw = seq { - let streams = HashSet() - let mutable batch = 0 - for x in pending do - batch <- batch + 1 - for s in x.streamToRequiredIndex.Keys do - if streams.Add s then - yield s,(batch,getStreamWeight s) } - raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst - - /// Manages writing of progress - /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) - /// - retries until success or a new item is posted - type Writer<'Res when 'Res: equality>() = - let pumpSleepMs = 100 - let due = expiredMs 5000L - let mutable committedEpoch = None - let mutable validatedPos = None - let result = Event>() - [] member __.Result = result.Publish - member __.Post(version,f) = - Volatile.Write(&validatedPos,Some (version,f)) - member __.CommittedEpoch = Volatile.Read(&committedEpoch) - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match Volatile.Read &validatedPos with - | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> - try do! f - Volatile.Write(&committedEpoch, Some v) - result.Trigger (Choice1Of2 v) - with e -> result.Trigger (Choice2Of2 e) - | _ -> do! Async.Sleep pumpSleepMs } - -module Scheduling = - - /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners - [] - type InternalMessage<'R> = - /// Enqueue a batch of items with supplied progress marking function - | Add of markCompleted: (unit -> unit) * items: StreamItem[] - /// Stats per submitted batch for stats listeners to aggregate - | Added of streams: int * skip: int * events: int - /// Submit new data pertaining to a stream that has commenced processing - | AddActive of KeyValuePair[] - /// Result of processing on stream - result (with basic stats) or the `exn` encountered - | Result of stream: string * outcome: Choice<'R,exn> - - /// Gathers stats pertaining to the core projection/ingestion activity - type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = - let cycles, filled, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats capacity (used,maxDop) = - log.Information("Projection Cycles {cycles} Filled {filled:P0} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, float !filled/float !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) - cycles := 0; filled := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 - abstract member Handle : InternalMessage<'R> -> unit - default __.Handle msg = msg |> function - | Add _ | AddActive _ -> () - | Added (streams, skipped, events) -> - incr batchesPended - streamsPended := !streamsPended + streams - eventsPended := !eventsPended + events - eventsSkipped := !eventsSkipped + skipped - | Result (_stream, Choice1Of2 _) -> - incr resultCompleted - | Result (_stream, Choice2Of2 _) -> - incr resultExn - member __.TryDump(wasFull,capacity,(used,max),streams : StreamStates) = - incr cycles - if wasFull then incr filled - if statsDue () then - dumpStats capacity (used,max) - __.DumpExtraStats() - streams.Dump log - /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) - abstract DumpExtraStats : unit -> unit - default __.DumpExtraStats () = () - - /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint - type Dispatcher<'R>(maxDop) = - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) - let result = Event<'R>() - let dop = new Sem(maxDop) - let dispatch work = async { - let! res = work - result.Trigger res - dop.Release() } - [] member __.Result = result.Publish - member __.HasCapacity = dop.HasCapacity - member __.CurrentCapacity = dop.CurrentCapacity - member __.State = dop.State - member __.TryAdd(item,?timeout) = async { - let! got = dop.TryAwait(?timeout=timeout) - if got then - work.Add(item) - return got } - member __.Pump () = async { - let! ct = Async.CancellationToken - for item in work.GetConsumingEnumerable ct do - Async.Start(dispatch item) } - - /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order - /// a) does not itself perform any reading activities - /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) - /// c) submits work to the supplied Dispatcher (which it triggers pumping of) - /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = - let sleepIntervalMs = 1 - let cts = new CancellationTokenSource() - let batches = Sem maxPendingBatches - let work = ConcurrentQueue>() - let streams = StreamStates() - let progressState = Progress.State() - - member private __.Pump(stats : Stats<'R>) = async { - use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) - Async.Start(dispatcher.Pump(), cts.Token) - let validVsSkip (streamState : StreamState) (item : StreamItem) = - match streamState.write, item.index + 1L with - | Some cw, required when cw >= required -> 0, 1 - | _ -> 1, 0 - let handle x = - match x with - | Add (releaseRead, items) -> - let reqs = Dictionary() - let mutable count, skipCount = 0, 0 - for item in items do - let stream,streamState = streams.Add(item.stream,item.index,item.event) - match validVsSkip streamState item with - | 0, skip -> - skipCount <- skipCount + skip - | required, _ -> - count <- count + required - reqs.[stream] <- item.index+1L - let markCompleted () = - releaseRead() - batches.Release() - progressState.AppendBatch(markCompleted,reqs) - work.Enqueue(Added (reqs.Count,skipCount,count)) - | AddActive events -> - for e in events do - streams.InternalMerge(e.Key,e.Value) - | Added _ -> - () - | Result (stream,r) -> - match interpretProgress streams stream r with - | Some index -> - progressState.MarkStreamProgress(stream,index) - streams.MarkCompleted(stream,index) - | None -> - streams.MarkFailed stream - - while not cts.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - let mutable idle = true - work |> ConcurrentQueue.drain (fun x -> - handle x - stats.Handle x - idle <- false) - // 2. top up provisioning of writers queue - let capacity = dispatcher.CurrentCapacity - let mutable addsBeingAccepted = capacity <> 0 - if addsBeingAccepted then - let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight) - let xs = potential.GetEnumerator() - while xs.MoveNext() && addsBeingAccepted do - let (_,{stream = s} : StreamSpan) as item = xs.Current - let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) - if succeeded then streams.MarkBusy s - idle <- idle && not succeeded // any add makes it not idle - addsBeingAccepted <- succeeded - // 3. Periodically emit status info - stats.TryDump(not addsBeingAccepted,capacity,dispatcher.State,streams) - // 4. Do a minimal sleep so we don't run completely hot when empty - if idle then do! Async.Sleep sleepIntervalMs } - static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = - let dispatcher = Dispatcher(processorDop) - let instance = new Engine<'R>(maxPendingBatches, dispatcher, project, interpretProgress) - Async.Start <| instance.Pump(stats) - instance - - /// Attempt to feed in a batch (subject to there being capacity to do so) - member __.TrySubmit(markCompleted, events) = async { - let! got = batches.TryAwait() - if got then - work.Enqueue <| Add (markCompleted, events) - return got } - - member __.AddOpenStreamData(events) = - work.Enqueue <| AddActive events - - member __.AllStreams = streams.All - - member __.Stop() = - cts.Cancel() - -type Projector = - static member Start(log, maxPendingBatches, maxActiveBatches, project : StreamSpan -> Async, ?statsInterval) = - let project (_maybeWritePos, batch) = async { - try let! count = project batch - return Choice1Of2 (batch.span.index + int64 count) - with e -> return Choice2Of2 e } - let interpretProgress _streams _stream = function - | Choice1Of2 index -> Some index - | Choice2Of2 _ -> None - let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.)) - Scheduling.Engine.Start(stats, maxPendingBatches, maxActiveBatches, project, interpretProgress) - -module Ingestion = - - [] - type Message = - | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq - //| StreamSegment of span: StreamSpan - | EndOfSeries of seriesIndex: int - - type private Streams() = - let states = Dictionary() - let merge stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - - member __.Merge(items : StreamItem seq) = - for item in items do - merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = Array.singleton item.event } |] } - - member __.Take(processingContains) = - let forward = [| for x in states do if processingContains x.Key then yield x |] - for x in forward do states.Remove x.Key |> ignore - forward - - member __.Dump(log : ILogger) = - let mutable waiting, waitingB = 0, 0L - let waitingCats, waitingStreams = CatStats(), CatStats() - for KeyValue (stream,state) in states do - let sz = int64 state.Size - waitingCats.Ingest(category stream) - waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) - waiting <- waiting + 1 - waitingB <- waitingB + sz - log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) - if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) - if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) - - type private Stats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = - let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None - let progCommitFails, progCommits = ref 0, ref 0 - let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (available,maxDop) = - log.Information("Holding Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", - !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 - if !progCommitFails <> 0 || !progCommits <> 0 then - match comittedEpoch with - | None -> - log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) - | Some committed when !progCommitFails <> 0 -> - log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) - | Some committed -> - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) - progCommits := 0; progCommitFails := 0 - else - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - member __.Handle : InternalMessage -> unit = function - | Batch _ | ActivateSeries _ | CloseSeries _-> () // stats are managed via Added internal message in same cycle - | ProgressResult (Choice1Of2 epoch) -> - incr progCommits - comittedEpoch <- Some epoch - | ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progCommitFails - | Added (streams,events) -> - incr batchesPended - streamsPended := !streamsPended + streams - eventsPended := !eventsPended + events - member __.HandleValidated(epoch, pendingBatches) = - validatedEpoch <- epoch - pendingBatchCount <- pendingBatches - member __.HandleCommitted epoch = - comittedEpoch <- epoch - member __.TryDump((available,maxDop),streams : Streams) = - incr cycles - if statsDue () then - dumpStats (available,maxDop) - streams.Dump log - - and [] private InternalMessage = - | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq - /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` - | ProgressResult of Choice - /// Internal message for stats purposes - | Added of steams: int * events: int - | CloseSeries of seriesIndex: int - | ActivateSeries of seriesIndex: int - - let tryRemove key (dict: Dictionary<_,_>) = - match dict.TryGetValue key with - | true, value -> - dict.Remove key |> ignore - Some value - | false, _ -> None - - /// Holds batches away from Core processing to limit in-flight processing - type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxQueued, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = - let cts = new CancellationTokenSource() - let pumpDelayMs = defaultArg pumpDelayMs 5 - let work = ConcurrentQueue() - let readMax = new Sem(maxQueued) - let submissionsMax = new Sem(maxSubmissions) - let streams = Streams() - let stats = Stats(log, maxQueued, statsInterval) - let pending = Queue<_>() - let readingAhead, ready = Dictionary>(), Dictionary>() - let progressWriter = Progress.Writer<_>() - let mutable activeSeries = initialSeriesIndex - let mutable validatedPos = None - - let handle = function - | Batch (seriesId, epoch, checkpoint, items) -> - let batchInfo = - let items = Array.ofSeq items - streams.Merge items - let markCompleted () = - submissionsMax.Release() - readMax.Release() - validatedPos <- Some (epoch,checkpoint) - work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) - markCompleted, items - if activeSeries = seriesId then pending.Enqueue batchInfo - else - match readingAhead.TryGetValue seriesId with - | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) - | true,current -> current.Add(batchInfo) - | ActivateSeries newActiveSeries -> - activeSeries <- newActiveSeries - let buffered = - match ready |> tryRemove newActiveSeries with - | Some completedChunkBatches -> - completedChunkBatches |> Seq.iter pending.Enqueue - work.Enqueue <| ActivateSeries (newActiveSeries + 1) - completedChunkBatches.Count - | None -> - match readingAhead |> tryRemove newActiveSeries with - | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count - | None -> 0 - log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", - newActiveSeries, buffered, ready.Count, readingAhead.Count) - | CloseSeries seriesIndex -> - if activeSeries = seriesIndex then - log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) - work.Enqueue <| ActivateSeries (activeSeries + 1) - else - match readingAhead |> tryRemove seriesIndex with - | Some batchesRead -> - ready.[seriesIndex] <- batchesRead - log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) - | None -> - ready.[seriesIndex] <- ResizeArray() - log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) - // These events are for stats purposes - | Added _ - | ProgressResult _ -> () - - member private __.Pump() = async { - use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) - Async.Start(progressWriter.Pump(), cts.Token) - while not cts.IsCancellationRequested do - work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - let mutable schedulerAccepting = true - // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && submissionsMax.HasCapacity && schedulerAccepting do - let markCompleted, events = pending.Peek() - let! submitted = scheduler.TrySubmit(markCompleted, events) - if submitted then - pending.Dequeue() |> ignore - // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) - do! submissionsMax.Await() - else - schedulerAccepting <- false - // 2. Update any progress into the stats - stats.HandleValidated(Option.map fst validatedPos, fst readMax.State) - validatedPos |> Option.iter progressWriter.Post - stats.HandleCommitted progressWriter.CommittedEpoch - // 3. Forward content for any active streams into processor immediately - let relevantBufferedStreams = streams.Take(scheduler.AllStreams.Contains) - scheduler.AddOpenStreamData(relevantBufferedStreams) - // 4. Periodically emit status info - stats.TryDump(submissionsMax.State,streams) - do! Async.Sleep pumpDelayMs } - - /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes - static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval) = - let instance = new Engine<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval = statsInterval) - Async.Start <| instance.Pump() - instance - - /// Awaits space in `read` to limit reading ahead - yields (used,maximum) counts from Read Semaphore for logging purposes - member __.Submit(content : Message) = async { - do! readMax.Await() - match content with - | Message.Batch (seriesId, epoch, markBatchCompleted, events) -> - work.Enqueue <| Batch (seriesId, epoch, markBatchCompleted, events) - // NB readMax.Release() is effected in the Batch handler's MarkCompleted() - | Message.EndOfSeries seriesId -> - work.Enqueue <| CloseSeries seriesId - readMax.Release() - return readMax.State } - - /// As range assignments get revoked, a user is expected to `Stop `the active processing thread for the Ingester before releasing references to it - member __.Stop() = cts.Cancel() - -type Ingester = - /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing - static member Start<'R>(log, scheduler, maxRead, maxSubmissions, ?statsInterval) = - let singleSeriesIndex = 0 - let instance = Ingestion.Engine<'R>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) - { new IIngester with - member __.Submit(epoch, markCompleted, items) : Async = - instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) - member __.Stop() = __.Stop() } \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection/State.fs b/equinox-projector/Equinox.Projection/State.fs deleted file mode 100644 index d23e92caf..000000000 --- a/equinox-projector/Equinox.Projection/State.fs +++ /dev/null @@ -1,162 +0,0 @@ -module Equinox.Projection.State - -open Serilog -open System.Collections.Generic - -let arrayBytes (x:byte[]) = if x = null then 0 else x.Length -let inline eventSize (x : Equinox.Codec.IEvent<_>) = arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16 -let mb x = float x / 1024. / 1024. -let category (streamName : string) = streamName.Split([|'-'|],2).[0] - -type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } -type [] StreamSpan = { stream: string; span: Span } -type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with - member __.Size = - if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy eventSize - member __.IsReady = - if __.queue = null || __.isMalformed then false - else - match __.write, Array.tryHead __.queue with - | Some w, Some { index = i } -> i = w - | None, _ -> true - | _ -> false - -module StreamState = - let (|NNA|) xs = if xs = null then Array.empty else xs - module Span = - let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength - let trim min = function - | x when x.index >= min -> x // don't adjust if min not within - | End n when n < min -> { index = min; events = [||] } // throw away if before min - | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice - let merge min (xs : Span seq) = - let xs = - seq { for x in xs -> { x with events = (|NNA|) x.events } } - |> Seq.map (trim min) - |> Seq.filter (fun x -> x.events.Length <> 0) - |> Seq.sortBy (fun x -> x.index) - let buffer = ResizeArray() - let mutable curr = None - for x in xs do - match curr, x with - // Not overlapping, no data buffered -> buffer - | None, _ -> - curr <- Some x - // Gap - | Some (End nextIndex as c), x when x.index > nextIndex -> - buffer.Add c - curr <- Some x - // Overlapping, join - | Some (End nextIndex as c), x -> - curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } - curr |> Option.iter buffer.Add - if buffer.Count = 0 then null else buffer.ToArray() - - let inline optionCombine f (r1: int64 option) (r2: int64 option) = - match r1, r2 with - | Some x, Some y -> f x y |> Some - | None, None -> None - | None, x | x, None -> x - let combine (s1: StreamState) (s2: StreamState) : StreamState = - let writePos = optionCombine max s1.write s2.write - let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 - { write = writePos; queue = Span.merge (defaultArg writePos 0L) items; isMalformed = s1.isMalformed || s2.isMalformed } - -/// Gathers stats relating to how many items of a given category have been observed -type CatStats() = - let cats = Dictionary() - member __.Ingest(cat,?weight) = - let weight = defaultArg weight 1L - match cats.TryGetValue cat with - | true, catCount -> cats.[cat] <- catCount + weight - | false, _ -> cats.[cat] <- weight - member __.Any = cats.Count <> 0 - member __.Clear() = cats.Clear() - member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd - -type StreamStates() = - let mutable streams = Set.empty - let states = Dictionary() - let update stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - streams <- streams.Add stream - stream, state - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - stream, updated - let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } - let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore - - let busy = HashSet() - let pending (requestedOrder : string seq) = seq { - for s in requestedOrder do - let state = states.[s] - if state.IsReady && not (busy.Contains s) then - yield state.write, { stream = s; span = state.queue.[0] } - // [lazily] Slipstream in futher events that have been posted to streams which we've already visited - for KeyValue(s,v) in states do - if v.IsReady && not (busy.Contains s) then - yield v.write, { stream = s; span = v.queue.[0] } } - let markBusy stream = busy.Add stream |> ignore - let markNotBusy stream = busy.Remove stream |> ignore - - // Result is intentionally a thread-safe persisent data structure - // This enables the (potentially multiple) Ingesters to determine streams (for which they potentially have successor events) that are in play - // Ingesters then supply these 'preview events' in advance of the processing being scheduled - // This enables the projection logic to roll future work into the current work in the interests of medium term throughput - member __.All = streams - member __.InternalMerge(stream, state) = update stream state |> ignore - member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } - member __.Add(stream, index, event, ?isMalformed) = - updateWritePos stream (defaultArg isMalformed false) None [| { index = index; events = [| event |] } |] - member __.Add(batch: StreamSpan, isMalformed) = - updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] - member __.SetMalformed(stream,isMalformed) = - updateWritePos stream isMalformed None [| { index = 0L; events = null } |] - member __.QueueWeight(stream) = - states.[stream].queue.[0].events |> Seq.sumBy eventSize - member __.MarkBusy stream = - markBusy stream - member __.MarkCompleted(stream, index) = - markNotBusy stream - markCompleted stream index - member __.MarkFailed stream = - markNotBusy stream - member __.Pending(byQueuedPriority : string seq) : (int64 option * StreamSpan) seq = - pending byQueuedPriority - member __.Dump(log : ILogger) = - let mutable busyCount, busyB, ready, readyB, unprefixed, unprefixedB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0 - let busyCats, readyCats, readyStreams, unprefixedStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats(), CatStats() - let kb sz = (sz + 512L) / 1024L - for KeyValue (stream,state) in states do - match int64 state.Size with - | 0L -> - synced <- synced + 1 - | sz when busy.Contains stream -> - busyCats.Ingest(category stream) - busyCount <- busyCount + 1 - busyB <- busyB + sz - | sz when state.isMalformed -> - malformedStreams.Ingest(stream, mb sz |> int64) - malformed <- malformed + 1 - malformedB <- malformedB + sz - | sz when not state.IsReady -> - unprefixedStreams.Ingest(stream, mb sz |> int64) - unprefixed <- unprefixed + 1 - unprefixedB <- unprefixedB + sz - | sz -> - readyCats.Ingest(category stream) - readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) - ready <- ready + 1 - readyB <- readyB + sz - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", - synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) - if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - if unprefixedStreams.Any then log.Information("Waiting Streams, KB {missingStreams}", Seq.truncate 3 unprefixedStreams.StatsDescending) - if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) \ No newline at end of file diff --git a/equinox-projector/Projector.Tests/ProgressTests.fs b/equinox-projector/Projector.Tests/ProgressTests.fs deleted file mode 100644 index 67677101b..000000000 --- a/equinox-projector/Projector.Tests/ProgressTests.fs +++ /dev/null @@ -1,41 +0,0 @@ -module ProgressTests - -open ProjectorTemplate.Projector.State - -open Swensen.Unquote -open Xunit -open System.Collections.Generic - -let mkDictionary xs = Dictionary(dict xs) - -let [] ``Empty has zero streams pending or progress to write`` () = - let sut = ProgressState<_>() - let validatedPos, batches = sut.Validate(fun _ -> None) - None =! validatedPos - 0 =! batches - -let [] ``Can add multiple batches`` () = - let sut = ProgressState<_>() - sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) - sut.AppendBatch(1,mkDictionary ["b",2L; "c",3L]) - let validatedPos, batches = sut.Validate(fun _ -> None) - None =! validatedPos - 2 =! batches - -let [] ``Marking Progress Removes batches and updates progress`` () = - let sut = ProgressState<_>() - sut.AppendBatch(0,mkDictionary ["a",1L; "b",2L]) - sut.MarkStreamProgress("a",1L) |> ignore - sut.MarkStreamProgress("b",1L) |> ignore - let validatedPos, batches = sut.Validate(fun _ -> None) - None =! validatedPos - 1 =! batches - -let [] ``Marking progress is not persistent`` () = - let sut = ProgressState<_>() - sut.AppendBatch(0, mkDictionary ["a",1L]) - sut.MarkStreamProgress("a",2L) |> ignore - sut.AppendBatch(1, mkDictionary ["a",1L; "b",2L]) - let validatedPos, batches = sut.Validate(fun _ -> None) - Some 0 =! validatedPos - 1 =! batches \ No newline at end of file diff --git a/equinox-projector/Projector.Tests/Projector.Tests.fsproj b/equinox-projector/Projector.Tests/Projector.Tests.fsproj deleted file mode 100644 index 954aa11db..000000000 --- a/equinox-projector/Projector.Tests/Projector.Tests.fsproj +++ /dev/null @@ -1,26 +0,0 @@ - - - - netcoreapp2.2 - - false - false - - - - - - - - - - - - - - - - - - - diff --git a/equinox-projector/Projector.Tests/StreamStateTests.fs b/equinox-projector/Projector.Tests/StreamStateTests.fs deleted file mode 100644 index 8d35d267f..000000000 --- a/equinox-projector/Projector.Tests/StreamStateTests.fs +++ /dev/null @@ -1,78 +0,0 @@ -module CosmosIngesterTests - -open ProjectorTemplate.Projector.State -open Swensen.Unquote -open Xunit - -let canonicalTime = System.DateTimeOffset.UtcNow -let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } -let mergeSpans = StreamState.Span.merge -let trimSpans = StreamState.Span.trim - -let [] ``nothing`` () = - let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] - r =! null - -let [] ``synced`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] - r =! null - -let [] ``no overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] - r =! [| mk 0L 1; mk 2L 2 |] - -let [] ``overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 0L 2 |] - -let [] ``remove nulls`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 1L 1 |] - -let [] ``adjacent`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 0L 3 |] - -let [] ``adjacent to min`` () = - let r = List.map (trimSpans 2L) [ mk 0L 1; mk 1L 2 ] - r =! [ mk 2L 0; mk 2L 1 ] - -let [] ``adjacent to min merge`` () = - let r = mergeSpans 2L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 2L 1 |] - -let [] ``adjacent to min no overlap`` () = - let r = mergeSpans 2L [ mk 0L 1; mk 2L 1 ] - r =! [| mk 2L 1|] - -let [] ``adjacent trim`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2 ] - r =! [ mk 1L 1; mk 2L 2 ] - -let [] ``adjacent trim merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] - r =! [| mk 1L 3 |] - -let [] ``adjacent trim append`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [ mk 1L 1; mk 2L 2; mk 5L 1 ] - -let [] ``adjacent trim append merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``mixed adjacent trim append`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 5L 1; mk 2L 2] - r =! [ mk 1L 1; mk 5L 1; mk 2L 2 ] - -let [] ``mixed adjacent trim append merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``fail`` () = - let r = mergeSpans 11614L [ {index=11614L; events=null}; mk 11614L 1 ] - r =! [| mk 11614L 1 |] - -let [] ``fail 2`` () = - let r = mergeSpans 11613L [ mk 11614L 1; {index=11614L; events=null} ] - r =! [| mk 11614L 1 |] diff --git a/equinox-projector/Projector/Infrastructure.fs b/equinox-projector/Projector/Infrastructure.fs index 5fdef5dcb..a44109769 100644 --- a/equinox-projector/Projector/Infrastructure.fs +++ b/equinox-projector/Projector/Infrastructure.fs @@ -17,23 +17,10 @@ type Async with let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore and d : IDisposable = Console.CancelKeyPress.Subscribe callback in ()) - static member AwaitTaskCorrect (task : Task<'T>) : Async<'T> = - Async.FromContinuations <| fun (k,ek,_) -> - task.ContinueWith (fun (t:Task<'T>) -> - if t.IsFaulted then - let e = t.Exception - if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] - else ek e - elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) - elif t.IsCompleted then k t.Result - else ek(Exception "invalid Task state!")) - |> ignore -type SemaphoreSlim with - /// F# friendly semaphore await function - member semaphore.Await(?timeout : TimeSpan) = async { - let! ct = Async.CancellationToken - let timeout = defaultArg timeout Timeout.InfiniteTimeSpan - let task = semaphore.WaitAsync(timeout, ct) - return! Async.AwaitTaskCorrect task - } \ No newline at end of file +// TODO remove when using 2.0.0-preview7 +open Equinox.Projection.Codec + +module RenderedEvent = + let ofStreamItem (x: Equinox.Projection.StreamItem) : RenderedEvent = + { s = x.stream; i = x.index; c = x.event.EventType; t = x.event.Timestamp; d = x.event.Data; m = x.event.Meta } \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index aa1d78820..7aa095eb6 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -159,34 +159,32 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = CompressionType.Lz4) let producer = KafkaProducer.Create(Log.Logger, cfg, topic) let disposeProducer = (producer :> IDisposable).Dispose - let projectBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { + let ingest (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let toKafkaEvent (e: DocumentParser.IEvent) : RenderedEvent = { s = e.Stream; i = e.Index; c = e.EventType; t = e.Timestamp; d = e.Data; m = e.Meta } - let pt,events = (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> Seq.map toKafkaEvent |> Array.ofSeq) |> Stopwatch.Time + let pt,events = (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> Seq.map RenderedEvent.ofStreamItem |> Array.ofSeq) |> Stopwatch.Time let es = [| for e in events -> e.s, JsonConvert.SerializeObject e |] let! et,() = async { let! _ = producer.ProduceBatch es - do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } |> Stopwatch.Time + return! ctx.Checkpoint() |> Stopwatch.Time } log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - ChangeFeedObserver.Create(log, projectBatch, dispose = disposeProducer) + ChangeFeedObserver.Create(log, ingest, dispose = disposeProducer) //#else let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) = let projectionEngine = Projector.Start (log, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) fun () -> let mutable rangeIngester = Unchecked.defaultof - let init rangeLog = rangeIngester <- Ingester.Start (rangeLog, projectionEngine, maxPendingBatches, processorDop, TimeSpan.FromMinutes 1.) + let init rangeLog = async { rangeIngester <- Ingester.Start (rangeLog, projectionEngine, maxPendingBatches, processorDop, TimeSpan.FromMinutes 1.) } let ingest epoch checkpoint docs = - let events = Seq.collect DocumentParser.enumEvents docs - let items = seq { for x in events -> { stream = x.Stream; index = x.Index; event = x } } + let items = Seq.collect DocumentParser.enumEvents docs rangeIngester.Submit(epoch, checkpoint, items) let dispose () = rangeIngester.Stop() let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok - let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { + let ingest (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved @@ -197,7 +195,7 @@ let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) let e = pt.Elapsed in e.TotalSeconds, cur, max) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) + ChangeFeedObserver.Create(log, ingest, assign=init, dispose=dispose) //#endif // Illustrates how to emit direct to the Console using Serilog diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 0461755b0..14ea9d861 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -14,16 +14,12 @@ + - - + + - - - - - \ No newline at end of file diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index 5ac584d60..fc5d3546d 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -12,16 +12,8 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consumer.fsproj", "{7ED94D2B-1744-48A0-9B20-94E4777617E9}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection", "Equinox.Projection\Equinox.Projection.fsproj", "{C0EDE411-0BF1-417E-8B17-806F55BE01FE}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Tests", "Equinox.Projection.Tests\Equinox.Projection.Tests.fsproj", "{D8C4B963-1415-4711-A585-80BA2BFC9010}" -EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "..\equinox-sync\Sync\Sync.fsproj", "{C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Projection.Codec", "Equinox.Projection.Codec\Equinox.Projection.Codec.fsproj", "{AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Equinox.Cosmos.ProjectionEx", "Equinox.Projection.Cosmos\Equinox.Cosmos.ProjectionEx.fsproj", "{2071A2C9-B5C8-4143-B437-6833666D0ACA}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -36,26 +28,10 @@ Global {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.Build.0 = Debug|Any CPU {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.ActiveCfg = Release|Any CPU {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.Build.0 = Release|Any CPU - {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Debug|Any CPU.Build.0 = Debug|Any CPU - {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Release|Any CPU.ActiveCfg = Release|Any CPU - {C0EDE411-0BF1-417E-8B17-806F55BE01FE}.Release|Any CPU.Build.0 = Release|Any CPU - {D8C4B963-1415-4711-A585-80BA2BFC9010}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {D8C4B963-1415-4711-A585-80BA2BFC9010}.Debug|Any CPU.Build.0 = Debug|Any CPU - {D8C4B963-1415-4711-A585-80BA2BFC9010}.Release|Any CPU.ActiveCfg = Release|Any CPU - {D8C4B963-1415-4711-A585-80BA2BFC9010}.Release|Any CPU.Build.0 = Release|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.Build.0 = Debug|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.ActiveCfg = Release|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.Build.0 = Release|Any CPU - {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AA48D9FA-EBFA-4BE5-800B-0F364AF1295F}.Release|Any CPU.Build.0 = Release|Any CPU - {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Debug|Any CPU.Build.0 = Debug|Any CPU - {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Release|Any CPU.ActiveCfg = Release|Any CPU - {2071A2C9-B5C8-4143-B437-6833666D0ACA}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs similarity index 100% rename from equinox-projector/Equinox.Projection.Cosmos/CosmosIngester.fs rename to equinox-sync/Sync/CosmosIngester.fs diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index ac9efd4df..ba19cab99 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -15,7 +15,7 @@ let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Cosmo let cosmosIngester = CosmosIngester.start (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) fun () -> let mutable rangeIngester = Unchecked.defaultof<_> - let init rangeLog = rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) + let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) } let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq in rangeIngester.Submit(epoch, checkpoint, events) let dispose () = rangeIngester.Stop () let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok @@ -94,7 +94,7 @@ let transformV0 catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { //#else let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq = seq { for e in DocumentParser.enumEvents changeFeedDocument do - if catFilter (category e.Stream) then // TODO yield parsed - // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level - yield { stream = e.Stream; index = e.Index; event = e } } + // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level + if catFilter (category e.stream) then + yield e } //#endif \ No newline at end of file diff --git a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs b/equinox-sync/Sync/Metrics.fs similarity index 98% rename from equinox-projector/Equinox.Projection.Cosmos/Metrics.fs rename to equinox-sync/Sync/Metrics.fs index 48c5d40ac..9d6d97a97 100644 --- a/equinox-projector/Equinox.Projection.Cosmos/Metrics.fs +++ b/equinox-sync/Sync/Metrics.fs @@ -65,9 +65,9 @@ let dumpRuStats duration (log: Serilog.ILogger) = totalRc <- totalRc + ru totalMs <- totalMs + stat.ms logActivity name stat.count ru stat.ms - logActivity "TOTAL" totalCount totalRc totalMs - // Yes, there's a minor race here! + // Yes, there's a minor race here between the capture and reset RuCounters.RuCounterSink.Reset() + logActivity "TOTAL" totalCount totalRc totalMs let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 1a770ddca..98d297ccc 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -9,6 +9,8 @@ + + @@ -17,8 +19,8 @@ - - + + @@ -26,8 +28,4 @@ - - - - \ No newline at end of file From 6152cd31c9ddf5db1cf011c138c9a238ac7555d8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 6 May 2019 10:50:38 +0100 Subject: [PATCH 219/353] Remove tests --- .../Sync.Tests/CosmosIngesterTests.fs | 78 ------------------- equinox-sync/Sync.Tests/ProgressTests.fs | 38 --------- equinox-sync/Sync.Tests/Sync.Tests.fsproj | 25 ------ equinox-sync/equinox-sync.sln | 12 --- 4 files changed, 153 deletions(-) delete mode 100644 equinox-sync/Sync.Tests/CosmosIngesterTests.fs delete mode 100644 equinox-sync/Sync.Tests/ProgressTests.fs delete mode 100644 equinox-sync/Sync.Tests/Sync.Tests.fsproj diff --git a/equinox-sync/Sync.Tests/CosmosIngesterTests.fs b/equinox-sync/Sync.Tests/CosmosIngesterTests.fs deleted file mode 100644 index 40ec7fe37..000000000 --- a/equinox-sync/Sync.Tests/CosmosIngesterTests.fs +++ /dev/null @@ -1,78 +0,0 @@ -module SyncTemplate.Tests.CosmosIngesterTests - -open SyncTemplate.CosmosIngester -open Swensen.Unquote -open Xunit - -let canonicalTime = System.DateTimeOffset.UtcNow -let mk p c : Span = { index = p; events = [| for x in 0..c-1 -> Equinox.Codec.Core.EventData.Create(p + int64 x |> string, null, timestamp=canonicalTime) |] } -let mergeSpans = StreamState.Span.merge -let trimSpans = StreamState.Span.trim - -let [] ``nothing`` () = - let r = mergeSpans 0L [ mk 0L 0; mk 0L 0 ] - r =! null - -let [] ``synced`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 0 ] - r =! null - -let [] ``no overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 2L 2 ] - r =! [| mk 0L 1; mk 2L 2 |] - -let [] ``overlap`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 0L 2 |] - -let [] ``remove nulls`` () = - let r = mergeSpans 1L [ mk 0L 1; mk 0L 2 ] - r =! [| mk 1L 1 |] - -let [] ``adjacent`` () = - let r = mergeSpans 0L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 0L 3 |] - -let [] ``adjacent to min`` () = - let r = List.map (trimSpans 2L) [ mk 0L 1; mk 1L 2 ] - r =! [ mk 2L 0; mk 2L 1 ] - -let [] ``adjacent to min merge`` () = - let r = mergeSpans 2L [ mk 0L 1; mk 1L 2 ] - r =! [| mk 2L 1 |] - -let [] ``adjacent to min no overlap`` () = - let r = mergeSpans 2L [ mk 0L 1; mk 2L 1 ] - r =! [| mk 2L 1|] - -let [] ``adjacent trim`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2 ] - r =! [ mk 1L 1; mk 2L 2 ] - -let [] ``adjacent trim merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2 ] - r =! [| mk 1L 3 |] - -let [] ``adjacent trim append`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [ mk 1L 1; mk 2L 2; mk 5L 1 ] - -let [] ``adjacent trim append merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 2L 2; mk 5L 1] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``mixed adjacent trim append`` () = - let r = List.map (trimSpans 1L) [ mk 0L 2; mk 5L 1; mk 2L 2] - r =! [ mk 1L 1; mk 5L 1; mk 2L 2 ] - -let [] ``mixed adjacent trim append merge`` () = - let r = mergeSpans 1L [ mk 0L 2; mk 5L 1; mk 2L 2] - r =! [| mk 1L 3; mk 5L 1 |] - -let [] ``fail`` () = - let r = mergeSpans 11614L [ {index=11614L; events=null}; mk 11614L 1 ] - r =! [| mk 11614L 1 |] - -let [] ``fail 2`` () = - let r = mergeSpans 11613L [ mk 11614L 1; {index=11614L; events=null} ] - r =! [| mk 11614L 1 |] \ No newline at end of file diff --git a/equinox-sync/Sync.Tests/ProgressTests.fs b/equinox-sync/Sync.Tests/ProgressTests.fs deleted file mode 100644 index 2cb9024a7..000000000 --- a/equinox-sync/Sync.Tests/ProgressTests.fs +++ /dev/null @@ -1,38 +0,0 @@ -module SyncTemplate.Tests.ProgressTests - -open SyncTemplate - -open Swensen.Unquote -open Xunit - -let [] ``Empty has zero streams pending or progress to write`` () = - let sut = ProgressBatcher.State<_>() - let validatedPos, batches = sut.Validate(fun _ -> None) - None =! validatedPos - 0 =! batches - -let [] ``Can add multiple batches`` () = - let sut = ProgressBatcher.State<_>() - sut.AppendBatch(0,["a",1L; "b",2L]) - sut.AppendBatch(1,["b",2L; "c",3L]) - let validatedPos, batches = sut.Validate(fun _ -> None) - None =! validatedPos - 2 =! batches - -let [] ``Marking Progress Removes batches and updates progress`` () = - let sut = ProgressBatcher.State<_>() - sut.AppendBatch(0,["a",1L; "b",2L]) - sut.MarkStreamProgress("a",1L) - sut.MarkStreamProgress("b",1L) - let validatedPos, batches = sut.Validate(fun _ -> None) - None =! validatedPos - 1 =! batches - -let [] ``Marking progress is not persistent`` () = - let sut = ProgressBatcher.State<_>() - sut.AppendBatch(0,["a",1L]) - sut.MarkStreamProgress("a",2L) - sut.AppendBatch(1,["a",1L; "b",2L]) - let validatedPos, batches = sut.Validate(fun _ -> None) - Some 0 =! validatedPos - 1 =! batches \ No newline at end of file diff --git a/equinox-sync/Sync.Tests/Sync.Tests.fsproj b/equinox-sync/Sync.Tests/Sync.Tests.fsproj deleted file mode 100644 index 2f44b6f9c..000000000 --- a/equinox-sync/Sync.Tests/Sync.Tests.fsproj +++ /dev/null @@ -1,25 +0,0 @@ - - - - netcoreapp2.1 - - false - - - - - - - - - - - - - - - - - - - diff --git a/equinox-sync/equinox-sync.sln b/equinox-sync/equinox-sync.sln index be656b901..0f752a0ff 100644 --- a/equinox-sync/equinox-sync.sln +++ b/equinox-sync/equinox-sync.sln @@ -9,10 +9,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync.Tests", "Sync.Tests\Sync.Tests.fsproj", "{1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Ingest", "Ingest\Ingest.fsproj", "{BB7079A7-53E8-4843-8981-78DD025F8C91}" -EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -23,14 +19,6 @@ Global {EB9D803D-2E0C-437E-9282-8E29E479F066}.Debug|Any CPU.Build.0 = Debug|Any CPU {EB9D803D-2E0C-437E-9282-8E29E479F066}.Release|Any CPU.ActiveCfg = Release|Any CPU {EB9D803D-2E0C-437E-9282-8E29E479F066}.Release|Any CPU.Build.0 = Release|Any CPU - {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Debug|Any CPU.Build.0 = Debug|Any CPU - {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Release|Any CPU.ActiveCfg = Release|Any CPU - {1A5997B1-48F3-43FC-B5AD-661EF4B8B15D}.Release|Any CPU.Build.0 = Release|Any CPU - {BB7079A7-53E8-4843-8981-78DD025F8C91}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {BB7079A7-53E8-4843-8981-78DD025F8C91}.Debug|Any CPU.Build.0 = Debug|Any CPU - {BB7079A7-53E8-4843-8981-78DD025F8C91}.Release|Any CPU.ActiveCfg = Release|Any CPU - {BB7079A7-53E8-4843-8981-78DD025F8C91}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 15a70f56179d3a8d8c32c990196f51b4e35f7072 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 6 May 2019 10:55:11 +0100 Subject: [PATCH 220/353] remove redundant metrics --- equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/Metrics.fs | 73 ----------------------------- equinox-sync/Sync/Program.fs | 2 +- equinox-sync/Sync/Sync.fsproj | 1 - 4 files changed, 2 insertions(+), 76 deletions(-) delete mode 100644 equinox-sync/Sync/Metrics.fs diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 285921220..271d5a3f3 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -78,7 +78,7 @@ type Stats(log : ILogger, statsInterval) = !rateLimited, !timedOut, !tooLarge, !malformed, !resultExnOther) rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0 if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() - Equinox.Cosmos.Metrics.dumpRuStats statsInterval log + Equinox.Cosmos.Store.Log.InternalMetrics.dump log override __.Handle message = base.Handle message diff --git a/equinox-sync/Sync/Metrics.fs b/equinox-sync/Sync/Metrics.fs deleted file mode 100644 index 9d6d97a97..000000000 --- a/equinox-sync/Sync/Metrics.fs +++ /dev/null @@ -1,73 +0,0 @@ -module Equinox.Cosmos.Metrics - -// TODO move into equinox.cosmos - -open System - -module RuCounters = - open Equinox.Cosmos.Store - open Serilog.Events - - let inline (|Stats|) ({ interval = i; ru = ru }: Log.Measurement) = ru, let e = i.Elapsed in int64 e.TotalMilliseconds - - let (|CosmosReadRc|CosmosWriteRc|CosmosResyncRc|CosmosResponseRc|) = function - | Log.Tip (Stats s) - | Log.TipNotFound (Stats s) - | Log.TipNotModified (Stats s) - | Log.Query (_,_, (Stats s)) -> CosmosReadRc s - // slices are rolled up into batches so be sure not to double-count - | Log.Response (_,(Stats s)) -> CosmosResponseRc s - | Log.SyncSuccess (Stats s) - | Log.SyncConflict (Stats s) -> CosmosWriteRc s - | Log.SyncResync (Stats s) -> CosmosResyncRc s - let (|SerilogScalar|_|) : LogEventPropertyValue -> obj option = function - | (:? ScalarValue as x) -> Some x.Value - | _ -> None - let (|CosmosMetric|_|) (logEvent : LogEvent) : Log.Event option = - match logEvent.Properties.TryGetValue("cosmosEvt") with - | true, SerilogScalar (:? Log.Event as e) -> Some e - | _ -> None - type RuCounter = - { mutable rux100: int64; mutable count: int64; mutable ms: int64 } - static member Create() = { rux100 = 0L; count = 0L; ms = 0L } - member __.Ingest (ru, ms) = - System.Threading.Interlocked.Increment(&__.count) |> ignore - System.Threading.Interlocked.Add(&__.rux100, int64 (ru*100.)) |> ignore - System.Threading.Interlocked.Add(&__.ms, ms) |> ignore - type RuCounterSink() = - static member val Read = RuCounter.Create() with get, set - static member val Write = RuCounter.Create() with get, set - static member val Resync = RuCounter.Create() with get, set - static member Reset() = - RuCounterSink.Read <- RuCounter.Create() - RuCounterSink.Write <- RuCounter.Create() - RuCounterSink.Resync <- RuCounter.Create() - interface Serilog.Core.ILogEventSink with - member __.Emit logEvent = logEvent |> function - | CosmosMetric (CosmosReadRc stats) -> RuCounterSink.Read.Ingest stats - | CosmosMetric (CosmosWriteRc stats) -> RuCounterSink.Write.Ingest stats - | CosmosMetric (CosmosResyncRc stats) -> RuCounterSink.Resync.Ingest stats - | _ -> () - -let dumpRuStats duration (log: Serilog.ILogger) = - let stats = - [ "Read", RuCounters.RuCounterSink.Read - "Write", RuCounters.RuCounterSink.Write - "Resync", RuCounters.RuCounterSink.Resync ] - let mutable totalCount, totalRc, totalMs = 0L, 0., 0L - let logActivity name count rc lat = - if count <> 0L then - log.Information("{name}: {count:n0} requests costing {ru:n0} RU (average: {avg:n2}); Average latency: {lat:n0}ms", - name, count, rc, (if count = 0L then Double.NaN else rc/float count), (if count = 0L then Double.NaN else float lat/float count)) - for name, stat in stats do - let ru = float stat.rux100 / 100. - totalCount <- totalCount + stat.count - totalRc <- totalRc + ru - totalMs <- totalMs + stat.ms - logActivity name stat.count ru stat.ms - // Yes, there's a minor race here between the capture and reset - RuCounters.RuCounterSink.Reset() - logActivity "TOTAL" totalCount totalRc totalMs - let measures : (string * (TimeSpan -> float)) list = [ "s", fun x -> x.TotalSeconds(*; "m", fun x -> x.TotalMinutes; "h", fun x -> x.TotalHours*) ] - let logPeriodicRate name count ru = log.Information("rp{name} {count:n0} = ~{ru:n0} RU", name, count, ru) - for uom, f in measures do let d = f duration in if d <> 0. then logPeriodicRate uom (float totalCount/d |> int64) (totalRc/d) \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index b45b49312..3ffb8561d 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -290,7 +290,7 @@ module Logging = |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let configure (a : Configuration.LoggerSinkConfiguration) : unit = a.Logger(fun l -> - l.WriteTo.Sink(Metrics.RuCounters.RuCounterSink()) |> ignore) |> ignore + l.WriteTo.Sink(Equinox.Cosmos.Store.Log.InternalMetrics.RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke let isWriter = Filters.Matching.FromSource().Invoke diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 98d297ccc..5ed13dca6 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -9,7 +9,6 @@ - From c97311ed40a93530f64d0802beafd34055454679 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 7 May 2019 09:29:28 +0100 Subject: [PATCH 221/353] Fix filter --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 3ffb8561d..12ff159ab 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -352,7 +352,7 @@ let main argv = || e.EventStreamId.StartsWith "Inventory-" // Too long || e.EventStreamId.StartsWith "InventoryCount-" // No Longer used || e.EventStreamId.StartsWith "InventoryLog" // 5GB, causes lopsided partitions, unused - //|| e.EventStreamId = "ReloadBatchId" // does not start at 0 + || e.EventStreamId = "SkuFileUpload-534e4362c641461ca27e3d23547f0852" || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some From 35c480fe9d265c043dfad9575142141a9e118299 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 7 May 2019 10:29:19 +0100 Subject: [PATCH 222/353] Another blacklist --- equinox-sync/Sync/Program.fs | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 12ff159ab..7bb525f7a 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -353,6 +353,7 @@ let main argv = || e.EventStreamId.StartsWith "InventoryCount-" // No Longer used || e.EventStreamId.StartsWith "InventoryLog" // 5GB, causes lopsided partitions, unused || e.EventStreamId = "SkuFileUpload-534e4362c641461ca27e3d23547f0852" + || e.EventStreamId = "SkuFileUpload-778f1efeab214f5bab2860d1f802ef24" || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some From 243d0dfbadc0f5510208dc558b0a5994a2e684c1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 7 May 2019 11:36:13 +0100 Subject: [PATCH 223/353] Tweaks --- equinox-sync/Sync/CosmosSource.fs | 2 +- equinox-sync/Sync/Program.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index ba19cab99..7e2e5722d 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -27,7 +27,7 @@ let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Cosmo let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., - let e = pt.Elapsed in e.TotalSeconds, cur, max) + (let e = pt.Elapsed in e.TotalSeconds), cur, max) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 7bb525f7a..dd84ac8bf 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -23,7 +23,7 @@ module CmdParser = | [] ConsumerGroupName of string | [] LocalSeq | [] Verbose - | [] FromTail + | [] FromTail | [] BatchSize of int | [] MaxPendingBatches of int | [] MaxProcessing of int From 29c2929b358c06d09c22f0bcf481c7104a2b3d9d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 7 May 2019 18:37:28 +0100 Subject: [PATCH 224/353] Consistency tweaks --- equinox-projector/Projector/Program.fs | 2 +- equinox-sync/Sync/CosmosSource.fs | 6 +++--- equinox-sync/Sync/Program.fs | 14 +++++++------- equinox-sync/Sync/Sync.fsproj | 1 + 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 88e6bc4b2..48bd51420 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -95,7 +95,7 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LeaseCollectionSuffix _ -> "specify Collection Name suffix for Leases collection (default: `-aux`)." | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." - | MaxDocuments _ -> "maxiumum document count to supply for the Change Feed query. Default: ude response size limit" + | MaxDocuments _ -> "maxiumum document count to supply for the Change Feed query. Default: use response size limit" | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" | ProcessorDop _ -> "Maximum number of streams to process concurrently. Default: 64" | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 7e2e5722d..09b2ecf49 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -32,7 +32,7 @@ let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Cosmo } ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) -let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, forceSkip, batchSize, lagReportFreq : TimeSpan option) +let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maxDocuments, lagReportFreq : TimeSpan option) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { Log.Information("Lags {@rangeLags} (Range, Docs count)", remainingWork) @@ -40,8 +40,8 @@ let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = ChangeFeedProcessor.Start - ( Log.Logger, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = forceSkip, - cfBatchSize = batchSize, createObserver = createRangeProjector, ?reportLagAndAwaitNextEstimation = maybeLogLag) + ( Log.Logger, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, + createObserver = createRangeProjector, ?cfBatchSize = maxDocuments, ?reportLagAndAwaitNextEstimation = maybeLogLag) do! Async.AwaitKeyboardInterrupt() } //#if marveleqx diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index aa3e54a87..6a8290f4e 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -86,7 +86,7 @@ module CmdParser = #if cosmos member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName - member __.BatchSize = a.GetResult(BatchSize,1000) + member __.MaxDocuments = a.TryGetResult MaxDocuments member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds #else member __.VerboseConsole = a.Contains VerboseConsole @@ -111,10 +111,10 @@ module CmdParser = | Some sc, None -> x.Source.Discovery, { database = x.Source.Database; collection = sc } | None, Some dc -> x.Destination.Discovery, { database = x.Destination.Database; collection = dc } | Some _, Some _ -> raise (InvalidArguments "LeaseCollectionSource and LeaseCollectionDestination are mutually exclusive - can only store in one database") - Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} in batches of {batchSize}", x.LeaseId, db.database, db.collection, x.BatchSize) + Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} in batches of {batchSize}", x.LeaseId, db.database, db.collection, x.MaxDocuments) if a.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) - disco, db, x.LeaseId, x.StartFromHere, x.BatchSize, x.LagFrequency + disco, db, x.LeaseId, a.Contains FromTail, x.MaxDocuments, x.LagFrequency #else member x.BuildFeedParams() : EventStoreSource.ReaderSpec = let startPos = @@ -326,16 +326,16 @@ let main argv = let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, storeLog) #if cosmos let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() - let auxDiscovery, aux, leaseId, startFromHere, batchSize, lagFrequency = args.BuildChangeFeedParams() + let auxDiscovery, aux, leaseId, startFromHere, maxDocuments, lagFrequency = args.BuildChangeFeedParams() #if marveleqx - let createSyncHandler () = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformV0 catFilter) + let createSyncHandler = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformV0 catFilter) #else - let createSyncHandler () = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformOrFilter catFilter) + let createSyncHandler = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformOrFilter catFilter) // Uncomment to test marveleqx mode // let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformV0 catFilter) #endif CosmosSource.run (discovery, source) (auxDiscovery, aux) connectionPolicy - (leaseId, startFromHere, batchSize, lagFrequency) + (leaseId, startFromHere, maxDocuments, lagFrequency) createSyncHandler #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index f3391e235..9839c7557 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,6 +4,7 @@ Exe netcoreapp2.1 5 + $(DefineConstants);cosmos From 4dd3ac371f36d5120fa13f109e322404e2825932 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 7 May 2019 16:43:57 +0100 Subject: [PATCH 225/353] Update all to 2.0.0-preview6 --- equinox-projector/Projector/Program.fs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 48bd51420..caace8a44 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -121,8 +121,9 @@ module CmdParser = member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.AuxCollectionName = __.Cosmos.Collection + __.Suffix member x.BuildChangeFeedParams() = - Log.Information("Processing {leaseId} in {auxCollName} with max Documents {maxDocuments} (<= {maxPending} pending) using {dop} processors", - x.LeaseId, x.AuxCollectionName, x.MaxDocuments, x.MaxPendingBatches, x.ProcessorDop) + match x.MaxDocuments with + | None -> Log.Information("Processing {leaseId} in {auxCollName} without document count limit (<= {maxPending} pending) using {dop} processors", x.LeaseId, x.AuxCollectionName, x.MaxPendingBatches, x.ProcessorDop) + | Some lim -> Log.Information("Processing {leaseId} in {auxCollName} with max {changeFeedMaxDocuments} documents (<= {maxPending} pending) using {dop} processors", x.LeaseId, x.AuxCollectionName, x.MaxDocuments, x.MaxPendingBatches, x.ProcessorDop) if args.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, args.Contains FromTail, x.MaxDocuments, x.MaxPendingBatches, x.ProcessorDop, x.LagFrequency From a3496599809f46d43804c79da363c35273ba87fd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 7 May 2019 16:49:49 +0100 Subject: [PATCH 226/353] Correct eqxsync progress management, rebasing on Equinox.Projection pipeline (#21) --- equinox-sync/Sync/Program.fs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 6a8290f4e..b31a5c6dc 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -111,7 +111,7 @@ module CmdParser = | Some sc, None -> x.Source.Discovery, { database = x.Source.Database; collection = sc } | None, Some dc -> x.Destination.Discovery, { database = x.Destination.Database; collection = dc } | Some _, Some _ -> raise (InvalidArguments "LeaseCollectionSource and LeaseCollectionDestination are mutually exclusive - can only store in one database") - Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} in batches of {batchSize}", x.LeaseId, db.database, db.collection, x.MaxDocuments) + Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} with maximum document count limited to {maxDocuments}", x.LeaseId, db.database, db.collection, x.MaxDocuments) if a.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) disco, db, x.LeaseId, a.Contains FromTail, x.MaxDocuments, x.LagFrequency @@ -294,8 +294,10 @@ module Logging = a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke let isWriter = Filters.Matching.FromSource().Invoke - let isCheckpointing = Filters.Matching.FromSource().Invoke - (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCheckpointing x || isWriter x)) + let isCfp429a = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.LeaseManagement.DocumentServiceLeaseUpdater").Invoke + let isCfp429b = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.LeaseRenewer").Invoke + let isCfp429c = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.PartitionLoadBalancer").Invoke + (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isWriter x || isCfp429a x || isCfp429b x || isCfp429c x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) From cb0c2ddf20dcff49f01340c9a96dcb7354791a5e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 06:44:48 +0100 Subject: [PATCH 227/353] Tidy lag logging --- equinox-sync/Sync/CosmosSource.fs | 6 +++--- equinox-sync/Sync/Program.fs | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 09b2ecf49..ade5498b1 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -32,15 +32,15 @@ let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Cosmo } ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) -let run (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maxDocuments, lagReportFreq : TimeSpan option) +let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maxDocuments, lagReportFreq : TimeSpan option) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - Log.Information("Lags {@rangeLags} (Range, Docs count)", remainingWork) + log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = ChangeFeedProcessor.Start - ( Log.Logger, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, + ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, createObserver = createRangeProjector, ?cfBatchSize = maxDocuments, ?reportLagAndAwaitNextEstimation = maybeLogLag) do! Async.AwaitKeyboardInterrupt() } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index b31a5c6dc..dd5b88666 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -336,7 +336,7 @@ let main argv = // Uncomment to test marveleqx mode // let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformV0 catFilter) #endif - CosmosSource.run (discovery, source) (auxDiscovery, aux) connectionPolicy + CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) createSyncHandler #else From ae60dc92767d1c675c0744b0bc6aec7fdb59a5fd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 06:47:37 +0100 Subject: [PATCH 228/353] Add local copy of Projection[2] --- equinox-sync/Sync/CosmosIngester.fs | 2 + equinox-sync/Sync/CosmosSource.fs | 1 + equinox-sync/Sync/EventStoreSource.fs | 1 + equinox-sync/Sync/Projection2.fs | 481 ++++++++++++++++++++++++++ equinox-sync/Sync/Sync.fsproj | 1 + 5 files changed, 486 insertions(+) create mode 100644 equinox-sync/Sync/Projection2.fs diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index a23baf589..cbb423810 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -3,6 +3,8 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Store open Equinox.Projection.Scheduling +open Equinox.Projection2 +open Equinox.Projection2.Scheduling open Equinox.Projection.State open Serilog diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index ade5498b1..7489bbf95 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -3,6 +3,7 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Projection open Equinox.Projection +open Equinox.Projection2 open Equinox.Projection.State open Equinox.Store // AwaitTaskCorrect open Microsoft.Azure.Documents diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 0ce4e46dd..3cf602f16 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -3,6 +3,7 @@ //open Equinox.Cosmos.Projection open Equinox.Store // AwaitTaskCorrect open Equinox.Projection +open Equinox.Projection2 open EventStore.ClientAPI open Serilog // NB Needs to shadow ILogger open System diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs new file mode 100644 index 000000000..018e8dc66 --- /dev/null +++ b/equinox-sync/Sync/Projection2.fs @@ -0,0 +1,481 @@ +namespace Equinox.Projection2 + +open Equinox.Projection +open Equinox.Projection.State +open Serilog +open System +open System.Collections.Concurrent +open System.Collections.Generic +open System.Diagnostics +open System.Threading + +[] +module private Helpers = + let expiredMs ms = + let timer = Stopwatch.StartNew() + fun () -> + let due = timer.ElapsedMilliseconds > ms + if due then timer.Restart() + due + type Sem(max) = + let inner = new SemaphoreSlim(max) + member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore + member __.State = max-inner.CurrentCount,max + /// Wait infinitely to get the semaphore + member __.Await() = inner.Await() |> Async.Ignore + /// Wait for the specified timeout to acquire (or return false instantly) + member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) + member __.HasCapacity = inner.CurrentCount > 0 + member __.CurrentCapacity = inner.CurrentCount + +module Progress = + + type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } + + type State<'Pos>() = + let pending = Queue<_>() + let trim () = + while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do + let batch = pending.Dequeue() + batch.markCompleted() + member __.AppendBatch(markCompleted, reqs : Dictionary) = + pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } + trim () + member __.MarkStreamProgress(stream, index) = + for x in pending do + match x.streamToRequiredIndex.TryGetValue stream with + | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore + | _, _ -> () + trim () + member __.InScheduledOrder getStreamWeight = + let raw = seq { + let streams = HashSet() + let mutable batch = 0 + for x in pending do + batch <- batch + 1 + for s in x.streamToRequiredIndex.Keys do + if streams.Add s then + yield s,(batch,getStreamWeight s) } + raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst + + /// Manages writing of progress + /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) + /// - retries until success or a new item is posted + type Writer<'Res when 'Res: equality>() = + let pumpSleepMs = 100 + let due = expiredMs 5000L + let mutable committedEpoch = None + let mutable validatedPos = None + let result = Event>() + [] member __.Result = result.Publish + member __.Post(version,f) = + Volatile.Write(&validatedPos,Some (version,f)) + member __.CommittedEpoch = Volatile.Read(&committedEpoch) + member __.Pump() = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + match Volatile.Read &validatedPos with + | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> + try do! f + Volatile.Write(&committedEpoch, Some v) + result.Trigger (Choice1Of2 v) + with e -> result.Trigger (Choice2Of2 e) + | _ -> do! Async.Sleep pumpSleepMs } + +module Scheduling = + + /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners + [] + type InternalMessage<'R> = + /// Enqueue a batch of items with supplied progress marking function + | Add of markCompleted: (unit -> unit) * items: StreamItem[] + /// Stats per submitted batch for stats listeners to aggregate + | Added of streams: int * skip: int * events: int + /// Submit new data pertaining to a stream that has commenced processing + | AddActive of KeyValuePair[] + /// Result of processing on stream - result (with basic stats) or the `exn` encountered + | Result of stream: string * outcome: Choice<'R,exn> + + /// Gathers stats pertaining to the core projection/ingestion activity + type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = + let cycles, filled, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + let dumpStats capacity (used,maxDop) = + log.Information("Projection Cycles {cycles} Filled {filled:P0} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + !cycles, float !filled/float !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + cycles := 0; filled := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + abstract member Handle : InternalMessage<'R> -> unit + default __.Handle msg = msg |> function + | Add _ | AddActive _ -> () + | Added (streams, skipped, events) -> + incr batchesPended + streamsPended := !streamsPended + streams + eventsPended := !eventsPended + events + eventsSkipped := !eventsSkipped + skipped + | Result (_stream, Choice1Of2 _) -> + incr resultCompleted + | Result (_stream, Choice2Of2 _) -> + incr resultExn + member __.TryDump(wasFull,capacity,(used,max),streams : StreamStates) = + incr cycles + if wasFull then incr filled + if statsDue () then + dumpStats capacity (used,max) + __.DumpExtraStats() + streams.Dump log + /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) + abstract DumpExtraStats : unit -> unit + default __.DumpExtraStats () = () + + /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint + type Dispatcher<'R>(maxDop) = + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let result = Event<'R>() + let dop = new Sem(maxDop) + let dispatch work = async { + let! res = work + result.Trigger res + dop.Release() } + [] member __.Result = result.Publish + member __.HasCapacity = dop.HasCapacity + member __.CurrentCapacity = dop.CurrentCapacity + member __.State = dop.State + member __.TryAdd(item,?timeout) = async { + let! got = dop.TryAwait(?timeout=timeout) + if got then + work.Add(item) + return got } + member __.Pump () = async { + let! ct = Async.CancellationToken + for item in work.GetConsumingEnumerable ct do + Async.Start(dispatch item) } + + /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order + /// a) does not itself perform any reading activities + /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) + /// c) submits work to the supplied Dispatcher (which it triggers pumping of) + /// d) periodically reports state (with hooks for ingestion engines to report same) + type Engine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = + let sleepIntervalMs = 1 + let cts = new CancellationTokenSource() + let batches = Sem maxPendingBatches + let work = ConcurrentQueue>() + let streams = StreamStates() + let progressState = Progress.State() + + member private __.Pump(stats : Stats<'R>) = async { + use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) + Async.Start(dispatcher.Pump(), cts.Token) + let validVsSkip (streamState : StreamState) (item : StreamItem) = + match streamState.write, item.index + 1L with + | Some cw, required when cw >= required -> 0, 1 + | _ -> 1, 0 + let handle x = + match x with + | Add (releaseRead, items) -> + let reqs = Dictionary() + let mutable count, skipCount = 0, 0 + for item in items do + let stream,streamState = streams.Add(item.stream,item.index,item.event) + match validVsSkip streamState item with + | 0, skip -> + skipCount <- skipCount + skip + | required, _ -> + count <- count + required + reqs.[stream] <- item.index+1L + let markCompleted () = + releaseRead() + batches.Release() + progressState.AppendBatch(markCompleted,reqs) + work.Enqueue(Added (reqs.Count,skipCount,count)) + | AddActive events -> + for e in events do + streams.InternalMerge(e.Key,e.Value) + | Added _ -> + () + | Result (stream,r) -> + match interpretProgress streams stream r with + | Some index -> + progressState.MarkStreamProgress(stream,index) + streams.MarkCompleted(stream,index) + | None -> + streams.MarkFailed stream + + while not cts.IsCancellationRequested do + // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state + let mutable idle = true + work |> ConcurrentQueue.drain (fun x -> + handle x + stats.Handle x + idle <- false) + // 2. top up provisioning of writers queue + let capacity = dispatcher.CurrentCapacity + let mutable addsBeingAccepted = capacity <> 0 + if addsBeingAccepted then + let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight) + let xs = potential.GetEnumerator() + while xs.MoveNext() && addsBeingAccepted do + let (_,{stream = s} : StreamSpan) as item = xs.Current + let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) + if succeeded then streams.MarkBusy s + idle <- idle && not succeeded // any add makes it not idle + addsBeingAccepted <- succeeded + // 3. Periodically emit status info + stats.TryDump(not addsBeingAccepted,capacity,dispatcher.State,streams) + // 4. Do a minimal sleep so we don't run completely hot when empty + if idle then do! Async.Sleep sleepIntervalMs } + static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = + let dispatcher = Dispatcher(processorDop) + let instance = new Engine<'R>(maxPendingBatches, dispatcher, project, interpretProgress) + Async.Start <| instance.Pump(stats) + instance + + /// Attempt to feed in a batch (subject to there being capacity to do so) + member __.TrySubmit(markCompleted, events) = async { + let! got = batches.TryAwait() + if got then + work.Enqueue <| Add (markCompleted, events) + return got } + + member __.AddOpenStreamData(events) = + work.Enqueue <| AddActive events + + member __.AllStreams = streams.All + + member __.Stop() = + cts.Cancel() + +type Projector = + + static member Start(log, maxPendingBatches, maxActiveBatches, project : StreamSpan -> Async, ?statsInterval) = + let project (_maybeWritePos, batch) = async { + try let! count = project batch + return Choice1Of2 (batch.span.index + int64 count) + with e -> return Choice2Of2 e } + let interpretProgress _streams _stream = function + | Choice1Of2 index -> Some index + | Choice2Of2 _ -> None + let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + Scheduling.Engine.Start(stats, maxPendingBatches, maxActiveBatches, project, interpretProgress) + +module Ingestion = + + [] + type Message = + | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq + //| StreamSegment of span: StreamSpan + | EndOfSeries of seriesIndex: int + + type private Streams() = + let states = Dictionary() + let merge stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + + member __.Merge(items : StreamItem seq) = + for item in items do + merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = [| item.event |] } |] } + + member __.Take(processingContains) = + let forward = [| for x in states do if processingContains x.Key then yield x |] + for x in forward do states.Remove x.Key |> ignore + forward + + member __.Dump(log : ILogger) = + let mutable waiting, waitingB = 0, 0L + let waitingCats, waitingStreams = CatStats(), CatStats() + for KeyValue (stream,state) in states do + let sz = int64 state.Size + waitingCats.Ingest(category stream) + waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) + waiting <- waiting + 1 + waitingB <- waitingB + sz + if waiting <> 0 then log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) + if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) + if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) + + type private Stats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = + let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None + let progCommitFails, progCommits = ref 0, ref 0 + let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 + let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + let dumpStats (available,maxDop) = + log.Information("Holding Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", + !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 + if !progCommitFails <> 0 || !progCommits <> 0 then + match comittedEpoch with + | None -> + log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) + | Some committed when !progCommitFails <> 0 -> + log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) + | Some committed -> + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) + progCommits := 0; progCommitFails := 0 + else + log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", + pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) + member __.Handle : InternalMessage -> unit = function + | Batch _ | ActivateSeries _ | CloseSeries _-> () // stats are managed via Added internal message in same cycle + | ProgressResult (Choice1Of2 epoch) -> + incr progCommits + comittedEpoch <- Some epoch + | ProgressResult (Choice2Of2 (_exn : exn)) -> + incr progCommitFails + | Added (streams,events) -> + incr batchesPended + streamsPended := !streamsPended + streams + eventsPended := !eventsPended + events + member __.HandleValidated(epoch, pendingBatches) = + validatedEpoch <- epoch + pendingBatchCount <- pendingBatches + member __.HandleCommitted epoch = + comittedEpoch <- epoch + member __.TryDump((available,maxDop),streams : Streams) = + incr cycles + if statsDue () then + dumpStats (available,maxDop) + streams.Dump log + + and [] private InternalMessage = + | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq + /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` + | ProgressResult of Choice + /// Internal message for stats purposes + | Added of steams: int * events: int + | CloseSeries of seriesIndex: int + | ActivateSeries of seriesIndex: int + + let tryRemove key (dict: Dictionary<_,_>) = + match dict.TryGetValue key with + | true, value -> + dict.Remove key |> ignore + Some value + | false, _ -> None + + /// Holds batches away from Core processing to limit in-flight processing + type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxQueued, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = + let cts = new CancellationTokenSource() + let pumpDelayMs = defaultArg pumpDelayMs 5 + let work = ConcurrentQueue() + let readMax = new Sem(maxQueued) + let submissionsMax = new Sem(maxSubmissions) + let streams = Streams() + let stats = Stats(log, maxQueued, statsInterval) + let pending = Queue<_>() + let readingAhead, ready = Dictionary>(), Dictionary>() + let progressWriter = Progress.Writer<_>() + let mutable activeSeries = initialSeriesIndex + let mutable validatedPos = None + + let handle = function + | Batch (seriesId, epoch, checkpoint, items) -> + let batchInfo = + let items = Array.ofSeq items + streams.Merge items + let markCompleted () = + submissionsMax.Release() + readMax.Release() + validatedPos <- Some (epoch,checkpoint) + work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) + markCompleted, items + if activeSeries = seriesId then pending.Enqueue batchInfo + else + match readingAhead.TryGetValue seriesId with + | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) + | true,current -> current.Add(batchInfo) + | ActivateSeries newActiveSeries -> + activeSeries <- newActiveSeries + let buffered = + match ready |> tryRemove newActiveSeries with + | Some completedChunkBatches -> + completedChunkBatches |> Seq.iter pending.Enqueue + work.Enqueue <| ActivateSeries (newActiveSeries + 1) + completedChunkBatches.Count + | None -> + match readingAhead |> tryRemove newActiveSeries with + | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count + | None -> 0 + log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", + newActiveSeries, buffered, ready.Count, readingAhead.Count) + | CloseSeries seriesIndex -> + if activeSeries = seriesIndex then + log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) + work.Enqueue <| ActivateSeries (activeSeries + 1) + else + match readingAhead |> tryRemove seriesIndex with + | Some batchesRead -> + ready.[seriesIndex] <- batchesRead + log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) + | None -> + ready.[seriesIndex] <- ResizeArray() + log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) + // These events are for stats purposes + | Added _ + | ProgressResult _ -> () + + member private __.Pump() = async { + use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) + Async.Start(progressWriter.Pump(), cts.Token) + while not cts.IsCancellationRequested do + work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) + let mutable schedulerAccepting = true + // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted + while pending.Count <> 0 && submissionsMax.HasCapacity && schedulerAccepting do + let markCompleted, events = pending.Peek() + let! submitted = scheduler.TrySubmit(markCompleted, events) + if submitted then + pending.Dequeue() |> ignore + // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) + do! submissionsMax.Await() + else + schedulerAccepting <- false + // 2. Update any progress into the stats + stats.HandleValidated(Option.map fst validatedPos, fst readMax.State) + validatedPos |> Option.iter progressWriter.Post + stats.HandleCommitted progressWriter.CommittedEpoch + // 3. Forward content for any active streams into processor immediately + let relevantBufferedStreams = streams.Take(scheduler.AllStreams.Contains) + scheduler.AddOpenStreamData(relevantBufferedStreams) + // 4. Periodically emit status info + stats.TryDump(submissionsMax.State,streams) + do! Async.Sleep pumpDelayMs } + + /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes + static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval) = + let instance = new Engine<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval = statsInterval) + Async.Start <| instance.Pump() + instance + + /// Awaits space in `read` to limit reading ahead - yields (used,maximum) counts from Read Semaphore for logging purposes + member __.Submit(content : Message) = async { + do! readMax.Await() + match content with + | Message.Batch (seriesId, epoch, markBatchCompleted, events) -> + work.Enqueue <| Batch (seriesId, epoch, markBatchCompleted, events) + // NB readMax.Release() is effected in the Batch handler's MarkCompleted() + | Message.EndOfSeries seriesId -> + work.Enqueue <| CloseSeries seriesId + readMax.Release() + return readMax.State } + + /// As range assignments get revoked, a user is expected to `Stop `the active processing thread for the Ingester before releasing references to it + member __.Stop() = cts.Cancel() + +type Ingester = + + /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing + static member Start<'R>(log, scheduler, maxRead, maxSubmissions, ?statsInterval) = + let singleSeriesIndex = 0 + let instance = Ingestion.Engine<'R>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + { new IIngester with + member __.Submit(epoch, markCompleted, items) : Async = + instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) + member __.Stop() = __.Stop() } \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 9839c7557..94b803ecc 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -10,6 +10,7 @@ + From 0900216f4bbf3c9a6a123f8745269696ac67c737 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 07:02:40 +0100 Subject: [PATCH 229/353] More reformatting --- equinox-sync/Sync/Projection2.fs | 101 ++++++++++++++++--------------- 1 file changed, 52 insertions(+), 49 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 018e8dc66..cc5974b4f 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -89,10 +89,10 @@ module Scheduling = type InternalMessage<'R> = /// Enqueue a batch of items with supplied progress marking function | Add of markCompleted: (unit -> unit) * items: StreamItem[] - /// Stats per submitted batch for stats listeners to aggregate - | Added of streams: int * skip: int * events: int /// Submit new data pertaining to a stream that has commenced processing | AddActive of KeyValuePair[] + /// Stats per submitted batch for stats listeners to aggregate + | Added of streams: int * skip: int * events: int /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> @@ -163,44 +163,46 @@ module Scheduling = let streams = StreamStates() let progressState = Progress.State() + let validVsSkip (streamState : StreamState) (item : StreamItem) = + match streamState.write, item.index + 1L with + | Some cw, required when cw >= required -> 0, 1 + | _ -> 1, 0 + + let handle x = + match x with + | Add (releaseRead, items) -> + let reqs = Dictionary() + let mutable count, skipCount = 0, 0 + for item in items do + let stream,streamState = streams.Add(item.stream,item.index,item.event) + match validVsSkip streamState item with + | 0, skip -> + skipCount <- skipCount + skip + | required, _ -> + count <- count + required + reqs.[stream] <- item.index+1L + let markCompleted () = + releaseRead() + batches.Release() + progressState.AppendBatch(markCompleted,reqs) + work.Enqueue(Added (reqs.Count,skipCount,count)) + | AddActive events -> + for e in events do + streams.InternalMerge(e.Key,e.Value) + | Added _ -> + () + | Result (stream,r) -> + match interpretProgress streams stream r with + | Some index -> + progressState.MarkStreamProgress(stream,index) + streams.MarkCompleted(stream,index) + | None -> + streams.MarkFailed stream + member private __.Pump(stats : Stats<'R>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) Async.Start(dispatcher.Pump(), cts.Token) - let validVsSkip (streamState : StreamState) (item : StreamItem) = - match streamState.write, item.index + 1L with - | Some cw, required when cw >= required -> 0, 1 - | _ -> 1, 0 - let handle x = - match x with - | Add (releaseRead, items) -> - let reqs = Dictionary() - let mutable count, skipCount = 0, 0 - for item in items do - let stream,streamState = streams.Add(item.stream,item.index,item.event) - match validVsSkip streamState item with - | 0, skip -> - skipCount <- skipCount + skip - | required, _ -> - count <- count + required - reqs.[stream] <- item.index+1L - let markCompleted () = - releaseRead() - batches.Release() - progressState.AppendBatch(markCompleted,reqs) - work.Enqueue(Added (reqs.Count,skipCount,count)) - | AddActive events -> - for e in events do - streams.InternalMerge(e.Key,e.Value) - | Added _ -> - () - | Result (stream,r) -> - match interpretProgress streams stream r with - | Some index -> - progressState.MarkStreamProgress(stream,index) - streams.MarkCompleted(stream,index) - | None -> - streams.MarkFailed stream - + while not cts.IsCancellationRequested do // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state let mutable idle = true @@ -224,6 +226,7 @@ module Scheduling = stats.TryDump(not addsBeingAccepted,capacity,dispatcher.State,streams) // 4. Do a minimal sleep so we don't run completely hot when empty if idle then do! Async.Sleep sleepIntervalMs } + static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = let dispatcher = Dispatcher(processorDop) let instance = new Engine<'R>(maxPendingBatches, dispatcher, project, interpretProgress) @@ -391,6 +394,18 @@ module Ingestion = match readingAhead.TryGetValue seriesId with | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) | true,current -> current.Add(batchInfo) + | CloseSeries seriesIndex -> + if activeSeries = seriesIndex then + log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) + work.Enqueue <| ActivateSeries (activeSeries + 1) + else + match readingAhead |> tryRemove seriesIndex with + | Some batchesRead -> + ready.[seriesIndex] <- batchesRead + log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) + | None -> + ready.[seriesIndex] <- ResizeArray() + log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) | ActivateSeries newActiveSeries -> activeSeries <- newActiveSeries let buffered = @@ -405,18 +420,6 @@ module Ingestion = | None -> 0 log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", newActiveSeries, buffered, ready.Count, readingAhead.Count) - | CloseSeries seriesIndex -> - if activeSeries = seriesIndex then - log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) - work.Enqueue <| ActivateSeries (activeSeries + 1) - else - match readingAhead |> tryRemove seriesIndex with - | Some batchesRead -> - ready.[seriesIndex] <- batchesRead - log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) - | None -> - ready.[seriesIndex] <- ResizeArray() - log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) // These events are for stats purposes | Added _ | ProgressResult _ -> () From 9fb9c0c63788dd246645ee29aedf2f65d8706443 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 11:30:01 +0100 Subject: [PATCH 230/353] Rewrite projection algorithm --- equinox-sync/Sync/CosmosIngester.fs | 28 ++- equinox-sync/Sync/CosmosSource.fs | 2 +- equinox-sync/Sync/EventStoreSource.fs | 2 +- equinox-sync/Sync/Projection2.fs | 270 +++++++++++++++++--------- 4 files changed, 193 insertions(+), 109 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index cbb423810..412d6f51e 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -85,8 +85,7 @@ type Stats(log : ILogger, statsInterval) = override __.Handle message = base.Handle message match message with - | Add (_,_) - | AddActive _ + | Merge _ | Added _ -> () | Result (_stream, Choice1Of2 ((es,bs),r)) -> events <- events + es @@ -104,29 +103,26 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed | ResultKind.TimedOut -> incr timedOut -let start (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, statsInterval) = +let start (log : Serilog.ILogger, cosmosContext, maxWriters, statsInterval) = let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() - let trim (writePos : int64 option, batch : StreamSpan) = - // Reduce the item count when we don't yet know the write position in order to efficiently discover the redundancy where data is already present - // 100K budget for first page of an event makes validations cheaper while retaining general efficiency - let mutable bytesBudget, countBudget = match writePos with None -> 100 * 1024, 100 | Some _ -> cosmosPayloadLimit, 4096 - let mutable count = 0 - let max2MbMax100EventsMax10EventsFirstTranche (y : Equinox.Codec.IEvent) = - bytesBudget <- bytesBudget - cosmosPayloadBytes y - countBudget <- countBudget - 1 + let trim (_currentWritePos : int64 option, batch : StreamSpan) = + let mutable count, countBudget, bytesBudget = 0, 4096, cosmosPayloadLimit + let withinLimits (y : Equinox.Codec.IEvent) = count <- count + 1 - // always send at least one event in order to surface the problem and have the stream mark malformed + countBudget <- countBudget - 1 + bytesBudget <- bytesBudget - cosmosPayloadBytes y + // always send at least one event in order to surface the problem and have the stream marked malformed count = 1 || (countBudget >= 0 && bytesBudget >= 0) - { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile max2MbMax100EventsMax10EventsFirstTranche } } - let project batch = async { + { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile withinLimits } } + let attemptWrite batch = async { let trimmed = trim batch try let! res = Writer.write log cosmosContext trimmed let stats = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes return Choice1Of2 (stats,res) with e -> return Choice2Of2 e } - let interpretProgress (streams: StreamStates) stream res = + let interpretWriteResultProgress (streams: Scheduling.StreamStates) stream res = let applyResultToStreamState = function | Choice1Of2 (_stats, Writer.Ok pos) -> streams.InternalUpdate stream pos null | Choice1Of2 (_stats, Writer.Duplicate pos) -> streams.InternalUpdate stream pos null @@ -139,4 +135,4 @@ let start (log : Serilog.ILogger, maxPendingBatches, cosmosContext, maxWriters, Writer.logTo writerResultLog (stream,res) wp let projectionAndCosmosStats = Stats(log.ForContext(), statsInterval) - Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxPendingBatches, maxWriters, project, interpretProgress) \ No newline at end of file + Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress) \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 7489bbf95..a5089a6aa 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -13,7 +13,7 @@ open System open System.Collections.Generic let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: CosmosContext, maxWriters) (transform : Document -> StreamItem seq) = - let cosmosIngester = CosmosIngester.start (log, maxPendingBatches, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) fun () -> let mutable rangeIngester = Unchecked.defaultof<_> let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) } diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 3cf602f16..2bf16156e 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -383,7 +383,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), maxProcessing, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) if spec.gorge then diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index cc5974b4f..e1076f049 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -83,30 +83,117 @@ module Progress = | _ -> do! Async.Sleep pumpSleepMs } module Scheduling = + type StreamStates() = + let mutable streams = Set.empty + let states = Dictionary() + let update stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + streams <- streams.Add stream + stream, state + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + stream, updated + let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } + let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore + + let busy = HashSet() + let pending trySlipstreamed (requestedOrder : string seq) = seq { + let proposed = HashSet() + for s in requestedOrder do + let state = states.[s] + if state.IsReady && not (busy.Contains s) then + proposed.Add s |> ignore + yield state.write, { stream = s; span = state.queue.[0] } + if trySlipstreamed then + // [lazily] Slipstream in futher events that have been posted to streams which we've already visited + for KeyValue(s,v) in states do + if v.IsReady && not (busy.Contains s) && proposed.Add s then + yield v.write, { stream = s; span = v.queue.[0] } } + let markBusy stream = busy.Add stream |> ignore + let markNotBusy stream = busy.Remove stream |> ignore + + // Result is intentionally a thread-safe persisent data structure + // This enables the (potentially multiple) Ingesters to determine streams (for which they potentially have successor events) that are in play + // Ingesters then supply these 'preview events' in advance of the processing being scheduled + // This enables the projection logic to roll future work into the current work in the interests of medium term throughput + member __.All = streams + member __.InternalMerge(stream, state) = update stream state |> ignore + member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } + member __.Add(stream, index, event, ?isMalformed) = + updateWritePos stream (defaultArg isMalformed false) None [| { index = index; events = [| event |] } |] + member __.Add(batch: StreamSpan, isMalformed) = + updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] + member __.SetMalformed(stream,isMalformed) = + updateWritePos stream isMalformed None [| { index = 0L; events = null } |] + member __.QueueWeight(stream) = + states.[stream].queue.[0].events |> Seq.sumBy eventSize + member __.MarkBusy stream = + markBusy stream + member __.MarkCompleted(stream, index) = + markNotBusy stream + markCompleted stream index + member __.MarkFailed stream = + markNotBusy stream + member __.Pending(trySlipsteamed, byQueuedPriority : string seq) : (int64 option * StreamSpan) seq = + pending trySlipsteamed byQueuedPriority + member __.Dump(log : ILogger) = + let mutable busyCount, busyB, ready, readyB, unprefixed, unprefixedB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0 + let busyCats, readyCats, readyStreams, unprefixedStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats(), CatStats() + let kb sz = (sz + 512L) / 1024L + for KeyValue (stream,state) in states do + match int64 state.Size with + | 0L -> + synced <- synced + 1 + | sz when busy.Contains stream -> + busyCats.Ingest(category stream) + busyCount <- busyCount + 1 + busyB <- busyB + sz + | sz when state.isMalformed -> + malformedStreams.Ingest(stream, mb sz |> int64) + malformed <- malformed + 1 + malformedB <- malformedB + sz + | sz when not state.IsReady -> + unprefixedStreams.Ingest(stream, mb sz |> int64) + unprefixed <- unprefixed + 1 + unprefixedB <- unprefixedB + sz + | sz -> + readyCats.Ingest(category stream) + readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) + ready <- ready + 1 + readyB <- readyB + sz + log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", + synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) + if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + if unprefixedStreams.Any then log.Information("Waiting Streams, KB {missingStreams}", Seq.truncate 3 unprefixedStreams.StatsDescending) + if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners [] type InternalMessage<'R> = - /// Enqueue a batch of items with supplied progress marking function - | Add of markCompleted: (unit -> unit) * items: StreamItem[] /// Submit new data pertaining to a stream that has commenced processing - | AddActive of KeyValuePair[] + | Merge of KeyValuePair[] /// Stats per submitted batch for stats listeners to aggregate | Added of streams: int * skip: int * events: int /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> + type BufferState = Idle | Partial | Full | Slipstreaming /// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = - let cycles, filled, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 + let cycles, states, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, CatStats(), ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats capacity (used,maxDop) = - log.Information("Projection Cycles {cycles} Filled {filled:P0} Capacity {capacity} Active {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, float !filled/float !cycles, capacity, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) - cycles := 0; filled := 0; batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + let dumpStats (used,maxDop) = + log.Information("Projection Cycles {cycles} States {states} Busy {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + !cycles, states.StatsDescending, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + cycles := 0; states.Clear(); batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function - | Add _ | AddActive _ -> () + | Merge _ -> () | Added (streams, skipped, events) -> incr batchesPended streamsPended := !streamsPended + streams @@ -116,13 +203,15 @@ module Scheduling = incr resultCompleted | Result (_stream, Choice2Of2 _) -> incr resultExn - member __.TryDump(wasFull,capacity,(used,max),streams : StreamStates) = + member __.TryDump(state,(used,max),streams : StreamStates) = incr cycles - if wasFull then incr filled - if statsDue () then - dumpStats capacity (used,max) + states.Ingest(string state) + let due = statsDue () + if due then + dumpStats (used,max) __.DumpExtraStats() streams.Dump log + due /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () @@ -138,7 +227,6 @@ module Scheduling = dop.Release() } [] member __.Result = result.Publish member __.HasCapacity = dop.HasCapacity - member __.CurrentCapacity = dop.CurrentCapacity member __.State = dop.State member __.TryAdd(item,?timeout) = async { let! got = dop.TryAwait(?timeout=timeout) @@ -149,17 +237,16 @@ module Scheduling = let! ct = Async.CancellationToken for item in work.GetConsumingEnumerable ct do Async.Start(dispatch item) } - /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order /// a) does not itself perform any reading activities /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) /// c) submits work to the supplied Dispatcher (which it triggers pumping of) /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R>(maxPendingBatches, dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = + type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() - let batches = Sem maxPendingBatches let work = ConcurrentQueue>() + let pending = ConcurrentQueue<_*StreamItem[]>() let streams = StreamStates() let progressState = Progress.State() @@ -167,81 +254,87 @@ module Scheduling = match streamState.write, item.index + 1L with | Some cw, required when cw >= required -> 0, 1 | _ -> 1, 0 + let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = + let reqs = Dictionary() + let mutable count, skipCount = 0, 0 + for item in items do + let stream,streamState = streams.Add(item.stream,item.index,item.event) + match validVsSkip streamState item with + | 0, skip -> + skipCount <- skipCount + skip + | required, _ -> + count <- count + required + reqs.[stream] <- item.index+1L + progressState.AppendBatch(markCompleted,reqs) + feedStats <| Added (reqs.Count,skipCount,count) + let tryDrainResults feedStats = + let mutable worked, more = false, true + while more do + match work.TryDequeue() with + | false, _ -> more <- false + | true, x -> + match x with + | Added _ -> () // Only processed in Stats + | Merge events -> for e in events do streams.InternalMerge(e.Key,e.Value) + | Result (stream,res) -> + match interpretProgress streams stream res with + | None -> streams.MarkFailed stream + | Some index -> + progressState.MarkStreamProgress(stream,index) + streams.MarkCompleted(stream,index) + feedStats x + worked <- true + worked + let tryFillDispatcher includeSlipstreamed = async { + let mutable worked, filled = false, true + if dispatcher.HasCapacity then + let potential = streams.Pending(includeSlipstreamed, progressState.InScheduledOrder streams.QueueWeight) + let xs = potential.GetEnumerator() + while xs.MoveNext() && not filled do + let (_,{stream = s} : StreamSpan) as item = xs.Current + let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) + if succeeded then streams.MarkBusy s + worked <- worked || succeeded // if we added any request, we also don't sleep + filled <- not succeeded + return worked, filled } - let handle x = - match x with - | Add (releaseRead, items) -> - let reqs = Dictionary() - let mutable count, skipCount = 0, 0 - for item in items do - let stream,streamState = streams.Add(item.stream,item.index,item.event) - match validVsSkip streamState item with - | 0, skip -> - skipCount <- skipCount + skip - | required, _ -> - count <- count + required - reqs.[stream] <- item.index+1L - let markCompleted () = - releaseRead() - batches.Release() - progressState.AppendBatch(markCompleted,reqs) - work.Enqueue(Added (reqs.Count,skipCount,count)) - | AddActive events -> - for e in events do - streams.InternalMerge(e.Key,e.Value) - | Added _ -> - () - | Result (stream,r) -> - match interpretProgress streams stream r with - | Some index -> - progressState.MarkStreamProgress(stream,index) - streams.MarkCompleted(stream,index) - | None -> - streams.MarkFailed stream - member private __.Pump(stats : Stats<'R>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) Async.Start(dispatcher.Pump(), cts.Token) - while not cts.IsCancellationRequested do - // 1. propagate read items to buffer; propagate write write results to buffer and progress write impacts to local state - let mutable idle = true - work |> ConcurrentQueue.drain (fun x -> - handle x - stats.Handle x - idle <- false) - // 2. top up provisioning of writers queue - let capacity = dispatcher.CurrentCapacity - let mutable addsBeingAccepted = capacity <> 0 - if addsBeingAccepted then - let potential = streams.Pending(progressState.InScheduledOrder streams.QueueWeight) - let xs = potential.GetEnumerator() - while xs.MoveNext() && addsBeingAccepted do - let (_,{stream = s} : StreamSpan) as item = xs.Current - let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) - if succeeded then streams.MarkBusy s - idle <- idle && not succeeded // any add makes it not idle - addsBeingAccepted <- succeeded - // 3. Periodically emit status info - stats.TryDump(not addsBeingAccepted,capacity,dispatcher.State,streams) + let mutable idle, dispatcherState, finished = true, Idle, false + while not finished do + // 1. propagate write write outcomes to buffer (can mark batches completed etc) + let processedResults = tryDrainResults stats.Handle + // 2. top up provisioning of writers queue + let! dispatched, filled = tryFillDispatcher (dispatcherState = Slipstreaming) + idle <- idle && not processedResults && not dispatched + if dispatcherState = Idle && filled then dispatcherState <- Full + elif dispatcherState = Idle && dispatched then dispatcherState <- Partial + if dispatcherState = Slipstreaming then finished <- true + if not filled then // need to bring more work into the pool as we can't fill the work queue + match pending.TryDequeue() with + | true, batch -> ingestPendingBatch stats.Handle batch + | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters + // 3. Supply state to accumulate (and periodically emit) status info + if stats.TryDump(dispatcherState,dispatcher.State,streams) then idle <- false // 4. Do a minimal sleep so we don't run completely hot when empty if idle then do! Async.Sleep sleepIntervalMs } - static member Start<'R>(stats, maxPendingBatches, processorDop, project, interpretProgress) = - let dispatcher = Dispatcher(processorDop) - let instance = new Engine<'R>(maxPendingBatches, dispatcher, project, interpretProgress) + static member Start<'R>(stats, projectorDop, project, interpretProgress) = + let dispatcher = Dispatcher(projectorDop) + let instance = new Engine<'R>(dispatcher, project, interpretProgress) Async.Start <| instance.Pump(stats) instance - /// Attempt to feed in a batch (subject to there being capacity to do so) - member __.TrySubmit(markCompleted, events) = async { - let! got = batches.TryAwait() - if got then - work.Enqueue <| Add (markCompleted, events) - return got } + /// Enqueue a batch of items with supplied progress marking function + /// Submission is accepted on trust; they are internally processed in order of submission + /// caller should ensure that (when multiple submitters are in play) no single Range submits more than their fair share + member __.Submit(markCompleted: (unit -> unit), items: StreamItem[]) = + pending.Enqueue (markCompleted, items) member __.AddOpenStreamData(events) = - work.Enqueue <| AddActive events + work.Enqueue <| Merge events member __.AllStreams = streams.All @@ -250,7 +343,7 @@ module Scheduling = type Projector = - static member Start(log, maxPendingBatches, maxActiveBatches, project : StreamSpan -> Async, ?statsInterval) = + static member Start(log, projectorDop, project : StreamSpan -> Async, ?statsInterval) = let project (_maybeWritePos, batch) = async { try let! count = project batch return Choice1Of2 (batch.span.index + int64 count) @@ -259,7 +352,7 @@ type Projector = | Choice1Of2 index -> Some index | Choice2Of2 _ -> None let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.)) - Scheduling.Engine.Start(stats, maxPendingBatches, maxActiveBatches, project, interpretProgress) + Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress) module Ingestion = @@ -429,17 +522,12 @@ module Ingestion = Async.Start(progressWriter.Pump(), cts.Token) while not cts.IsCancellationRequested do work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - let mutable schedulerAccepting = true // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && submissionsMax.HasCapacity && schedulerAccepting do - let markCompleted, events = pending.Peek() - let! submitted = scheduler.TrySubmit(markCompleted, events) - if submitted then - pending.Dequeue() |> ignore - // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) - do! submissionsMax.Await() - else - schedulerAccepting <- false + while pending.Count <> 0 && submissionsMax.HasCapacity do + // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) + do! submissionsMax.Await() + let markCompleted, events = pending.Dequeue() + scheduler.Submit(markCompleted, events) // 2. Update any progress into the stats stats.HandleValidated(Option.map fst validatedPos, fst readMax.State) validatedPos |> Option.iter progressWriter.Post From 2ddf2f4063d9c00acb0b058cec6a4cba92562219 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 12:13:35 +0100 Subject: [PATCH 231/353] Fix read vs submit counts --- equinox-sync/Sync/CosmosSource.fs | 2 +- equinox-sync/Sync/Projection2.fs | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index a5089a6aa..2e8d26320 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -16,7 +16,7 @@ let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: Cosmo let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) fun () -> let mutable rangeIngester = Unchecked.defaultof<_> - let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches, maxWriters, TimeSpan.FromMinutes 1.) } + let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches*2, maxPendingBatches, TimeSpan.FromMinutes 1.) } let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq in rangeIngester.Submit(epoch, checkpoint, events) let dispose () = rangeIngester.Stop () let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index e1076f049..78a92a8df 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -457,14 +457,14 @@ module Ingestion = | false, _ -> None /// Holds batches away from Core processing to limit in-flight processing - type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxQueued, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = + type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxRead, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = let cts = new CancellationTokenSource() let pumpDelayMs = defaultArg pumpDelayMs 5 let work = ConcurrentQueue() - let readMax = new Sem(maxQueued) + let readMax = new Sem(maxRead) let submissionsMax = new Sem(maxSubmissions) let streams = Streams() - let stats = Stats(log, maxQueued, statsInterval) + let stats = Stats(log, maxRead, statsInterval) let pending = Queue<_>() let readingAhead, ready = Dictionary>(), Dictionary>() let progressWriter = Progress.Writer<_>() @@ -529,7 +529,7 @@ module Ingestion = let markCompleted, events = pending.Dequeue() scheduler.Submit(markCompleted, events) // 2. Update any progress into the stats - stats.HandleValidated(Option.map fst validatedPos, fst readMax.State) + stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch // 3. Forward content for any active streams into processor immediately From 3f674ab07c4373e1994ea3eb788757a71edfee9a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 13:01:27 +0100 Subject: [PATCH 232/353] Tidy builders --- equinox-sync/Sync/CosmosSource.fs | 44 +++++++++++++++---------------- equinox-sync/Sync/Program.fs | 7 ++--- 2 files changed, 25 insertions(+), 26 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 2e8d26320..5305a0fc7 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -1,6 +1,5 @@ module SyncTemplate.CosmosSource -open Equinox.Cosmos.Core open Equinox.Cosmos.Projection open Equinox.Projection open Equinox.Projection2 @@ -12,37 +11,36 @@ open Serilog open System open System.Collections.Generic -let createRangeSyncHandler (log:ILogger) maxPendingBatches (cosmosContext: CosmosContext, maxWriters) (transform : Document -> StreamItem seq) = - let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) - fun () -> - let mutable rangeIngester = Unchecked.defaultof<_> - let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxPendingBatches*2, maxPendingBatches, TimeSpan.FromMinutes 1.) } - let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq in rangeIngester.Submit(epoch, checkpoint, events) - let dispose () = rangeIngester.Stop () - let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok - let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved - let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } - let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time - log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", - epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., - (let e = pt.Elapsed in e.TotalSeconds), cur, max) - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - } - ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) +let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq) (maxReads, maxSubmissions) cosmosIngester () = + let mutable rangeIngester = Unchecked.defaultof<_> + let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxReads, maxSubmissions, TimeSpan.FromMinutes 1.) } + let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq in rangeIngester.Submit(epoch, checkpoint, events) + let dispose () = rangeIngester.Stop () + let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok + let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { + sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 + // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved + let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } + let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time + log.Information("Read -{token,6} {count,4} docs {requestCharge,6}RU {l:n1}s Post {pt:n3}s {cur}/{max}", + epoch, docs.Count, (let c = ctx.FeedResponse.RequestCharge in c.ToString("n1")), float sw.ElapsedMilliseconds / 1000., + (let e = pt.Elapsed in e.TotalSeconds), cur, max) + sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor + } + ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maxDocuments, lagReportFreq : TimeSpan option) - createRangeProjector = async { + (cosmosContext, maxWriters) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag + let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let! _feedEventHost = ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, - createObserver = createRangeProjector, ?cfBatchSize = maxDocuments, ?reportLagAndAwaitNextEstimation = maybeLogLag) + createObserver = createRangeProjector cosmosIngester, ?cfBatchSize = maxDocuments, ?reportLagAndAwaitNextEstimation = maybeLogLag) do! Async.AwaitKeyboardInterrupt() } //#if marveleqx diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index dd5b88666..947b4df40 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -330,15 +330,16 @@ let main argv = let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() let auxDiscovery, aux, leaseId, startFromHere, maxDocuments, lagFrequency = args.BuildChangeFeedParams() #if marveleqx - let createSyncHandler = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformV0 catFilter) + let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformV0 catFilter) #else - let createSyncHandler = CosmosSource.createRangeSyncHandler log args.MaxPendingBatches (target, args.MaxWriters) (CosmosSource.transformOrFilter catFilter) + let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformOrFilter catFilter) // Uncomment to test marveleqx mode // let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformV0 catFilter) #endif CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) - createSyncHandler + (target, args.MaxWriters) + (createSyncHandler (args.MaxPendingBatches*2,args.MaxPendingBatches)) #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let catFilter = args.Source.CategoryFilterFunction From 93771e4d54d37e096de649f05d174b3ca3a9b683 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 13:04:02 +0100 Subject: [PATCH 233/353] CosmosIngester lifetime --- equinox-sync/Sync/CosmosSource.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 5305a0fc7..c430c7e9c 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -41,7 +41,8 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, createObserver = createRangeProjector cosmosIngester, ?cfBatchSize = maxDocuments, ?reportLagAndAwaitNextEstimation = maybeLogLag) - do! Async.AwaitKeyboardInterrupt() } + do! Async.AwaitKeyboardInterrupt() + cosmosIngester.Stop() } //#if marveleqx [] From 149ece88714b2474e95a5a69d897df2f122b55b0 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 13:18:58 +0100 Subject: [PATCH 234/353] Fix dispatcher logic --- equinox-sync/Sync/Projection2.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 78a92a8df..910f3e933 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -309,9 +309,9 @@ module Scheduling = // 2. top up provisioning of writers queue let! dispatched, filled = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched - if dispatcherState = Idle && filled then dispatcherState <- Full + if dispatcherState = Idle && dispatched && filled then dispatcherState <- Full; finished <- true + elif dispatcherState = Slipstreaming then finished <- true elif dispatcherState = Idle && dispatched then dispatcherState <- Partial - if dispatcherState = Slipstreaming then finished <- true if not filled then // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with | true, batch -> ingestPendingBatch stats.Handle batch From 6bed13eb0c83bf7d3036d1aef56c78a36107c57c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 13:39:08 +0100 Subject: [PATCH 235/353] Remove Partial Dispatcher state --- equinox-sync/Sync/Projection2.fs | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 910f3e933..7889e2f1c 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -182,7 +182,7 @@ module Scheduling = /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> - type BufferState = Idle | Partial | Full | Slipstreaming + type BufferState = Idle | Full | Slipstreaming /// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, states, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, CatStats(), ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 @@ -286,17 +286,17 @@ module Scheduling = worked <- true worked let tryFillDispatcher includeSlipstreamed = async { - let mutable worked, filled = false, true - if dispatcher.HasCapacity then + let mutable hasCapacity, worked = dispatcher.HasCapacity, false + if hasCapacity then let potential = streams.Pending(includeSlipstreamed, progressState.InScheduledOrder streams.QueueWeight) let xs = potential.GetEnumerator() - while xs.MoveNext() && not filled do + while xs.MoveNext() && hasCapacity do let (_,{stream = s} : StreamSpan) as item = xs.Current let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) if succeeded then streams.MarkBusy s worked <- worked || succeeded // if we added any request, we also don't sleep - filled <- not succeeded - return worked, filled } + hasCapacity <- succeeded + return hasCapacity, worked } member private __.Pump(stats : Stats<'R>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) @@ -307,12 +307,11 @@ module Scheduling = // 1. propagate write write outcomes to buffer (can mark batches completed etc) let processedResults = tryDrainResults stats.Handle // 2. top up provisioning of writers queue - let! dispatched, filled = tryFillDispatcher (dispatcherState = Slipstreaming) + let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched - if dispatcherState = Idle && dispatched && filled then dispatcherState <- Full; finished <- true + if dispatcherState = Idle && not hasCapacity then dispatcherState <- Full; finished <- true elif dispatcherState = Slipstreaming then finished <- true - elif dispatcherState = Idle && dispatched then dispatcherState <- Partial - if not filled then // need to bring more work into the pool as we can't fill the work queue + if hasCapacity && not finished then // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with | true, batch -> ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters From e44ea38e6374f79c58fc6168a80df5ab23930d77 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 13:56:14 +0100 Subject: [PATCH 236/353] Destruct states --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 7889e2f1c..84f813181 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -188,7 +188,7 @@ module Scheduling = let cycles, states, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, CatStats(), ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (used,maxDop) = - log.Information("Projection Cycles {cycles} States {states} Busy {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", + log.Information("Projection Cycles {cycles} States {@states} Busy {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", !cycles, states.StatsDescending, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) cycles := 0; states.Clear(); batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : InternalMessage<'R> -> unit From 6275aa1ff7fd77edd8a2abf343a7848ccb21f4da Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 14:11:09 +0100 Subject: [PATCH 237/353] Logging @ --- equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/Projection2.fs | 26 +++++++++++++------------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 412d6f51e..9d13aaa08 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -79,7 +79,7 @@ type Stats(log : ILogger, statsInterval) = log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", !rateLimited, !timedOut, !tooLarge, !malformed, !resultExnOther) rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0 - if badCats.Any then log.Error("Malformed categories {badCats}", badCats.StatsDescending); badCats.Clear() + if badCats.Any then log.Error("Malformed categories {@badCats}", badCats.StatsDescending); badCats.Clear() Equinox.Cosmos.Store.Log.InternalMetrics.dump log override __.Handle message = diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 84f813181..0125ff970 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -166,11 +166,11 @@ module Scheduling = readyB <- readyB + sz log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) - if busyCats.Any then log.Information("Active Categories, events {busyCats}", Seq.truncate 5 busyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Categories, events {readyCats}", Seq.truncate 5 readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams, KB {readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - if unprefixedStreams.Any then log.Information("Waiting Streams, KB {missingStreams}", Seq.truncate 3 unprefixedStreams.StatsDescending) - if malformedStreams.Any then log.Information("Malformed Streams, MB {malformedStreams}", malformedStreams.StatsDescending) + if busyCats.Any then log.Information("Active Categories, events {@busyCats}", Seq.truncate 5 busyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Categories, events {@readyCats}", Seq.truncate 5 readyCats.StatsDescending) + if readyCats.Any then log.Information("Ready Streams, KB {@readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) + if unprefixedStreams.Any then log.Information("Waiting Streams, KB {@missingStreams}", Seq.truncate 3 unprefixedStreams.StatsDescending) + if malformedStreams.Any then log.Information("Malformed Streams, MB {@malformedStreams}", malformedStreams.StatsDescending) /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners [] @@ -187,9 +187,9 @@ module Scheduling = type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = let cycles, states, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, CatStats(), ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (used,maxDop) = - log.Information("Projection Cycles {cycles} States {@states} Busy {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Completed {completed} Exceptions {exns}", - !cycles, states.StatsDescending, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !resultCompleted, !resultExn) + let dumpStats (used,maxDop) pendingCount = + log.Information("Projection Cycles {cycles} States {@states} Busy {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Pending {pending} Completed {completed} Exceptions {exns}", + !cycles, states.StatsDescending, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, pendingCount, !resultCompleted, !resultExn) cycles := 0; states.Clear(); batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function @@ -203,12 +203,12 @@ module Scheduling = incr resultCompleted | Result (_stream, Choice2Of2 _) -> incr resultExn - member __.TryDump(state,(used,max),streams : StreamStates) = + member __.TryDump(state,(used,max),streams : StreamStates, pendingCount) = incr cycles states.Ingest(string state) let due = statsDue () if due then - dumpStats (used,max) + dumpStats (used,max) pendingCount __.DumpExtraStats() streams.Dump log due @@ -316,7 +316,7 @@ module Scheduling = | true, batch -> ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters // 3. Supply state to accumulate (and periodically emit) status info - if stats.TryDump(dispatcherState,dispatcher.State,streams) then idle <- false + if stats.TryDump(dispatcherState,dispatcher.State,streams,pending.Count) then idle <- false // 4. Do a minimal sleep so we don't run completely hot when empty if idle then do! Async.Sleep sleepIntervalMs } @@ -390,8 +390,8 @@ module Ingestion = waiting <- waiting + 1 waitingB <- waitingB + sz if waiting <> 0 then log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) - if waitingCats.Any then log.Information("Waiting Categories, events {readyCats}", Seq.truncate 5 waitingCats.StatsDescending) - if waitingCats.Any then log.Information("Waiting Streams, KB {readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) + if waitingCats.Any then log.Information("Waiting Categories, events {@readyCats}", Seq.truncate 5 waitingCats.StatsDescending) + if waitingCats.Any then log.Information("Waiting Streams, KB {@readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) type private Stats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None From b61cf19c0b1e6b7174cf7f08aa347b7647d287c8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 14:23:00 +0100 Subject: [PATCH 238/353] Delay materialize --- equinox-sync/Sync/CosmosSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index c430c7e9c..fe5127bb0 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -14,7 +14,7 @@ open System.Collections.Generic let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq) (maxReads, maxSubmissions) cosmosIngester () = let mutable rangeIngester = Unchecked.defaultof<_> let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxReads, maxSubmissions, TimeSpan.FromMinutes 1.) } - let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform |> Array.ofSeq in rangeIngester.Submit(epoch, checkpoint, events) + let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform in rangeIngester.Submit(epoch, checkpoint, events) let dispose () = rangeIngester.Stop () let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let processBatch (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { From 63942b818d06c71c79e06b25056e725b3c0ef2ea Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 14:40:00 +0100 Subject: [PATCH 239/353] ES --- equinox-sync/Sync/Sync.fsproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 94b803ecc..5ff15a172 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,7 +4,7 @@ Exe netcoreapp2.1 5 - $(DefineConstants);cosmos + $(DefineConstants);cosmos_ From 3e6ed71a476bb77bc08618e8842f97b260a10551 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 15:13:58 +0100 Subject: [PATCH 240/353] Disable checkpoint logging --- equinox-sync/Sync/Program.fs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 947b4df40..bc4ba7c67 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -53,7 +53,7 @@ module CmdParser = | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Verbose -> "request Verbose Logging. Default: off" | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" - | MaxProcessing _ -> "Maximum number of batches to process concurrently. Default: 128" + | MaxProcessing _ -> "Maximum number of batches to process concurrently. Default: 16" | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" #if cosmos | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" @@ -80,8 +80,8 @@ module CmdParser = and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.Verbose = a.Contains Verbose - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,1000) - member __.MaxProcessing = a.GetResult(MaxProcessing,32) + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) + member __.MaxProcessing = a.GetResult(MaxProcessing,16) member __.MaxWriters = a.GetResult(MaxWriters,1024) #if cosmos member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose @@ -293,11 +293,12 @@ module Logging = l.WriteTo.Sink(Equinox.Cosmos.Store.Log.InternalMetrics.RuCounters.RuCounterSink()) |> ignore) |> ignore a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke + let isCp = Filters.Matching.FromSource().Invoke let isWriter = Filters.Matching.FromSource().Invoke let isCfp429a = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.LeaseManagement.DocumentServiceLeaseUpdater").Invoke let isCfp429b = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.LeaseRenewer").Invoke let isCfp429c = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.PartitionLoadBalancer").Invoke - (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isWriter x || isCfp429a x || isCfp429b x || isCfp429c x)) + (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCp x || isWriter x || isCfp429a x || isCfp429b x || isCfp429c x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) From 3992d0f71682e2ba1d6e006e2f029b4dc54a3d3a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 15:40:17 +0100 Subject: [PATCH 241/353] Terminate slipstreaming state --- equinox-sync/Sync/Projection2.fs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 0125ff970..27a4b99ff 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -309,9 +309,12 @@ module Scheduling = // 2. top up provisioning of writers queue let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched - if dispatcherState = Idle && not hasCapacity then dispatcherState <- Full; finished <- true - elif dispatcherState = Slipstreaming then finished <- true - if hasCapacity && not finished then // need to bring more work into the pool as we can't fill the work queue + match dispatcherState with + | Idle when not hasCapacity -> dispatcherState <- Full; finished <- true + | Slipstreaming when not dispatched -> dispatcherState <- Idle; finished <- true + | Slipstreaming -> finished <- true + | _ when not hasCapacity -> () + | _ -> // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with | true, batch -> ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters From 6d536daf49e543c31023dbe9a59711e70bd95b4c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 17:16:17 +0100 Subject: [PATCH 242/353] Log releases --- equinox-sync/Sync/Projection2.fs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 27a4b99ff..428b3140d 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -26,7 +26,6 @@ module private Helpers = /// Wait for the specified timeout to acquire (or return false instantly) member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) member __.HasCapacity = inner.CurrentCount > 0 - member __.CurrentCapacity = inner.CurrentCount module Progress = @@ -286,7 +285,7 @@ module Scheduling = worked <- true worked let tryFillDispatcher includeSlipstreamed = async { - let mutable hasCapacity, worked = dispatcher.HasCapacity, false + let mutable hasCapacity, dispatched = dispatcher.HasCapacity, false if hasCapacity then let potential = streams.Pending(includeSlipstreamed, progressState.InScheduledOrder streams.QueueWeight) let xs = potential.GetEnumerator() @@ -294,9 +293,9 @@ module Scheduling = let (_,{stream = s} : StreamSpan) as item = xs.Current let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) if succeeded then streams.MarkBusy s - worked <- worked || succeeded // if we added any request, we also don't sleep + dispatched <- dispatched || succeeded // if we added any request, we also don't sleep hasCapacity <- succeeded - return hasCapacity, worked } + return hasCapacity, dispatched } member private __.Pump(stats : Stats<'R>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) @@ -313,11 +312,11 @@ module Scheduling = | Idle when not hasCapacity -> dispatcherState <- Full; finished <- true | Slipstreaming when not dispatched -> dispatcherState <- Idle; finished <- true | Slipstreaming -> finished <- true - | _ when not hasCapacity -> () - | _ -> // need to bring more work into the pool as we can't fill the work queue + | _ when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with | true, batch -> ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters + | _ -> () // 3. Supply state to accumulate (and periodically emit) status info if stats.TryDump(dispatcherState,dispatcher.State,streams,pending.Count) then idle <- false // 4. Do a minimal sleep so we don't run completely hot when empty @@ -481,6 +480,7 @@ module Ingestion = let markCompleted () = submissionsMax.Release() readMax.Release() + Log.Error("MC {rm} {sm}",readMax.State,submissionsMax.State) validatedPos <- Some (epoch,checkpoint) work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) markCompleted, items @@ -528,8 +528,7 @@ module Ingestion = while pending.Count <> 0 && submissionsMax.HasCapacity do // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) do! submissionsMax.Await() - let markCompleted, events = pending.Dequeue() - scheduler.Submit(markCompleted, events) + scheduler.Submit(pending.Dequeue()) // 2. Update any progress into the stats stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) validatedPos |> Option.iter progressWriter.Post From 8bb7f4e4d7446002b65895a0c6965dce61e761af Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 17:53:57 +0100 Subject: [PATCH 243/353] logging --- equinox-sync/Sync/Projection2.fs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 428b3140d..3c74fce26 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -253,19 +253,6 @@ module Scheduling = match streamState.write, item.index + 1L with | Some cw, required when cw >= required -> 0, 1 | _ -> 1, 0 - let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = - let reqs = Dictionary() - let mutable count, skipCount = 0, 0 - for item in items do - let stream,streamState = streams.Add(item.stream,item.index,item.event) - match validVsSkip streamState item with - | 0, skip -> - skipCount <- skipCount + skip - | required, _ -> - count <- count + required - reqs.[stream] <- item.index+1L - progressState.AppendBatch(markCompleted,reqs) - feedStats <| Added (reqs.Count,skipCount,count) let tryDrainResults feedStats = let mutable worked, more = false, true while more do @@ -296,6 +283,19 @@ module Scheduling = dispatched <- dispatched || succeeded // if we added any request, we also don't sleep hasCapacity <- succeeded return hasCapacity, dispatched } + let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = + let reqs = Dictionary() + let mutable count, skipCount = 0, 0 + for item in items do + let stream,streamState = streams.Add(item.stream,item.index,item.event) + match validVsSkip streamState item with + | 0, skip -> + skipCount <- skipCount + skip + | required, _ -> + count <- count + required + reqs.[stream] <- item.index+1L + progressState.AppendBatch(markCompleted,reqs) + feedStats <| Added (reqs.Count,skipCount,count) member private __.Pump(stats : Stats<'R>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) @@ -314,7 +314,9 @@ module Scheduling = | Slipstreaming -> finished <- true | _ when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with - | true, batch -> ingestPendingBatch stats.Handle batch + | true, batch -> + Log.Error("Ingest") + ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters | _ -> () // 3. Supply state to accumulate (and periodically emit) status info From 01db2eb4dcdb584490b4e6e837ba4556cbf78d1e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 18:20:21 +0100 Subject: [PATCH 244/353] remove logging --- equinox-sync/Sync/Projection2.fs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 3c74fce26..aa7150581 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -314,9 +314,7 @@ module Scheduling = | Slipstreaming -> finished <- true | _ when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with - | true, batch -> - Log.Error("Ingest") - ingestPendingBatch stats.Handle batch + | true, batch -> ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters | _ -> () // 3. Supply state to accumulate (and periodically emit) status info @@ -482,7 +480,6 @@ module Ingestion = let markCompleted () = submissionsMax.Release() readMax.Release() - Log.Error("MC {rm} {sm}",readMax.State,submissionsMax.State) validatedPos <- Some (epoch,checkpoint) work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) markCompleted, items From 0333adc08897018651f1af98ecc1e9c7aa50092d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 8 May 2019 21:33:43 +0100 Subject: [PATCH 245/353] tidy scheduler --- equinox-sync/Sync/Projection2.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index aa7150581..51fd90383 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -309,13 +309,13 @@ module Scheduling = let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched match dispatcherState with - | Idle when not hasCapacity -> dispatcherState <- Full; finished <- true + | Idle when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue + match pending.TryDequeue() with + | true, batch -> ingestPendingBatch stats.Handle batch + | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters + | Idle -> dispatcherState <- Full; finished <- true | Slipstreaming when not dispatched -> dispatcherState <- Idle; finished <- true | Slipstreaming -> finished <- true - | _ when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue - match pending.TryDequeue() with - | true, batch -> ingestPendingBatch stats.Handle batch - | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters | _ -> () // 3. Supply state to accumulate (and periodically emit) status info if stats.TryDump(dispatcherState,dispatcher.State,streams,pending.Count) then idle <- false From cc557cd5a7be2382576647af613a1fd01368fb90 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 00:34:15 +0100 Subject: [PATCH 246/353] Reorg MaxProcessing --- equinox-sync/Sync/EventStoreSource.fs | 4 ++-- equinox-sync/Sync/Program.fs | 13 ++++++------- 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 2bf16156e..cdcda0f65 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -352,7 +352,7 @@ type ReaderSpec = type StartMode = Starting | Resuming | Overridding -let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxProcessing (cosmosContext, maxWriters) resolveCheckpointStream = async { +let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmosContext, maxWriters) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let conn = connect () let! maxInParallel = Async.StartChild <| establishMax conn @@ -393,7 +393,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead maxPro chunk startPos |> int, conns, conns.Length else 0, [|conn|], spec.stripes+1 - let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","ES"), cosmosIngestionEngine, maxReadAhead, maxProcessing, initialSeriesId, TimeSpan.FromMinutes 1.) + let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","ES"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, TimeSpan.FromMinutes 1.) let post = function | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId | Res.Batch (seriesId, pos, xs) -> diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index bc4ba7c67..3a07e5717 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -25,9 +25,9 @@ module CmdParser = | [] Verbose | [] FromTail | [] MaxPendingBatches of int - | [] MaxProcessing of int | [] MaxWriters of int #if cosmos + | [] MaxProcessing of int | [] MaxDocuments of int | [] ChangeFeedVerbose | [] LeaseCollectionSource of string @@ -53,12 +53,12 @@ module CmdParser = | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Verbose -> "request Verbose Logging. Default: off" | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" - | MaxProcessing _ -> "Maximum number of batches to process concurrently. Default: 16" | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" #if cosmos | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" + | MaxProcessing _ -> "Maximum number of batches to submit concurrently. Default: 16" | LeaseCollectionSource _ ->"specify Collection Name for Leases collection, within `source` connection/database (default: `source`'s `collection` + `-aux`)." | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" @@ -81,12 +81,12 @@ module CmdParser = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.Verbose = a.Contains Verbose member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) - member __.MaxProcessing = a.GetResult(MaxProcessing,16) member __.MaxWriters = a.GetResult(MaxWriters,1024) #if cosmos member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName member __.MaxDocuments = a.TryGetResult MaxDocuments + member __.MaxProcessing = a.GetResult(MaxProcessing,16) member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds #else member __.VerboseConsole = a.Contains VerboseConsole @@ -111,6 +111,7 @@ module CmdParser = | Some sc, None -> x.Source.Discovery, { database = x.Source.Database; collection = sc } | None, Some dc -> x.Destination.Discovery, { database = x.Destination.Database; collection = dc } | Some _, Some _ -> raise (InvalidArguments "LeaseCollectionSource and LeaseCollectionDestination are mutually exclusive - can only store in one database") + Log.Information("Max batches to process concurrently per Range: {maxProcessing}", x.MaxProcessing) Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} with maximum document count limited to {maxDocuments}", x.LeaseId, db.database, db.collection, x.MaxDocuments) if a.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) @@ -128,8 +129,6 @@ module CmdParser = x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}], reading up to {maxPendingBatches} uncommitted batches ahead", x.MinBatchSize, x.StartingBatchSize, x.MaxPendingBatches) - Log.Information("Max batches to process concurrently: {maxProcessing}", - x.MaxProcessing) { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; gorge = x.Gorge; stripes = x.Stripes } #endif @@ -340,7 +339,7 @@ let main argv = CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) (target, args.MaxWriters) - (createSyncHandler (args.MaxPendingBatches*2,args.MaxPendingBatches)) + (createSyncHandler (args.MaxPendingBatches*2,args.MaxProcessing)) #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let catFilter = args.Source.CategoryFilterFunction @@ -361,7 +360,7 @@ let main argv = || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some - EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches args.MaxProcessing (target, args.MaxWriters) resolveCheckpointStream + EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches (target, args.MaxWriters) resolveCheckpointStream #endif |> Async.RunSynchronously 0 From 4a7ba98ab81adc42f2785fba6926dba4e44c399b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 10:19:18 +0100 Subject: [PATCH 247/353] Reorg commandline parsing --- equinox-sync/Sync/EventStoreSource.fs | 38 +++++++++++++-------------- equinox-sync/Sync/Program.fs | 14 +++++----- 2 files changed, 26 insertions(+), 26 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index cdcda0f65..03d4a3a7e 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -319,6 +319,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re | (false, _), Some _ -> dop.Release() |> ignore Log.Warning("No further ingestion work to commence, transitioning to tailing...") + // TODO release connections, reduce DOP, implement stream readers remainder <- None | (false, _), None -> dop.Release() |> ignore @@ -338,13 +339,11 @@ type ReaderSpec = checkpointInterval: TimeSpan /// Delay when reading yields an empty batch tailInterval: TimeSpan - /// Enable initial phase where interleaved reading stripes a 256MB chunk apart attain a balance between good reading speed and not killing the server - gorge: bool - /// Maximum number of striped readers to permit - /// - for gorging, this dictates how many concurrent readers there will be - /// - when tailing, this dictates how many stream readers will be used to perform catchup work on streams that are missing a prefix - /// (e.g. due to not starting from the start of the $all stream, and/or deleting data from the destination store) - stripes: int + /// Specify initial phase where interleaved reading stripes a 256MB chunk apart attain a balance between good reading speed and not killing the server + gorge: int option + /// Maximum number of striped readers to permit when tailing; this dictates how many stream readers will be used to perform catchup work on streams + /// that are missing a prefix (e.g. due to not starting from the start of the $all stream, and/or deleting data from the destination store) + streamReaders: int // TODO /// Initial batch size to use when commencing reading batchSize: int /// Smallest batch size to degrade to in the presence of failures @@ -357,15 +356,15 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo let conn = connect () let! maxInParallel = Async.StartChild <| establishMax conn let! initialCheckpointState = checkpoints.Read - let! max = maxInParallel + let! maxPos = maxInParallel let! startPos = async { let mkPos x = EventStore.ClientAPI.Position(x, 0L) let requestedStartPos = match spec.start with | Absolute p -> mkPos p | Chunk c -> posFromChunk c - | Percentage pct -> posFromPercentage (pct, max) - | TailOrCheckpoint -> max + | Percentage pct -> posFromPercentage (pct, maxPos) + | TailOrCheckpoint -> maxPos | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start let! startMode, startPos, checkpointFreq = async { match initialCheckpointState, requestedStartPos with @@ -380,19 +379,20 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo return Overridding, r, spec.checkpointInterval } log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) checkpointing every {checkpointFreq:n1}m", - startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float max.CommitPosition, + startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) let initialSeriesId, conns, dop = - log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.stripes) - if spec.gorge then - log.Information("Commencing Gorging with {stripes} $all reader stripes covering a 256MB chunk each", spec.stripes) - let extraConns = Seq.init (spec.stripes-1) (ignore >> connect) + log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) + match spec.gorge with + | Some factor -> + log.Information("Commencing Gorging with {stripes} $all reader stripes covering a 256MB chunk each", factor) + let extraConns = Seq.init (factor-1) (ignore >> connect) let conns = [| yield conn; yield! extraConns |] - chunk startPos |> int, conns, conns.Length - else - 0, [|conn|], spec.stripes+1 + chunk startPos |> int, conns, (max (conns.Length) (spec.streamReaders+1)) + | None -> + 0, [|conn|], spec.streamReaders+1 let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","ES"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, TimeSpan.FromMinutes 1.) let post = function | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId @@ -400,5 +400,5 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo let cp = pos.CommitPosition trancheEngine.Submit <| Ingestion.Message.Batch(seriesId, cp, checkpoints.Commit cp, xs) let reader = Reader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, post, spec.tailInterval, dop) - do! reader.Start (initialSeriesId,startPos) max + do! reader.Start (initialSeriesId,startPos) maxPos do! Async.AwaitKeyboardInterrupt() } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 3a07e5717..0734e9009 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -41,8 +41,8 @@ module CmdParser = | [] Position of int64 | [] Chunk of int | [] Percent of float - | [] Gorge - | [] Stripes of int + | [] Gorge of int + | [] StreamReaders of int | [] Tail of intervalS: float #endif | [] Source of ParseResults @@ -72,8 +72,8 @@ module CmdParser = | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | Gorge -> "Parallel readers (instead of reading by stream)" - | Stripes _ -> "number of concurrent readers to run one chunk (256MB) apart. Default: 1" + | Gorge _ -> "Parallel readers (instead of reading by stream)" + | StreamReaders _ -> "number of concurrent readers to run one chunk (256MB) apart. Default: 1" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | Source _ -> "EventStore input parameters." #endif @@ -94,8 +94,8 @@ module CmdParser = member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning member __.StartingBatchSize = a.GetResult(BatchSize,4096) member __.MinBatchSize = a.GetResult(MinBatchSize,512) - member __.Gorge = a.Contains(Gorge) - member __.Stripes = a.GetResult(Stripes,1) + member __.Gorge = a.TryGetResult Gorge + member __.StreamReaders = a.GetResult(StreamReaders,1) member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds member __.CheckpointInterval = TimeSpan.FromHours 1. member __.ForceRestart = a.Contains ForceRestart @@ -130,7 +130,7 @@ module CmdParser = Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}], reading up to {maxPendingBatches} uncommitted batches ahead", x.MinBatchSize, x.StartingBatchSize, x.MaxPendingBatches) { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart - batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; gorge = x.Gorge; stripes = x.Stripes } + batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; gorge = x.Gorge; streamReaders = x.StreamReaders } #endif and [] SourceParameters = #if cosmos From 9de69800761cc4bb2782684558e5a8a003b8738a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 10:48:47 +0100 Subject: [PATCH 248/353] Cosmos pooling connection pooling --- equinox-sync/Sync/CosmosIngester.fs | 8 ++++++-- equinox-sync/Sync/EventStoreSource.fs | 4 ++-- equinox-sync/Sync/Program.fs | 15 +++++++++------ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 9d13aaa08..6767456af 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -7,6 +7,7 @@ open Equinox.Projection2 open Equinox.Projection2.Scheduling open Equinox.Projection.State open Serilog +open System.Threading [] module Writer = @@ -103,7 +104,7 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed | ResultKind.TimedOut -> incr timedOut -let start (log : Serilog.ILogger, cosmosContext, maxWriters, statsInterval) = +let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() @@ -116,9 +117,12 @@ let start (log : Serilog.ILogger, cosmosContext, maxWriters, statsInterval) = // always send at least one event in order to surface the problem and have the stream marked malformed count = 1 || (countBudget >= 0 && bytesBudget >= 0) { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile withinLimits } } + let mutable robin = 0 let attemptWrite batch = async { let trimmed = trim batch - try let! res = Writer.write log cosmosContext trimmed + let index = Interlocked.Increment(&robin) % cosmosContexts.Length + let selectedConnection = cosmosContexts.[index] + try let! res = Writer.write log selectedConnection trimmed let stats = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes return Choice1Of2 (stats,res) with e -> return Choice2Of2 e } diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 03d4a3a7e..0f5558922 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -351,7 +351,7 @@ type ReaderSpec = type StartMode = Starting | Resuming | Overridding -let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmosContext, maxWriters) resolveCheckpointStream = async { +let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmosContexts, maxWriters) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let conn = connect () let! maxInParallel = Async.StartChild <| establishMax conn @@ -382,7 +382,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), cosmosContexts, maxWriters, TimeSpan.FromMinutes 1.) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 0734e9009..2d604fded 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -26,6 +26,7 @@ module CmdParser = | [] FromTail | [] MaxPendingBatches of int | [] MaxWriters of int + | [] MaxCosmosConnections of int #if cosmos | [] MaxProcessing of int | [] MaxDocuments of int @@ -54,6 +55,7 @@ module CmdParser = | Verbose -> "request Verbose Logging. Default: off" | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" + | MaxCosmosConnections _ -> "Size of CosmosDb connection pool to maintain. Default: 1" #if cosmos | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." @@ -72,8 +74,8 @@ module CmdParser = | Position _ -> "EventStore $all Stream Position to commence from" | Chunk _ -> "EventStore $all Chunk to commence from" | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | Gorge _ -> "Parallel readers (instead of reading by stream)" - | StreamReaders _ -> "number of concurrent readers to run one chunk (256MB) apart. Default: 1" + | Gorge _ -> "Request Parallel readers phase during initial catchup, running one chunk (256MB) apart. Default: off" + | StreamReaders _ -> "number of concurrent readers that will fetch a missing stream when in tailing mode. Default: 1. TODO: IMPLEMENT!" | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" | Source _ -> "EventStore input parameters." #endif @@ -82,6 +84,7 @@ module CmdParser = member __.Verbose = a.Contains Verbose member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) member __.MaxWriters = a.GetResult(MaxWriters,1024) + member __.CosmosConnectionPool =a.GetResult(MaxCosmosConnections,1) #if cosmos member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName @@ -313,10 +316,10 @@ let main argv = #else let log,storeLog = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint #endif - let destination = args.Destination.Connect "SyncTemplate" |> Async.RunSynchronously + let destinations = Seq.init args.CosmosConnectionPool (fun i -> args.Destination.Connect (sprintf "%s Pool %d" "SyncTemplate" i)) |> Async.Parallel |> Async.RunSynchronously let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) let resolveCheckpointStream = - let gateway = CosmosGateway(destination, CosmosBatchingPolicy()) + let gateway = CosmosGateway(destinations.[0], CosmosBatchingPolicy()) let store = Equinox.Cosmos.CosmosStore(gateway, colls) let settings = Newtonsoft.Json.JsonSerializerSettings() let codec = Equinox.Codec.NewtonsoftJson.Json.Create settings @@ -325,7 +328,7 @@ let main argv = Equinox.Cosmos.CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve - let target = Equinox.Cosmos.Core.CosmosContext(destination, colls, storeLog) + let targets = destinations |> Array.mapi (fun i x -> Equinox.Cosmos.Core.CosmosContext(x, colls, storeLog.ForContext("PoolId", i))) #if cosmos let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() let auxDiscovery, aux, leaseId, startFromHere, maxDocuments, lagFrequency = args.BuildChangeFeedParams() @@ -360,7 +363,7 @@ let main argv = || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some - EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches (target, args.MaxWriters) resolveCheckpointStream + EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches (targets, args.MaxWriters) resolveCheckpointStream #endif |> Async.RunSynchronously 0 From ec06a501f425bfa5452b3627fb4cde348ce51823 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 13:55:15 +0100 Subject: [PATCH 249/353] Delay presubmits --- equinox-sync/Sync/Projection2.fs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 51fd90383..1dea90c8b 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -521,6 +521,7 @@ module Ingestion = member private __.Pump() = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) + let presubmitInterval = expiredMs (1000L*30L) while not cts.IsCancellationRequested do work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted @@ -533,8 +534,9 @@ module Ingestion = validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch // 3. Forward content for any active streams into processor immediately - let relevantBufferedStreams = streams.Take(scheduler.AllStreams.Contains) - scheduler.AddOpenStreamData(relevantBufferedStreams) + if presubmitInterval () then + let relevantBufferedStreams = streams.Take(scheduler.AllStreams.Contains) + scheduler.AddOpenStreamData(relevantBufferedStreams) // 4. Periodically emit status info stats.TryDump(submissionsMax.State,streams) do! Async.Sleep pumpDelayMs } From b0e67eccfcd028c15f4da87e05078d8b8372cacc Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 14:42:16 +0100 Subject: [PATCH 250/353] Remove false idle state --- equinox-sync/Sync/Projection2.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 1dea90c8b..0ef20a3b9 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -314,7 +314,6 @@ module Scheduling = | true, batch -> ingestPendingBatch stats.Handle batch | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters | Idle -> dispatcherState <- Full; finished <- true - | Slipstreaming when not dispatched -> dispatcherState <- Idle; finished <- true | Slipstreaming -> finished <- true | _ -> () // 3. Supply state to accumulate (and periodically emit) status info From a5d19578d24f48dc83a5de9f2765263deb78daf4 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 14:44:20 +0100 Subject: [PATCH 251/353] Submit all things --- equinox-sync/Sync/Projection2.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 0ef20a3b9..05ec88db7 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -520,7 +520,7 @@ module Ingestion = member private __.Pump() = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) - let presubmitInterval = expiredMs (1000L*30L) + let presubmitInterval = expiredMs (1000L*10L) while not cts.IsCancellationRequested do work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted @@ -534,7 +534,7 @@ module Ingestion = stats.HandleCommitted progressWriter.CommittedEpoch // 3. Forward content for any active streams into processor immediately if presubmitInterval () then - let relevantBufferedStreams = streams.Take(scheduler.AllStreams.Contains) + let relevantBufferedStreams = streams.Take(fun x -> true (*scheduler.AllStreams.Contains*)) scheduler.AddOpenStreamData(relevantBufferedStreams) // 4. Periodically emit status info stats.TryDump(submissionsMax.State,streams) From 373cf4d998268e0dd57682444b4cc815b53c3506 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 15:05:33 +0100 Subject: [PATCH 252/353] Render holding buffers --- equinox-sync/Sync/Projection2.fs | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 05ec88db7..2907a820f 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -399,9 +399,10 @@ module Ingestion = let progCommitFails, progCommits = ref 0, ref 0 let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (available,maxDop) = - log.Information("Holding Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", - !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) + let dumpStats (available,maxDop) (readingAhead,ready) = + let inline rd (xs : IDictionary>) = seq { for x in xs -> x.Key, x.Value.Count } |> Seq.sortByDescending snd + log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Reading {@reading} Ready {@ready} Submissions {active}/{writers}", + !cycles, !batchesPended, !streamsPended, !eventsPended, rd readingAhead, rd ready, available, maxDop) cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with @@ -434,10 +435,10 @@ module Ingestion = pendingBatchCount <- pendingBatches member __.HandleCommitted epoch = comittedEpoch <- epoch - member __.TryDump((available,maxDop),streams : Streams) = + member __.TryDump((available,maxDop),streams : Streams,readingAhead,ready) = incr cycles if statsDue () then - dumpStats (available,maxDop) + dumpStats (available,maxDop) (readingAhead,ready) streams.Dump log and [] private InternalMessage = @@ -537,7 +538,7 @@ module Ingestion = let relevantBufferedStreams = streams.Take(fun x -> true (*scheduler.AllStreams.Contains*)) scheduler.AddOpenStreamData(relevantBufferedStreams) // 4. Periodically emit status info - stats.TryDump(submissionsMax.State,streams) + stats.TryDump(submissionsMax.State,streams,readingAhead,ready) do! Async.Sleep pumpDelayMs } /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes From dbf2cc2dbc690a4385f31bd1749fb275538b5314 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 15:11:48 +0100 Subject: [PATCH 253/353] reorder log output --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 2907a820f..1bd219564 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -400,7 +400,7 @@ module Ingestion = let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (available,maxDop) (readingAhead,ready) = - let inline rd (xs : IDictionary>) = seq { for x in xs -> x.Key, x.Value.Count } |> Seq.sortByDescending snd + let inline rd (xs : IDictionary>) = seq { for x in xs -> x.Key, x.Value.Count } |> Seq.sortBy fst log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Reading {@reading} Ready {@ready} Submissions {active}/{writers}", !cycles, !batchesPended, !streamsPended, !eventsPended, rd readingAhead, rd ready, available, maxDop) cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 From 5d09abb8a73e46e6a0454a8a631ef2d75103b5ce Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 15:22:49 +0100 Subject: [PATCH 254/353] Render preloading separately --- equinox-sync/Sync/Projection2.fs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 1bd219564..0264091e4 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -400,9 +400,12 @@ module Ingestion = let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (available,maxDop) (readingAhead,ready) = - let inline rd (xs : IDictionary>) = seq { for x in xs -> x.Key, x.Value.Count } |> Seq.sortBy fst - log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Reading {@reading} Ready {@ready} Submissions {active}/{writers}", - !cycles, !batchesPended, !streamsPended, !eventsPended, rd readingAhead, rd ready, available, maxDop) + let mutable buffered = 0 + let count (xs : IDictionary>) = seq { for x in xs do buffered <- buffered + x.Value.Count; yield x.Key, x.Value.Count } |> Seq.sortBy fst |> Seq.toArray + let ahead, ready = count readingAhead, count ready + if buffered <> 0 then log.Information("Preloading {buffered} Reading {@reading} Ready {@ready}", ahead, ready) + log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", + !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with From 29c6ec1b6c154cfa87f565e163f240b83befa819 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 15:27:43 +0100 Subject: [PATCH 255/353] Fix buffered message --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 0264091e4..7701ca8ab 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -403,7 +403,7 @@ module Ingestion = let mutable buffered = 0 let count (xs : IDictionary>) = seq { for x in xs do buffered <- buffered + x.Value.Count; yield x.Key, x.Value.Count } |> Seq.sortBy fst |> Seq.toArray let ahead, ready = count readingAhead, count ready - if buffered <> 0 then log.Information("Preloading {buffered} Reading {@reading} Ready {@ready}", ahead, ready) + if buffered <> 0 then log.Information("Holding {buffered} Reading {@reading} Ready {@ready}", buffered, ahead, ready) log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 From 9c3de1ad96339f5ab81a3477438a5a21fb1f7918 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 16:00:09 +0100 Subject: [PATCH 256/353] 512MB --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 6767456af..1f7d36190 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,7 +105,7 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = - let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 + let cosmosPayloadLimit = (*2 * 1024*) 512 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = From 90e95ba6dd710f1fccdf629adcefd79c9d0024a9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 16:49:20 +0100 Subject: [PATCH 257/353] Add catch to buffer thread --- equinox-sync/Sync/Projection2.fs | 41 ++++++++++++++++---------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 7701ca8ab..9106fb15e 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -400,13 +400,13 @@ module Ingestion = let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) let dumpStats (available,maxDop) (readingAhead,ready) = + log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", + !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) + cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 let mutable buffered = 0 let count (xs : IDictionary>) = seq { for x in xs do buffered <- buffered + x.Value.Count; yield x.Key, x.Value.Count } |> Seq.sortBy fst |> Seq.toArray let ahead, ready = count readingAhead, count ready if buffered <> 0 then log.Information("Holding {buffered} Reading {@reading} Ready {@ready}", buffered, ahead, ready) - log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", - !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 if !progCommitFails <> 0 || !progCommits <> 0 then match comittedEpoch with | None -> @@ -526,23 +526,24 @@ module Ingestion = Async.Start(progressWriter.Pump(), cts.Token) let presubmitInterval = expiredMs (1000L*10L) while not cts.IsCancellationRequested do - work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && submissionsMax.HasCapacity do - // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) - do! submissionsMax.Await() - scheduler.Submit(pending.Dequeue()) - // 2. Update any progress into the stats - stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) - validatedPos |> Option.iter progressWriter.Post - stats.HandleCommitted progressWriter.CommittedEpoch - // 3. Forward content for any active streams into processor immediately - if presubmitInterval () then - let relevantBufferedStreams = streams.Take(fun x -> true (*scheduler.AllStreams.Contains*)) - scheduler.AddOpenStreamData(relevantBufferedStreams) - // 4. Periodically emit status info - stats.TryDump(submissionsMax.State,streams,readingAhead,ready) - do! Async.Sleep pumpDelayMs } + try work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) + // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted + while pending.Count <> 0 && submissionsMax.HasCapacity do + // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) + do! submissionsMax.Await() + scheduler.Submit(pending.Dequeue()) + // 2. Update any progress into the stats + stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) + validatedPos |> Option.iter progressWriter.Post + stats.HandleCommitted progressWriter.CommittedEpoch + // 3. Forward content for any active streams into processor immediately + if presubmitInterval () then + let relevantBufferedStreams = streams.Take(fun x -> true (*scheduler.AllStreams.Contains*)) + scheduler.AddOpenStreamData(relevantBufferedStreams) + // 4. Periodically emit status info + stats.TryDump(submissionsMax.State,streams,readingAhead,ready) + do! Async.Sleep pumpDelayMs + with e -> log.Error(e,"Buffer thread exception") } /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval) = From 4ffd99ec65a1490415fafee0d958eecdede131e5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 17:16:15 +0100 Subject: [PATCH 258/353] Limit count in holding loop --- equinox-sync/Sync/Projection2.fs | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 9106fb15e..70bd5c2cb 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -526,20 +526,24 @@ module Ingestion = Async.Start(progressWriter.Pump(), cts.Token) let presubmitInterval = expiredMs (1000L*10L) while not cts.IsCancellationRequested do - try work |> ConcurrentQueue.drain (fun x -> handle x; stats.Handle x) - // 1. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && submissionsMax.HasCapacity do - // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) - do! submissionsMax.Await() - scheduler.Submit(pending.Dequeue()) - // 2. Update any progress into the stats + try let mutable itemLimit = 4096 + while itemLimit > 0 do + match work.TryDequeue() with + | true, x -> handle x; stats.Handle x; itemLimit <- itemLimit - 1 + | false, _ -> itemLimit <- 0 + // 1. Update any progress into the stats stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch - // 3. Forward content for any active streams into processor immediately + // 2. Forward content for any active streams into processor immediately if presubmitInterval () then let relevantBufferedStreams = streams.Take(fun x -> true (*scheduler.AllStreams.Contains*)) scheduler.AddOpenStreamData(relevantBufferedStreams) + // 3. Submit to ingester until read queue, tranche limit or ingester limit exhausted + while pending.Count <> 0 && submissionsMax.HasCapacity do + // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) + do! submissionsMax.Await() + scheduler.Submit(pending.Dequeue()) // 4. Periodically emit status info stats.TryDump(submissionsMax.State,streams,readingAhead,ready) do! Async.Sleep pumpDelayMs From 018f6c99a408797285584471771cd88d2541dbc7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 17:20:47 +0100 Subject: [PATCH 259/353] Raise thread limits --- equinox-sync/Sync/Program.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2d604fded..3729c10c2 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -310,7 +310,8 @@ module Logging = [] let main argv = - try let args = CmdParser.parse argv + try if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") + let args = CmdParser.parse argv #if cosmos let log,storeLog = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint #else From d743dcc4aad729148cca7bfa63f58164baf9081a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 21:34:51 +0100 Subject: [PATCH 260/353] remove threads, limit --- equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/Program.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 1f7d36190..6767456af 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,7 +105,7 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = - let cosmosPayloadLimit = (*2 * 1024*) 512 * 1024 - (*fudge*)4096 + let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 3729c10c2..2041bc5c1 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -310,7 +310,7 @@ module Logging = [] let main argv = - try if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") + try //if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") let args = CmdParser.parse argv #if cosmos let log,storeLog = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint From 98470c62af4042f8bf02dd8e6f08ad09e08d1fea Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 22:03:33 +0100 Subject: [PATCH 261/353] default to cosmos --- equinox-sync/Sync/Program.fs | 2 +- equinox-sync/Sync/Sync.fsproj | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2041bc5c1..e7cac7d57 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -342,7 +342,7 @@ let main argv = #endif CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) - (target, args.MaxWriters) + (targets, args.MaxWriters) (createSyncHandler (args.MaxPendingBatches*2,args.MaxProcessing)) #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 5ff15a172..94b803ecc 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,7 +4,7 @@ Exe netcoreapp2.1 5 - $(DefineConstants);cosmos_ + $(DefineConstants);cosmos From 3cf5998642793a82a2c1cdf061a081f3246a5a6c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 22:10:38 +0100 Subject: [PATCH 262/353] fix default read --- equinox-sync/Sync/Program.fs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index e7cac7d57..a70e66db9 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -53,10 +53,10 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" | Verbose -> "request Verbose Logging. Default: off" - | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" | MaxCosmosConnections _ -> "Size of CosmosDb connection pool to maintain. Default: 1" #if cosmos + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 256" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" @@ -66,6 +66,7 @@ module CmdParser = | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | Source _ -> "CosmosDb input parameters." #else + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" | VerboseConsole -> "request Verbose Console Logging. Default: off" | FromTail -> "Start the processing from the Tail" | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." @@ -82,16 +83,17 @@ module CmdParser = and Arguments(a : ParseResults) = member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None member __.Verbose = a.Contains Verbose - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) member __.MaxWriters = a.GetResult(MaxWriters,1024) member __.CosmosConnectionPool =a.GetResult(MaxCosmosConnections,1) #if cosmos + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,256) member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName member __.MaxDocuments = a.TryGetResult MaxDocuments member __.MaxProcessing = a.GetResult(MaxProcessing,16) member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds #else + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) member __.VerboseConsole = a.Contains VerboseConsole member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning @@ -343,7 +345,7 @@ let main argv = CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) (targets, args.MaxWriters) - (createSyncHandler (args.MaxPendingBatches*2,args.MaxProcessing)) + (createSyncHandler (args.MaxPendingBatches,args.MaxProcessing/2)) #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let catFilter = args.Source.CategoryFilterFunction From 40d3bc11ce02d48f35397bb079a67b05c622c160 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 22:15:22 +0100 Subject: [PATCH 263/353] Fix maxprocessing --- equinox-sync/Sync/Program.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index a70e66db9..5e73593d0 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -56,11 +56,11 @@ module CmdParser = | MaxWriters _ -> "Maximum number of concurrent writes to target permitted. Default: 512" | MaxCosmosConnections _ -> "Size of CosmosDb connection pool to maintain. Default: 1" #if cosmos - | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 256" + | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 32" + | MaxProcessing _ -> "Maximum number of batches to submit concurrently. Default: 16" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" - | MaxProcessing _ -> "Maximum number of batches to submit concurrently. Default: 16" | LeaseCollectionSource _ ->"specify Collection Name for Leases collection, within `source` connection/database (default: `source`'s `collection` + `-aux`)." | LeaseCollectionDestination _ -> "specify Collection Name for Leases collection, within [destination] `cosmos` connection/database (default: defined relative to `source`'s `collection`)." | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" @@ -86,11 +86,11 @@ module CmdParser = member __.MaxWriters = a.GetResult(MaxWriters,1024) member __.CosmosConnectionPool =a.GetResult(MaxCosmosConnections,1) #if cosmos - member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,256) + member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,32) + member __.MaxProcessing = a.GetResult(MaxProcessing,16) member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose member __.LeaseId = a.GetResult ConsumerGroupName member __.MaxDocuments = a.TryGetResult MaxDocuments - member __.MaxProcessing = a.GetResult(MaxProcessing,16) member __.LagFrequency = a.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds #else member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) @@ -345,7 +345,7 @@ let main argv = CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) (targets, args.MaxWriters) - (createSyncHandler (args.MaxPendingBatches,args.MaxProcessing/2)) + (createSyncHandler (args.MaxPendingBatches,args.MaxProcessing)) #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let catFilter = args.Source.CategoryFilterFunction From 81a3aa964831c5a657a4dc1af185de3c46f3843b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 22:20:44 +0100 Subject: [PATCH 264/353] More cosmos arg fixes --- equinox-sync/Sync/CosmosSource.fs | 2 +- equinox-sync/Sync/Program.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index fe5127bb0..abbe8f892 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -40,7 +40,7 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection let! _feedEventHost = ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, - createObserver = createRangeProjector cosmosIngester, ?cfBatchSize = maxDocuments, ?reportLagAndAwaitNextEstimation = maybeLogLag) + createObserver = createRangeProjector cosmosIngester, ?reportLagAndAwaitNextEstimation = maybeLogLag, cfBatchSize = defaultArg maxDocuments 999999) do! Async.AwaitKeyboardInterrupt() cosmosIngester.Stop() } diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 5e73593d0..2a1b5f532 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -116,7 +116,7 @@ module CmdParser = | Some sc, None -> x.Source.Discovery, { database = x.Source.Database; collection = sc } | None, Some dc -> x.Destination.Discovery, { database = x.Destination.Database; collection = dc } | Some _, Some _ -> raise (InvalidArguments "LeaseCollectionSource and LeaseCollectionDestination are mutually exclusive - can only store in one database") - Log.Information("Max batches to process concurrently per Range: {maxProcessing}", x.MaxProcessing) + Log.Information("Max read backlog: {maxPending}, of which up to {maxProcessing} processing", x.MaxPendingBatches, x.MaxProcessing) Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} with maximum document count limited to {maxDocuments}", x.LeaseId, db.database, db.collection, x.MaxDocuments) if a.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) From 2899953b0c9c2cedd502a1cfc75a5a757fceafc8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 22:46:03 +0100 Subject: [PATCH 265/353] 512MB again --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 6767456af..1f7d36190 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,7 +105,7 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = - let cosmosPayloadLimit = 2 * 1024 * 1024 - (*fudge*)4096 + let cosmosPayloadLimit = (*2 * 1024*) 512 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = From 3bb8318c32da929ff35731337040cfbf26b649cd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 23:28:46 +0100 Subject: [PATCH 266/353] 256K --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 1f7d36190..f44cb8fc8 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,7 +105,7 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = - let cosmosPayloadLimit = (*2 * 1024*) 512 * 1024 - (*fudge*)4096 + let cosmosPayloadLimit = (*2 * 1024*) 256 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = From 53e4a884f90946617b658a57439d95ab9373d4f7 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 23:29:38 +0100 Subject: [PATCH 267/353] 16384 --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index f44cb8fc8..8820c802d 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -109,7 +109,7 @@ let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterv let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = - let mutable count, countBudget, bytesBudget = 0, 4096, cosmosPayloadLimit + let mutable count, countBudget, bytesBudget = 0, (*4096*)16384, cosmosPayloadLimit let withinLimits (y : Equinox.Codec.IEvent) = count <- count + 1 countBudget <- countBudget - 1 From 7949c67daa1ca9f6d4f7dead4f400922bc664575 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 9 May 2019 23:47:40 +0100 Subject: [PATCH 268/353] more threads --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2a1b5f532..544226f1d 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -312,7 +312,7 @@ module Logging = [] let main argv = - try //if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") + try if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") let args = CmdParser.parse argv #if cosmos let log,storeLog = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint From d9863849c3188ae904cb5b9959b9f8ed0e65e8ed Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 00:01:33 +0100 Subject: [PATCH 269/353] 64K blocks --- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 8820c802d..41d174d75 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,11 +105,11 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = - let cosmosPayloadLimit = (*2 * 1024*) 256 * 1024 - (*fudge*)4096 + let cosmosPayloadLimit = 65536 // 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = - let mutable count, countBudget, bytesBudget = 0, (*4096*)16384, cosmosPayloadLimit + let mutable count, countBudget, bytesBudget = 0, (*4096*)65536, cosmosPayloadLimit let withinLimits (y : Equinox.Codec.IEvent) = count <- count + 1 countBudget <- countBudget - 1 From 63eeae98663f7a8c22922c8e17d7c4292965a86f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 06:25:56 +0100 Subject: [PATCH 270/353] zero len writes --- equinox-sync/Sync/CosmosSource.fs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index abbe8f892..b8d6d3904 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -96,5 +96,12 @@ let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq for e in DocumentParser.enumEvents changeFeedDocument do // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level if catFilter (category e.stream) then - yield e } + let e2 = + { new Equinox.Codec.IEvent<_> with + member __.Data = null + member __.Meta = null + member __.EventType = e.event.EventType + member __.Timestamp = e.event.Timestamp } + yield { e with event = e2 } +} //#endif \ No newline at end of file From 3b6687eb2cd359b8caaf0c863ab4b0fcb8109da3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 09:50:39 +0100 Subject: [PATCH 271/353] Use ConcurrentBags --- equinox-sync/Sync/Projection2.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 70bd5c2cb..87771300d 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -217,7 +217,7 @@ module Scheduling = /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher<'R>(maxDop) = - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) + let work = new BlockingCollection<_>(ConcurrentBag<_>()) let result = Event<'R>() let dop = new Sem(maxDop) let dispatch work = async { @@ -244,7 +244,7 @@ module Scheduling = type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() - let work = ConcurrentQueue>() + let work = ConcurrentBag>() let pending = ConcurrentQueue<_*StreamItem[]>() let streams = StreamStates() let progressState = Progress.State() @@ -256,7 +256,7 @@ module Scheduling = let tryDrainResults feedStats = let mutable worked, more = false, true while more do - match work.TryDequeue() with + match work.TryTake() with | false, _ -> more <- false | true, x -> match x with @@ -298,7 +298,7 @@ module Scheduling = feedStats <| Added (reqs.Count,skipCount,count) member private __.Pump(stats : Stats<'R>) = async { - use _ = dispatcher.Result.Subscribe(Result >> work.Enqueue) + use _ = dispatcher.Result.Subscribe(Result >> work.Add) Async.Start(dispatcher.Pump(), cts.Token) while not cts.IsCancellationRequested do let mutable idle, dispatcherState, finished = true, Idle, false @@ -334,7 +334,7 @@ module Scheduling = pending.Enqueue (markCompleted, items) member __.AddOpenStreamData(events) = - work.Enqueue <| Merge events + work.Add <| Merge events member __.AllStreams = streams.All From 3b12fff8373a62e59148ebd0e0d128df07bdf154 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 10:58:34 +0100 Subject: [PATCH 272/353] Tune concurrency --- equinox-sync/Sync/Projection2.fs | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 87771300d..dde42cca5 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -217,7 +217,8 @@ module Scheduling = /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher<'R>(maxDop) = - let work = new BlockingCollection<_>(ConcurrentBag<_>()) + // Using a Queue as a) the ordering is more correct, favoring more important work b) we are adding from many threads so no value in ConcurrentBag'sthread-affinity + let work = new BlockingCollection<_>(ConcurrentQueue<_>()) let result = Event<'R>() let dop = new Sem(maxDop) let dispatch work = async { @@ -244,8 +245,8 @@ module Scheduling = type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() - let work = ConcurrentBag>() - let pending = ConcurrentQueue<_*StreamItem[]>() + let work = ConcurrentStack>() // dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better + let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering let streams = StreamStates() let progressState = Progress.State() @@ -253,12 +254,14 @@ module Scheduling = match streamState.write, item.index + 1L with | Some cw, required when cw >= required -> 0, 1 | _ -> 1, 0 + static let workLocalBuffer = Array.zeroCreate 200 let tryDrainResults feedStats = let mutable worked, more = false, true while more do - match work.TryTake() with - | false, _ -> more <- false - | true, x -> + let c = work.TryPopRange(workLocalBuffer) + if c = 0 then more <- false else worked <- true + for i in 0..c do + let x = workLocalBuffer.[i] match x with | Added _ -> () // Only processed in Stats | Merge events -> for e in events do streams.InternalMerge(e.Key,e.Value) @@ -269,7 +272,6 @@ module Scheduling = progressState.MarkStreamProgress(stream,index) streams.MarkCompleted(stream,index) feedStats x - worked <- true worked let tryFillDispatcher includeSlipstreamed = async { let mutable hasCapacity, dispatched = dispatcher.HasCapacity, false @@ -298,7 +300,7 @@ module Scheduling = feedStats <| Added (reqs.Count,skipCount,count) member private __.Pump(stats : Stats<'R>) = async { - use _ = dispatcher.Result.Subscribe(Result >> work.Add) + use _ = dispatcher.Result.Subscribe(Result >> work.Push) Async.Start(dispatcher.Pump(), cts.Token) while not cts.IsCancellationRequested do let mutable idle, dispatcherState, finished = true, Idle, false @@ -334,7 +336,7 @@ module Scheduling = pending.Enqueue (markCompleted, items) member __.AddOpenStreamData(events) = - work.Add <| Merge events + work.Push <| Merge events member __.AllStreams = streams.All @@ -464,7 +466,7 @@ module Ingestion = type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxRead, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = let cts = new CancellationTokenSource() let pumpDelayMs = defaultArg pumpDelayMs 5 - let work = ConcurrentQueue() + let work = ConcurrentQueue() // Queue as need ordering semantically let readMax = new Sem(maxRead) let submissionsMax = new Sem(maxSubmissions) let streams = Streams() From 85441fa8b6cb3a83018ef1072f44cbf70c641c38 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 11:02:37 +0100 Subject: [PATCH 273/353] Make log context Ingest --- equinox-sync/Sync/EventStoreSource.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 0f5558922..8e51b4c20 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -382,7 +382,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Cosmos"), cosmosContexts, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Ingest"), cosmosContexts, maxWriters, TimeSpan.FromMinutes 1.) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with From eea552badab9bf1b79c9af364facef20bfa7539d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 11:07:39 +0100 Subject: [PATCH 274/353] Off by one! --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index dde42cca5..8b6181120 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -260,7 +260,7 @@ module Scheduling = while more do let c = work.TryPopRange(workLocalBuffer) if c = 0 then more <- false else worked <- true - for i in 0..c do + for i in 0..c-1 do let x = workLocalBuffer.[i] match x with | Added _ -> () // Only processed in Stats From 7586c0ed36fe398b1ae6372be12adda8484be3ec Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 11:16:15 +0100 Subject: [PATCH 275/353] Tidy ingest; better buffer size --- equinox-sync/Sync/Projection2.fs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 8b6181120..2e7c788c9 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -250,11 +250,7 @@ module Scheduling = let streams = StreamStates() let progressState = Progress.State() - let validVsSkip (streamState : StreamState) (item : StreamItem) = - match streamState.write, item.index + 1L with - | Some cw, required when cw >= required -> 0, 1 - | _ -> 1, 0 - static let workLocalBuffer = Array.zeroCreate 200 + static let workLocalBuffer = Array.zeroCreate 1024 let tryDrainResults feedStats = let mutable worked, more = false, true while more do @@ -286,6 +282,10 @@ module Scheduling = hasCapacity <- succeeded return hasCapacity, dispatched } let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = + let inline validVsSkip (streamState : StreamState) (item : StreamItem) = + match streamState.write, item.index + 1L with + | Some cw, required when cw >= required -> 0, 1 + | _ -> 1, 0 let reqs = Dictionary() let mutable count, skipCount = 0, 0 for item in items do From db64cc29fa4d17da3e244a9be6b2078220bcbf7d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 12:14:55 +0100 Subject: [PATCH 276/353] Tidy stats --- equinox-sync/Sync/CosmosIngester.fs | 8 ++--- equinox-sync/Sync/CosmosSource.fs | 2 +- equinox-sync/Sync/EventStoreSource.fs | 2 +- equinox-sync/Sync/Program.fs | 2 +- equinox-sync/Sync/Projection2.fs | 49 +++++++++++++++++---------- 5 files changed, 38 insertions(+), 25 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 41d174d75..830b96312 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -64,8 +64,8 @@ module Writer = | ResultKind.RateLimited | ResultKind.TimedOut | ResultKind.Other -> false | ResultKind.TooLarge | ResultKind.Malformed -> true -type Stats(log : ILogger, statsInterval) = - inherit Stats<(int*int)*Writer.Result>(log, statsInterval) +type Stats(log : ILogger, statsInterval, statesInterval) = + inherit Stats<(int*int)*Writer.Result>(log, statsInterval, statesInterval) let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 let mutable events, bytes = 0, 0L @@ -104,7 +104,7 @@ type Stats(log : ILogger, statsInterval) = | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed | ResultKind.TimedOut -> incr timedOut -let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterval) = +let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInterval, statesInterval)) = let cosmosPayloadLimit = 65536 // 2 * 1024 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() @@ -138,5 +138,5 @@ let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, statsInterv let _stream, { write = wp } = applyResultToStreamState res Writer.logTo writerResultLog (stream,res) wp - let projectionAndCosmosStats = Stats(log.ForContext(), statsInterval) + let projectionAndCosmosStats = Stats(log.ForContext(), statsInterval, statesInterval) Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress) \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index b8d6d3904..717a2810f 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -36,7 +36,7 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag - let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) let! _feedEventHost = ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 8e51b4c20..96fc85c3f 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -382,7 +382,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Ingest"), cosmosContexts, maxWriters, TimeSpan.FromMinutes 1.) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Ingest"), cosmosContexts, maxWriters, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 544226f1d..2a1b5f532 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -312,7 +312,7 @@ module Logging = [] let main argv = - try if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") + try //if not (System.Threading.ThreadPool.SetMaxThreads(512,512)) then raise (CmdParser.InvalidArguments "Could not set thread limits") let args = CmdParser.parse argv #if cosmos let log,storeLog = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 2e7c788c9..2e8721385 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -183,16 +183,21 @@ module Scheduling = type BufferState = Idle | Full | Slipstreaming /// Gathers stats pertaining to the core projection/ingestion activity - type Stats<'R>(log : ILogger, statsInterval : TimeSpan) = - let cycles, states, batchesPended, streamsPended, eventsSkipped, eventsPended, resultCompleted, resultExn = ref 0, CatStats(), ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) + type Stats<'R>(log : ILogger, statsInterval : TimeSpan, stateInterval : TimeSpan) = + let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 + let merges, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0 + let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) let dumpStats (used,maxDop) pendingCount = - log.Information("Projection Cycles {cycles} States {@states} Busy {busy}/{processors} Ingested {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Pending {pending} Completed {completed} Exceptions {exns}", - !cycles, states.StatsDescending, used, maxDop, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, pendingCount, !resultCompleted, !resultExn) - cycles := 0; states.Clear(); batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; resultCompleted := 0; resultExn:= 0 + log.Information("Projection Cycles {cycles}/{fullCycles} States {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", + !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) + cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 + log.Information("Ingestions {batches} {streams:n0}s {events:n0}-{skipped:n0}e Merged {merges} Pending {pending}", + !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingCount) + batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function - | Merge _ -> () + | Merge _ -> + incr merges | Added (streams, skipped, events) -> incr batchesPended streamsPended := !streamsPended + streams @@ -202,18 +207,24 @@ module Scheduling = incr resultCompleted | Result (_stream, Choice2Of2 _) -> incr resultExn - member __.TryDump(state,(used,max),streams : StreamStates, pendingCount) = + member __.DumpStats((used,max), pendingCount) = incr cycles - states.Ingest(string state) - let due = statsDue () - if due then + if statsDue () then dumpStats (used,max) pendingCount __.DumpExtraStats() + member __.TryDumpState(state, streams : StreamStates) = + incr fullCycles + states.Ingest(string state) + let due = stateDue () + if due then + __.DumpExtraState() streams.Dump log due /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () + abstract DumpExtraState : unit -> unit + default __.DumpExtraState () = () /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher<'R>(maxDop) = @@ -259,7 +270,7 @@ module Scheduling = for i in 0..c-1 do let x = workLocalBuffer.[i] match x with - | Added _ -> () // Only processed in Stats + | Added _ -> () // Only processed in Stats (and actually never enters this queue) | Merge events -> for e in events do streams.InternalMerge(e.Key,e.Value) | Result (stream,res) -> match interpretProgress streams stream res with @@ -318,10 +329,12 @@ module Scheduling = | Idle -> dispatcherState <- Full; finished <- true | Slipstreaming -> finished <- true | _ -> () - // 3. Supply state to accumulate (and periodically emit) status info - if stats.TryDump(dispatcherState,dispatcher.State,streams,pending.Count) then idle <- false - // 4. Do a minimal sleep so we don't run completely hot when empty - if idle then do! Async.Sleep sleepIntervalMs } + // This loop can take a long time; attempt logging of stats per iteration + stats.DumpStats(dispatcher.State,pending.Count) + // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently + if not (stats.TryDumpState(dispatcherState,streams)) && not idle then + // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) + do! Async.Sleep sleepIntervalMs } static member Start<'R>(stats, projectorDop, project, interpretProgress) = let dispatcher = Dispatcher(projectorDop) @@ -345,7 +358,7 @@ module Scheduling = type Projector = - static member Start(log, projectorDop, project : StreamSpan -> Async, ?statsInterval) = + static member Start(log, projectorDop, project : StreamSpan -> Async, ?statsInterval, ?statesInterval) = let project (_maybeWritePos, batch) = async { try let! count = project batch return Choice1Of2 (batch.span.index + int64 count) @@ -353,7 +366,7 @@ type Projector = let interpretProgress _streams _stream = function | Choice1Of2 index -> Some index | Choice2Of2 _ -> None - let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.), defaultArg statesInterval (TimeSpan.FromMinutes 5.)) Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress) module Ingestion = From 8281ec2eb759cecae3761160a285686f4401dfea Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 12:20:03 +0100 Subject: [PATCH 277/353] 512K --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 830b96312..025993280 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,7 +105,7 @@ type Stats(log : ILogger, statsInterval, statesInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInterval, statesInterval)) = - let cosmosPayloadLimit = 65536 // 2 * 1024 * 1024 - (*fudge*)4096 + let cosmosPayloadLimit = 512 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = From 490a05fe97e154eee631632dca60cf9d2f15d056 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 12:22:21 +0100 Subject: [PATCH 278/353] Fix budgets at 16K events or 512KB --- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 025993280..a2780c58d 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -105,11 +105,11 @@ type Stats(log : ILogger, statsInterval, statesInterval) = | ResultKind.TimedOut -> incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInterval, statesInterval)) = - let cosmosPayloadLimit = 512 * 1024 - (*fudge*)4096 let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = - let mutable count, countBudget, bytesBudget = 0, (*4096*)65536, cosmosPayloadLimit + let mutable countBudget, bytesBudget = 16384, 512 * 1024 - (*fudge*)4096 + let mutable count = 0 let withinLimits (y : Equinox.Codec.IEvent) = count <- count + 1 countBudget <- countBudget - 1 From e1961ecc8b4f07cf9fd7c581dd24f5e1cb19d5c8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 13:00:08 +0100 Subject: [PATCH 279/353] Add Stream counts --- equinox-sync/Sync/CosmosIngester.fs | 30 +++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index a2780c58d..f1e90d660 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -8,6 +8,7 @@ open Equinox.Projection2.Scheduling open Equinox.Projection.State open Serilog open System.Threading +open System.Collections.Generic [] module Writer = @@ -69,26 +70,30 @@ type Stats(log : ILogger, statsInterval, statesInterval) = let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 let mutable events, bytes = 0, 0L - let badCats = CatStats() + let badCats, failStreams, okStreams, toStreams, rlStreams, tlStreams, mfStreams, oStreams = + CatStats(), HashSet(), HashSet(), HashSet(), HashSet(), HashSet(), HashSet(), HashSet() override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Completed {completed:n0} {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", - results, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) - resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L + log.Information("Completed {completed:n0}r {mb:n0}MB {streams:n0}s {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + results, mb bytes, okStreams.Count, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - log.Warning("Exceptions {rateLimited:n0} rate-limited, {timedOut:n0} timed out, {tooLarge} too large, {malformed} malformed, {other} other", - !rateLimited, !timedOut, !tooLarge, !malformed, !resultExnOther) + log.Warning("Exceptions {streams:n0} s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s Too large {tooLarge:n0}e {@tlStreams} Malformed {malformed:n0} {@mfStrea,s} Other {other:n0} {@oStreams}", + failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count, !tooLarge, tlStreams, !malformed, mfStreams, !resultExnOther, oStreams) rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0 if badCats.Any then log.Error("Malformed categories {@badCats}", badCats.StatsDescending); badCats.Clear() + toStreams.Clear(); rlStreams.Clear(); tlStreams.Clear(); mfStreams.Clear(); oStreams.Clear() Equinox.Cosmos.Store.Log.InternalMetrics.dump log override __.Handle message = base.Handle message + let inline adds x (set:HashSet<_>) = set.Add x |> ignore match message with | Merge _ | Added _ -> () - | Result (_stream, Choice1Of2 ((es,bs),r)) -> + | Result (stream, Choice1Of2 ((es,bs),r)) -> + okStreams.Add stream |> ignore events <- events + es bytes <- bytes + int64 bs match r with @@ -97,12 +102,13 @@ type Stats(log : ILogger, statsInterval, statesInterval) = | Writer.Result.PartialDuplicate _ -> incr resultPartialDup | Writer.Result.PrefixMissing _ -> incr resultPrefix | Result (stream, Choice2Of2 exn) -> + failStreams.Add stream |> ignore match Writer.classify exn with - | ResultKind.Other -> incr resultExnOther - | ResultKind.RateLimited -> incr rateLimited - | ResultKind.TooLarge -> category stream |> badCats.Ingest; incr tooLarge - | ResultKind.Malformed -> category stream |> badCats.Ingest; incr malformed - | ResultKind.TimedOut -> incr timedOut + | ResultKind.Other -> adds stream oStreams; incr resultExnOther + | ResultKind.RateLimited -> adds stream rlStreams; incr rateLimited + | ResultKind.TooLarge -> category stream |> badCats.Ingest; adds stream tlStreams; incr tooLarge + | ResultKind.Malformed -> category stream |> badCats.Ingest; adds stream mfStreams; incr malformed + | ResultKind.TimedOut -> adds stream toStreams; incr timedOut let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInterval, statesInterval)) = let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 From 9c22045979476fb4de2fcd3abd818e8af2fbaa5a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 13:05:51 +0100 Subject: [PATCH 280/353] Fix typo --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index f1e90d660..77d6281d7 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -79,7 +79,7 @@ type Stats(log : ILogger, statsInterval, statesInterval) = results, mb bytes, okStreams.Count, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - log.Warning("Exceptions {streams:n0} s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s Too large {tooLarge:n0}e {@tlStreams} Malformed {malformed:n0} {@mfStrea,s} Other {other:n0} {@oStreams}", + log.Warning("Exceptions {streams:n0} s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s Too large {tooLarge:n0}e {@tlStreams} Malformed {malformed:n0} {@mfStreams} Other {other:n0} {@oStreams}", failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count, !tooLarge, tlStreams, !malformed, mfStreams, !resultExnOther, oStreams) rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0 if badCats.Any then log.Error("Malformed categories {@badCats}", badCats.StatsDescending); badCats.Clear() From f6161e082c63b01c4874c95f52707ab0c20cffc9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 14:15:25 +0100 Subject: [PATCH 281/353] Tidy results breakdown --- equinox-sync/Sync/CosmosIngester.fs | 41 ++++++++++++++------------- equinox-sync/Sync/EventStoreSource.fs | 4 +-- equinox-sync/Sync/Projection2.fs | 2 +- 3 files changed, 24 insertions(+), 23 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 77d6281d7..dccd7371a 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -67,33 +67,34 @@ module Writer = type Stats(log : ILogger, statsInterval, statesInterval) = inherit Stats<(int*int)*Writer.Result>(log, statsInterval, statesInterval) - let resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = ref 0, ref 0, ref 0, ref 0, ref 0 - let rateLimited, timedOut, tooLarge, malformed = ref 0, ref 0, ref 0, ref 0 + let okStreams, resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = HashSet(), ref 0, ref 0, ref 0, ref 0, ref 0 + let badCats, failStreams, rateLimited, timedOut, tooLarge, malformed = CatStats(), HashSet(), ref 0, ref 0, ref 0, ref 0 + let rlStreams, toStreams, tlStreams, mfStreams, oStreams = HashSet(), HashSet(), HashSet(), HashSet(), HashSet() let mutable events, bytes = 0, 0L - let badCats, failStreams, okStreams, toStreams, rlStreams, tlStreams, mfStreams, oStreams = - CatStats(), HashSet(), HashSet(), HashSet(), HashSet(), HashSet(), HashSet(), HashSet() override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Completed {completed:n0}r {mb:n0}MB {streams:n0}s {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", - results, mb bytes, okStreams.Count, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + log.Information("Completed {completed:n0}r {streams:n0}s {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + results, okStreams.Count, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L - if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 then - log.Warning("Exceptions {streams:n0} s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s Too large {tooLarge:n0}e {@tlStreams} Malformed {malformed:n0} {@mfStreams} Other {other:n0} {@oStreams}", - failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count, !tooLarge, tlStreams, !malformed, mfStreams, !resultExnOther, oStreams) - rateLimited := 0; timedOut := 0; tooLarge := 0; malformed := 0; resultExnOther := 0 - if badCats.Any then log.Error("Malformed categories {@badCats}", badCats.StatsDescending); badCats.Clear() - toStreams.Clear(); rlStreams.Clear(); tlStreams.Clear(); mfStreams.Clear(); oStreams.Clear() + if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 || !resultExnOther <> 0 then + log.Warning("Failures {streams:n0}s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s Other {other:n0} {@oStreams}", + failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count, !resultExnOther, oStreams) + rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear(); oStreams.Clear() + if badCats.Any then + log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0} {@tlStreams} Malformed {malformed:n0} {@mfStreams}", + badCats.StatsDescending, !tooLarge, tlStreams, !malformed, mfStreams) + badCats.Clear(); tooLarge := 0; malformed := 0; tlStreams.Clear(); mfStreams.Clear() Equinox.Cosmos.Store.Log.InternalMetrics.dump log override __.Handle message = - base.Handle message let inline adds x (set:HashSet<_>) = set.Add x |> ignore + let inline bads x (set:HashSet<_>) = badCats.Ingest(category x); adds x set + base.Handle message match message with - | Merge _ - | Added _ -> () + | Merge _ | Added _ -> () // Processed by standard logging already; we have nothing to add | Result (stream, Choice1Of2 ((es,bs),r)) -> - okStreams.Add stream |> ignore + adds stream okStreams events <- events + es bytes <- bytes + int64 bs match r with @@ -102,13 +103,13 @@ type Stats(log : ILogger, statsInterval, statesInterval) = | Writer.Result.PartialDuplicate _ -> incr resultPartialDup | Writer.Result.PrefixMissing _ -> incr resultPrefix | Result (stream, Choice2Of2 exn) -> - failStreams.Add stream |> ignore + adds stream failStreams match Writer.classify exn with - | ResultKind.Other -> adds stream oStreams; incr resultExnOther | ResultKind.RateLimited -> adds stream rlStreams; incr rateLimited - | ResultKind.TooLarge -> category stream |> badCats.Ingest; adds stream tlStreams; incr tooLarge - | ResultKind.Malformed -> category stream |> badCats.Ingest; adds stream mfStreams; incr malformed | ResultKind.TimedOut -> adds stream toStreams; incr timedOut + | ResultKind.TooLarge -> bads stream tlStreams; incr tooLarge + | ResultKind.Malformed -> bads stream mfStreams; incr malformed + | ResultKind.Other -> bads stream oStreams; incr resultExnOther let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInterval, statesInterval)) = let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 96fc85c3f..174f0be68 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -382,7 +382,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Ingest"), cosmosContexts, maxWriters, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with @@ -393,7 +393,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo chunk startPos |> int, conns, (max (conns.Length) (spec.streamReaders+1)) | None -> 0, [|conn|], spec.streamReaders+1 - let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","ES"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, TimeSpan.FromMinutes 1.) + let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","EventStore"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, TimeSpan.FromMinutes 1.) let post = function | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId | Res.Batch (seriesId, pos, xs) -> diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 2e8721385..c7928af96 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -188,7 +188,7 @@ module Scheduling = let merges, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) let dumpStats (used,maxDop) pendingCount = - log.Information("Projection Cycles {cycles}/{fullCycles} States {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", + log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 log.Information("Ingestions {batches} {streams:n0}s {events:n0}-{skipped:n0}e Merged {merges} Pending {pending}", From b17d68d3f5b39aa773a6320f21a5c8fd1b975bdf Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 14:41:27 +0100 Subject: [PATCH 282/353] Fix categorization --- equinox-sync/Sync/CosmosIngester.fs | 19 ++++++++++--------- equinox-sync/Sync/Projection2.fs | 3 ++- 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index dccd7371a..bb00d41c4 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -74,22 +74,23 @@ type Stats(log : ILogger, statsInterval, statesInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Completed {completed:n0}r {streams:n0}s {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + log.Information("Completed {completed}r {streams}s {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", results, okStreams.Count, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L - if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 || !resultExnOther <> 0 then - log.Warning("Failures {streams:n0}s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s Other {other:n0} {@oStreams}", - failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count, !resultExnOther, oStreams) - rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear(); oStreams.Clear() + if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 || badCats.Any then + let fails = !rateLimited + !timedOut + !tooLarge + !malformed + !resultExnOther + log.Warning("Failures {fails}r {streams:n0}s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s", + failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) + rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear() if badCats.Any then - log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0} {@tlStreams} Malformed {malformed:n0} {@mfStreams}", - badCats.StatsDescending, !tooLarge, tlStreams, !malformed, mfStreams) - badCats.Clear(); tooLarge := 0; malformed := 0; tlStreams.Clear(); mfStreams.Clear() + log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0}r {@tlStreams} Malformed {malformed:n0}r {@mfStreams} Other {other:n0}r {@oStreams}", + badCats.StatsDescending, !tooLarge, tlStreams, !malformed, mfStreams, !resultExnOther, oStreams) + badCats.Clear(); tooLarge := 0; malformed := 0; resultExnOther := 0; tlStreams.Clear(); mfStreams.Clear(); oStreams.Clear() Equinox.Cosmos.Store.Log.InternalMetrics.dump log override __.Handle message = let inline adds x (set:HashSet<_>) = set.Add x |> ignore - let inline bads x (set:HashSet<_>) = badCats.Ingest(category x); adds x set + let inline bads x (set:HashSet<_>) = badCats.Ingest(Helpers.category x); adds x set base.Handle message match message with | Merge _ | Added _ -> () // Processed by standard logging already; we have nothing to add diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index c7928af96..b83392cbc 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -10,7 +10,8 @@ open System.Diagnostics open System.Threading [] -module private Helpers = +module Helpers = + let category (streamName : string) = streamName.Split([|'-';'_'|],2).[0] let expiredMs ms = let timer = Stopwatch.StartNew() fun () -> From 0154d8a1e859a53742d9441de80ef96c0c311c6c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 15:18:17 +0100 Subject: [PATCH 283/353] Fix --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index b83392cbc..b3163e645 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -192,7 +192,7 @@ module Scheduling = log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 - log.Information("Ingestions {batches} {streams:n0}s {events:n0}-{skipped:n0}e Merged {merges} Pending {pending}", + log.Information("Ingestions {batches}b {streams:n0}s {events:n0}-{skipped:n0}e Merged {merges} Pending {pending}", !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingCount) batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0 abstract member Handle : InternalMessage<'R> -> unit From da2a94b6d4729e0a62e7aabb0b5dff94db967586 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 16:35:58 +0100 Subject: [PATCH 284/353] Inline state --- equinox-projector/Projector/Program.fs | 3 +- equinox-sync/Sync/CosmosIngester.fs | 20 ++-- equinox-sync/Sync/CosmosSource.fs | 19 ++-- equinox-sync/Sync/EventStoreSource.fs | 6 +- equinox-sync/Sync/Program.fs | 14 +-- equinox-sync/Sync/Projection2.fs | 142 +++++++++++++++++++------ 6 files changed, 144 insertions(+), 60 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index caace8a44..3db69e3aa 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -5,7 +5,6 @@ open Confluent.Kafka //#endif open Equinox.Cosmos open Equinox.Cosmos.Projection -open Equinox.Projection.State //#if kafka open Equinox.Projection.Codec open Equinox.Store @@ -229,7 +228,7 @@ let main argv = //let targetParams = args.Target.BuildTargetParams() //let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams //#endif - let project (batch : StreamSpan) = async { + let project (batch : Equinox.Projection.State.StreamSpan) = async { let r = Random() let ms = r.Next(1,batch.span.events.Length * 10) do! Async.Sleep ms diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index bb00d41c4..a02a2854a 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -2,14 +2,18 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Store -open Equinox.Projection.Scheduling open Equinox.Projection2 +open Equinox.Projection2.Buffer open Equinox.Projection2.Scheduling -open Equinox.Projection.State open Serilog open System.Threading open System.Collections.Generic +[] +module private Impl = + let arrayBytes (x:byte[]) = if x = null then 0 else x.Length + let inline mb x = float x / 1024. / 1024. + [] module Writer = type [] ResultKind = TimedOut | RateLimited | TooLarge | Malformed | Other @@ -32,7 +36,7 @@ module Writer = | stream, Choice2Of2 exn -> log.Warning(exn,"Writing {stream} failed, retrying", stream) - let write (log : ILogger) (ctx : CosmosContext) ({ stream = s; span = { index = i; events = e}} as batch) = async { + let write (log : ILogger) (ctx : CosmosContext) ({ stream = s; span = { index = i; events = e}} : StreamSpan as batch) = async { let stream = ctx.CreateStream s log.Debug("Writing {s}@{i}x{n}",s,i,e.Length) let! res = ctx.Sync(stream, { index = i; etag = None }, e) @@ -65,7 +69,7 @@ module Writer = | ResultKind.RateLimited | ResultKind.TimedOut | ResultKind.Other -> false | ResultKind.TooLarge | ResultKind.Malformed -> true -type Stats(log : ILogger, statsInterval, statesInterval) = +type Stats(log : ILogger, categorize, statsInterval, statesInterval) = inherit Stats<(int*int)*Writer.Result>(log, statsInterval, statesInterval) let okStreams, resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = HashSet(), ref 0, ref 0, ref 0, ref 0, ref 0 let badCats, failStreams, rateLimited, timedOut, tooLarge, malformed = CatStats(), HashSet(), ref 0, ref 0, ref 0, ref 0 @@ -90,7 +94,7 @@ type Stats(log : ILogger, statsInterval, statesInterval) = override __.Handle message = let inline adds x (set:HashSet<_>) = set.Add x |> ignore - let inline bads x (set:HashSet<_>) = badCats.Ingest(Helpers.category x); adds x set + let inline bads x (set:HashSet<_>) = badCats.Ingest(categorize x); adds x set base.Handle message match message with | Merge _ | Added _ -> () // Processed by standard logging already; we have nothing to add @@ -112,7 +116,7 @@ type Stats(log : ILogger, statsInterval, statesInterval) = | ResultKind.Malformed -> bads stream mfStreams; incr malformed | ResultKind.Other -> bads stream oStreams; incr resultExnOther -let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInterval, statesInterval)) = +let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, categorize, (statsInterval, statesInterval)) = let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = @@ -146,5 +150,5 @@ let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, (statsInter let _stream, { write = wp } = applyResultToStreamState res Writer.logTo writerResultLog (stream,res) wp - let projectionAndCosmosStats = Stats(log.ForContext(), statsInterval, statesInterval) - Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress) \ No newline at end of file + let projectionAndCosmosStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) + Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 717a2810f..31ca9e889 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -3,7 +3,6 @@ open Equinox.Cosmos.Projection open Equinox.Projection open Equinox.Projection2 -open Equinox.Projection.State open Equinox.Store // AwaitTaskCorrect open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing @@ -11,9 +10,9 @@ open Serilog open System open System.Collections.Generic -let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq) (maxReads, maxSubmissions) cosmosIngester () = +let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq) (maxReads, maxSubmissions) categorize cosmosIngester () = let mutable rangeIngester = Unchecked.defaultof<_> - let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxReads, maxSubmissions, TimeSpan.FromMinutes 1.) } + let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxReads, maxSubmissions, categorize, TimeSpan.FromMinutes 1.) } let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform in rangeIngester.Submit(epoch, checkpoint, events) let dispose () = rangeIngester.Stop () let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok @@ -31,12 +30,14 @@ let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maxDocuments, lagReportFreq : TimeSpan option) - (cosmosContext, maxWriters) createRangeProjector = async { + (cosmosContext, maxWriters) + categorize + createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag - let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) + let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) let! _feedEventHost = ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, @@ -87,15 +88,15 @@ module EventV0Parser = let (StandardCodecEvent e) as x = d.Cast() { stream = x.s; index = x.i; event = e } : Equinox.Projection.StreamItem -let transformV0 catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { +let transformV0 categorize catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { let parsed = EventV0Parser.parse v0SchemaDocument let streamName = (*if parsed.Stream.Contains '-' then parsed.Stream else "Prefixed-"+*)parsed.stream - if catFilter (category streamName) then yield parsed } + if catFilter (categorize streamName) then yield parsed } //#else -let transformOrFilter catFilter (changeFeedDocument: Document) : StreamItem seq = seq { +let transformOrFilter categorize catFilter (changeFeedDocument: Document) : StreamItem seq = seq { for e in DocumentParser.enumEvents changeFeedDocument do // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level - if catFilter (category e.stream) then + if catFilter (categorize e.stream) then let e2 = { new Equinox.Codec.IEvent<_> with member __.Data = null diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 174f0be68..9d55a9d3f 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -351,7 +351,7 @@ type ReaderSpec = type StartMode = Starting | Resuming | Overridding -let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmosContexts, maxWriters) resolveCheckpointStream = async { +let run (log : Serilog.ILogger) (connect, spec, categorize, tryMapEvent) maxReadAhead (cosmosContexts, maxWriters) resolveCheckpointStream = async { let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let conn = connect () let! maxInParallel = Async.StartChild <| establishMax conn @@ -382,7 +382,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) + let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) let initialSeriesId, conns, dop = log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with @@ -393,7 +393,7 @@ let run (log : Serilog.ILogger) (connect, spec, tryMapEvent) maxReadAhead (cosmo chunk startPos |> int, conns, (max (conns.Length) (spec.streamReaders+1)) | None -> 0, [|conn|], spec.streamReaders+1 - let trancheEngine = Ingestion.Engine.Start (log.ForContext("Tranche","EventStore"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, TimeSpan.FromMinutes 1.) + let trancheEngine = Ingestion.Engine.Start(log.ForContext("Tranche","EventStore"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, categorize, TimeSpan.FromMinutes 1.) let post = function | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId | Res.Batch (seriesId, pos, xs) -> diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2a1b5f532..2f602e69f 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -286,7 +286,7 @@ module Logging = let cfpLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpLevel) |> fun c -> let ingesterLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information - c.MinimumLevel.Override(typeof.FullName, ingesterLevel) + c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning c.MinimumLevel.Override(typeof.FullName, generalLevel) @@ -308,7 +308,7 @@ module Logging = c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() - Log.ForContext(), Log.ForContext() + Log.ForContext(), Log.ForContext() [] let main argv = @@ -332,20 +332,22 @@ let main argv = let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve let targets = destinations |> Array.mapi (fun i x -> Equinox.Cosmos.Core.CosmosContext(x, colls, storeLog.ForContext("PoolId", i))) + let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] #if cosmos let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() let auxDiscovery, aux, leaseId, startFromHere, maxDocuments, lagFrequency = args.BuildChangeFeedParams() #if marveleqx let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformV0 catFilter) #else - let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformOrFilter catFilter) + let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformOrFilter categorize catFilter) // Uncomment to test marveleqx mode - // let createSyncHandler () = CosmosSource.createRangeSyncHandler log target (CosmosSource.transformV0 catFilter) + // let createSyncHandler () = CosmosSource.createRangeSyncHandler log categorize target (CosmosSource.transformV0 categorize catFilter) #endif CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromHere, maxDocuments, lagFrequency) (targets, args.MaxWriters) - (createSyncHandler (args.MaxPendingBatches,args.MaxProcessing)) + categorize + (createSyncHandler (args.MaxPendingBatches,args.MaxProcessing) categorize) #else let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let catFilter = args.Source.CategoryFilterFunction @@ -366,7 +368,7 @@ let main argv = || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None | e -> e |> EventStoreSource.toIngestionItem |> Some - EventStoreSource.run log (connect, spec, tryMapEvent catFilter) args.MaxPendingBatches (targets, args.MaxWriters) resolveCheckpointStream + EventStoreSource.run log (connect, spec, categorize, tryMapEvent catFilter) args.MaxPendingBatches (targets, args.MaxWriters) resolveCheckpointStream #endif |> Async.RunSynchronously 0 diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index b3163e645..cc388af8f 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -1,7 +1,6 @@ namespace Equinox.Projection2 open Equinox.Projection -open Equinox.Projection.State open Serilog open System open System.Collections.Concurrent @@ -9,9 +8,28 @@ open System.Collections.Generic open System.Diagnostics open System.Threading +/// Gathers stats relating to how many items of a given category have been observed +type CatStats() = + let cats = Dictionary() + member __.Ingest(cat,?weight) = + let weight = defaultArg weight 1L + match cats.TryGetValue cat with + | true, catCount -> cats.[cat] <- catCount + weight + | false, _ -> cats.[cat] <- weight + member __.Any = cats.Count <> 0 + member __.Clear() = cats.Clear() +#if NET461 + member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortBy (fun (_,s) -> -s) +#else + member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd +#endif + [] -module Helpers = - let category (streamName : string) = streamName.Split([|'-';'_'|],2).[0] +module private Impl = + let (|NNA|) xs = if xs = null then Array.empty else xs + let arrayBytes (x:byte[]) = if x = null then 0 else x.Length + let inline eventSize (x : Equinox.Codec.IEvent<_>) = arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16 + let inline mb x = float x / 1024. / 1024. let expiredMs ms = let timer = Stopwatch.StartNew() fun () -> @@ -82,7 +100,68 @@ module Progress = with e -> result.Trigger (Choice2Of2 e) | _ -> do! Async.Sleep pumpSleepMs } +module Buffer = + + type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } + module Span = + let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength + let trim min : Span -> Span = function + | x when x.index >= min -> x // don't adjust if min not within + | End n when n < min -> { index = min; events = [||] } // throw away if before min +#if NET461 + | x -> { index = min; events = x.events |> Seq.skip (min - x.index |> int) |> Seq.toArray } +#else + | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice +#endif + let merge min (xs : Span seq) = + let xs = + seq { for x in xs -> { x with events = (|NNA|) x.events } } + |> Seq.map (trim min) + |> Seq.filter (fun x -> x.events.Length <> 0) + |> Seq.sortBy (fun x -> x.index) + let buffer = ResizeArray() + let mutable curr = None + for x in xs do + match curr, x with + // Not overlapping, no data buffered -> buffer + | None, _ -> + curr <- Some x + // Gap + | Some (End nextIndex as c), x when x.index > nextIndex -> + buffer.Add c + curr <- Some x + // Overlapping, join + | Some (End nextIndex as c), x -> + curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } + curr |> Option.iter buffer.Add + if buffer.Count = 0 then null else buffer.ToArray() + type [] StreamSpan = { stream: string; span: Span } + type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with + member __.Size = + if __.queue = null then 0 + else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy eventSize + member __.IsReady = + if __.queue = null || __.isMalformed then false + else + match __.write, Array.tryHead __.queue with + | Some w, Some { index = i } -> i = w + | None, _ -> true + | _ -> false + module StreamState = + let inline optionCombine f (r1: int64 option) (r2: int64 option) = + match r1, r2 with + | Some x, Some y -> f x y |> Some + | None, None -> None + | None, x | x, None -> x + let combine (s1: StreamState) (s2: StreamState) : StreamState = + let writePos = optionCombine max s1.write s2.write + let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 + { write = writePos; queue = Span.merge (defaultArg writePos 0L) items; isMalformed = s1.isMalformed || s2.isMalformed } + module Scheduling = + + open Buffer + type StreamStates() = let mutable streams = Set.empty let states = Dictionary() @@ -137,9 +216,9 @@ module Scheduling = markCompleted stream index member __.MarkFailed stream = markNotBusy stream - member __.Pending(trySlipsteamed, byQueuedPriority : string seq) : (int64 option * StreamSpan) seq = - pending trySlipsteamed byQueuedPriority - member __.Dump(log : ILogger) = + member __.Pending(trySlipstreamed, byQueuedPriority : string seq) : (int64 option * StreamSpan) seq = + pending trySlipstreamed byQueuedPriority + member __.Dump(log : ILogger, categorize) = let mutable busyCount, busyB, ready, readyB, unprefixed, unprefixedB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0 let busyCats, readyCats, readyStreams, unprefixedStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats(), CatStats() let kb sz = (sz + 512L) / 1024L @@ -148,7 +227,7 @@ module Scheduling = | 0L -> synced <- synced + 1 | sz when busy.Contains stream -> - busyCats.Ingest(category stream) + busyCats.Ingest(categorize stream) busyCount <- busyCount + 1 busyB <- busyB + sz | sz when state.isMalformed -> @@ -160,7 +239,7 @@ module Scheduling = unprefixed <- unprefixed + 1 unprefixedB <- unprefixedB + sz | sz -> - readyCats.Ingest(category stream) + readyCats.Ingest(categorize stream) readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) ready <- ready + 1 readyB <- readyB + sz @@ -213,19 +292,16 @@ module Scheduling = if statsDue () then dumpStats (used,max) pendingCount __.DumpExtraStats() - member __.TryDumpState(state, streams : StreamStates) = + member __.TryDumpState(state,dump) = incr fullCycles states.Ingest(string state) let due = stateDue () if due then - __.DumpExtraState() - streams.Dump log + dump log due /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) abstract DumpExtraStats : unit -> unit default __.DumpExtraStats () = () - abstract DumpExtraState : unit -> unit - default __.DumpExtraState () = () /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint type Dispatcher<'R>(maxDop) = @@ -254,7 +330,7 @@ module Scheduling = /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) /// c) submits work to the supplied Dispatcher (which it triggers pumping of) /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress) = + type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() let work = ConcurrentStack>() // dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better @@ -333,13 +409,13 @@ module Scheduling = // This loop can take a long time; attempt logging of stats per iteration stats.DumpStats(dispatcher.State,pending.Count) // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently - if not (stats.TryDumpState(dispatcherState,streams)) && not idle then + if not (stats.TryDumpState(dispatcherState,dumpStreams streams)) && not idle then // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) do! Async.Sleep sleepIntervalMs } - static member Start<'R>(stats, projectorDop, project, interpretProgress) = + static member Start<'R>(stats, projectorDop, project, interpretProgress, dumpStreams) = let dispatcher = Dispatcher(projectorDop) - let instance = new Engine<'R>(dispatcher, project, interpretProgress) + let instance = new Engine<'R>(dispatcher, project, interpretProgress, dumpStreams) Async.Start <| instance.Pump(stats) instance @@ -359,7 +435,7 @@ module Scheduling = type Projector = - static member Start(log, projectorDop, project : StreamSpan -> Async, ?statsInterval, ?statesInterval) = + static member Start(log, projectorDop, project : Buffer.StreamSpan -> Async, categorize, ?statsInterval, ?statesInterval) = let project (_maybeWritePos, batch) = async { try let! count = project batch return Choice1Of2 (batch.span.index + int64 count) @@ -368,7 +444,9 @@ type Projector = | Choice1Of2 index -> Some index | Choice2Of2 _ -> None let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.), defaultArg statesInterval (TimeSpan.FromMinutes 5.)) - Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress) + //let category (streamName : string) = streamName.Split([|'-';'_'|],2).[0] + let dumpStreams (streams: Scheduling.StreamStates) log = streams.Dump(log, categorize) + Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress, dumpStreams) module Ingestion = @@ -379,13 +457,13 @@ module Ingestion = | EndOfSeries of seriesIndex: int type private Streams() = - let states = Dictionary() - let merge stream (state : StreamState) = + let states = Dictionary() + let merge stream (state : Buffer.StreamState) = match states.TryGetValue stream with | false, _ -> states.Add(stream, state) | true, current -> - let updated = StreamState.combine current state + let updated = Buffer.StreamState.combine current state states.[stream] <- updated member __.Merge(items : StreamItem seq) = @@ -397,12 +475,12 @@ module Ingestion = for x in forward do states.Remove x.Key |> ignore forward - member __.Dump(log : ILogger) = + member __.Dump(categorize, log : ILogger) = let mutable waiting, waitingB = 0, 0L let waitingCats, waitingStreams = CatStats(), CatStats() for KeyValue (stream,state) in states do let sz = int64 state.Size - waitingCats.Ingest(category stream) + waitingCats.Ingest(categorize stream) waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) waiting <- waiting + 1 waitingB <- waitingB + sz @@ -410,7 +488,7 @@ module Ingestion = if waitingCats.Any then log.Information("Waiting Categories, events {@readyCats}", Seq.truncate 5 waitingCats.StatsDescending) if waitingCats.Any then log.Information("Waiting Streams, KB {@readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) - type private Stats(log : ILogger, maxPendingBatches, statsInterval : TimeSpan) = + type private Stats(log : ILogger, maxPendingBatches, categorize, statsInterval : TimeSpan) = let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None let progCommitFails, progCommits = ref 0, ref 0 let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 @@ -458,7 +536,7 @@ module Ingestion = incr cycles if statsDue () then dumpStats (available,maxDop) (readingAhead,ready) - streams.Dump log + streams.Dump(categorize,log) and [] private InternalMessage = | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq @@ -477,14 +555,14 @@ module Ingestion = | false, _ -> None /// Holds batches away from Core processing to limit in-flight processing - type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxRead, maxSubmissions, initialSeriesIndex, statsInterval : TimeSpan, ?pumpDelayMs) = + type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxRead, maxSubmissions, initialSeriesIndex, categorize, statsInterval : TimeSpan, ?pumpDelayMs) = let cts = new CancellationTokenSource() let pumpDelayMs = defaultArg pumpDelayMs 5 let work = ConcurrentQueue() // Queue as need ordering semantically let readMax = new Sem(maxRead) let submissionsMax = new Sem(maxSubmissions) let streams = Streams() - let stats = Stats(log, maxRead, statsInterval) + let stats = Stats(log, maxRead, categorize, statsInterval) let pending = Queue<_>() let readingAhead, ready = Dictionary>(), Dictionary>() let progressWriter = Progress.Writer<_>() @@ -566,8 +644,8 @@ module Ingestion = with e -> log.Error(e,"Buffer thread exception") } /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes - static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval) = - let instance = new Engine<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, statsInterval = statsInterval) + static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval) = + let instance = new Engine<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval = statsInterval) Async.Start <| instance.Pump() instance @@ -589,9 +667,9 @@ module Ingestion = type Ingester = /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing - static member Start<'R>(log, scheduler, maxRead, maxSubmissions, ?statsInterval) = + static member Start<'R>(log, scheduler, maxRead, maxSubmissions, categorize, ?statsInterval) = let singleSeriesIndex = 0 - let instance = Ingestion.Engine<'R>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + let instance = Ingestion.Engine<'R>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) { new IIngester with member __.Submit(epoch, markCompleted, items) : Async = instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) From 660c7965fe7a0430147104f288898e7d58013d62 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 16:40:52 +0100 Subject: [PATCH 285/353] Logging fix --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index a02a2854a..abea0c4e7 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -84,7 +84,7 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 || badCats.Any then let fails = !rateLimited + !timedOut + !tooLarge + !malformed + !resultExnOther log.Warning("Failures {fails}r {streams:n0}s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s", - failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) + fails, failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear() if badCats.Any then log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0}r {@tlStreams} Malformed {malformed:n0}r {@mfStreams} Other {other:n0}r {@oStreams}", From 2720355d962f81174cfdcb19c1f5c35d9cee0608 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 17:03:44 +0100 Subject: [PATCH 286/353] Fix Batch stats --- equinox-sync/Sync/Projection2.fs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index cc388af8f..3fc4a977d 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -265,18 +265,19 @@ module Scheduling = /// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan, stateInterval : TimeSpan) = let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 - let merges, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0 + let merges, mergedStreams, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) let dumpStats (used,maxDop) pendingCount = log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 - log.Information("Ingestions {batches}b {streams:n0}s {events:n0}-{skipped:n0}e Merged {merges} Pending {pending}", - !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingCount) - batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0 + log.Information("Batches Pending {pending) Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}b {mergedStreams}s", + pendingCount, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, !mergedStreams) + batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function - | Merge _ -> + | Merge items -> + mergedStreams := !mergedStreams + items.Length incr merges | Added (streams, skipped, events) -> incr batchesPended From 525fe945e56326dda3a7085a879ffef1fa5f84a6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 17:19:13 +0100 Subject: [PATCH 287/353] Fix log typo --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 3fc4a977d..f6439b09b 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -271,7 +271,7 @@ module Scheduling = log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 - log.Information("Batches Pending {pending) Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}b {mergedStreams}s", + log.Information("Batches Pending {pending} Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}b {mergedStreams}s", pendingCount, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, !mergedStreams) batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 abstract member Handle : InternalMessage<'R> -> unit From 46db864cfb2e5f8f4198e70e8109cbd1eac3b34f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 17:47:03 +0100 Subject: [PATCH 288/353] reinstate bodies --- equinox-sync/Sync/CosmosSource.fs | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 31ca9e889..5a4c7b873 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -97,12 +97,14 @@ let transformOrFilter categorize catFilter (changeFeedDocument: Document) : Stre for e in DocumentParser.enumEvents changeFeedDocument do // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level if catFilter (categorize e.stream) then - let e2 = - { new Equinox.Codec.IEvent<_> with - member __.Data = null - member __.Meta = null - member __.EventType = e.event.EventType - member __.Timestamp = e.event.Timestamp } - yield { e with event = e2 } + //let removeBody e = + //let e2 = + // { new Equinox.Codec.IEvent<_> with + // member __.Data = null + // member __.Meta = null + // member __.EventType = e.event.EventType + // member __.Timestamp = e.event.Timestamp } + //yield { e with event = e2 } + yield e } //#endif \ No newline at end of file From 9a52563aa878546f057d1a5a85b970fa9bdfbbc9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 10 May 2019 21:39:49 +0100 Subject: [PATCH 289/353] Stack Pop fix --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index f6439b09b..7534dca14 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -344,7 +344,7 @@ module Scheduling = let mutable worked, more = false, true while more do let c = work.TryPopRange(workLocalBuffer) - if c = 0 then more <- false else worked <- true + if c = 0 && work.IsEmpty then more <- false else worked <- true for i in 0..c-1 do let x = workLocalBuffer.[i] match x with From a651b30dd1f0cf3e22f606ffb8b502de87f9700d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 11 May 2019 07:09:58 +0100 Subject: [PATCH 290/353] Lengthen timeouts --- equinox-sync/Sync/CosmosSource.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 5a4c7b873..1b1f95f9d 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -41,7 +41,8 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection let! _feedEventHost = ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, - createObserver = createRangeProjector cosmosIngester, ?reportLagAndAwaitNextEstimation = maybeLogLag, cfBatchSize = defaultArg maxDocuments 999999) + createObserver = createRangeProjector cosmosIngester, ?reportLagAndAwaitNextEstimation = maybeLogLag, cfBatchSize = defaultArg maxDocuments 999999, + leaseAcquireInterval=TimeSpan.FromSeconds 5., leaseRenewInterval=TimeSpan.FromSeconds 5., leaseTtl=TimeSpan.FromMinutes 1.) do! Async.AwaitKeyboardInterrupt() cosmosIngester.Stop() } From d492d34b79ba1d70217f5c47c1e17c1dec3d85d8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sun, 12 May 2019 12:24:32 +0100 Subject: [PATCH 291/353] Correct categorization --- equinox-sync/Sync/EventStoreSource.fs | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 9d55a9d3f..cffa467d0 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -25,8 +25,6 @@ let toIngestionItem (e : RecordedEvent) : StreamItem = let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ { stream = e.EventStreamId; index = e.EventNumber; event = event} -let category (streamName : string) = streamName.Split([|'-'|],2).[0] - /// Maintains ingestion stats (thread safe via lock free data structures so it can be used across multiple overlapping readers) type OverallStats(?statsInterval) = let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 @@ -46,14 +44,14 @@ type OverallStats(?statsInterval) = progressStart.Restart() /// Maintains stats for traversals of $all; Threadsafe [via naive locks] so can be used by multiple stripes reading concurrently -type SliceStatsBuffer(?interval) = +type SliceStatsBuffer(categorize, ?interval) = let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 let recentCats, accStart = Dictionary(), Stopwatch.StartNew() member __.Ingest(slice: AllEventsSlice) = lock recentCats <| fun () -> let mutable batchBytes = 0 for x in slice.Events do - let cat = category x.OriginalStreamId + let cat = categorize x.OriginalStreamId let eventBytes = payloadBytes x match recentCats.TryGetValue cat with | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) @@ -197,11 +195,11 @@ type Res = /// Holds work queue, together with stats relating to the amount and/or categories of data being traversed /// Processing is driven by external callers running multiple concurrent invocations of `Process` -type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Res -> Async, tailInterval, dop, ?statsInterval) = +type Reader(conns : _ [], defaultBatchSize, minBatchSize, categorize, tryMapEvent, post : Res -> Async, tailInterval, dop, ?statsInterval) = let work = System.Collections.Concurrent.ConcurrentQueue() let sleepIntervalMs = 100 let overallStats = OverallStats(?statsInterval=statsInterval) - let slicesStats = SliceStatsBuffer() + let slicesStats = SliceStatsBuffer(categorize) let mutable eofSpottedInChunk = 0 /// Invoked by pump to process a tranche of work; can have parallel invocations @@ -259,7 +257,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, tryMapEvent, post : Re | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) | _ -> () tailSw.Restart() } - let slicesStats, stats = SliceStatsBuffer(), OverallStats() + let slicesStats, stats = SliceStatsBuffer(categorize), OverallStats() let progressSw = Stopwatch.StartNew() while true do let currentPos = range.Current @@ -399,6 +397,6 @@ let run (log : Serilog.ILogger) (connect, spec, categorize, tryMapEvent) maxRead | Res.Batch (seriesId, pos, xs) -> let cp = pos.CommitPosition trancheEngine.Submit <| Ingestion.Message.Batch(seriesId, cp, checkpoints.Commit cp, xs) - let reader = Reader(conns, spec.batchSize, spec.minBatchSize, tryMapEvent, post, spec.tailInterval, dop) + let reader = Reader(conns, spec.batchSize, spec.minBatchSize, categorize, tryMapEvent, post, spec.tailInterval, dop) do! reader.Start (initialSeriesId,startPos) maxPos do! Async.AwaitKeyboardInterrupt() } \ No newline at end of file From cc8ea9625c9538c107ea22e6a7e0420b08821ac1 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 00:38:36 +0100 Subject: [PATCH 292/353] Clean slipstreaming --- equinox-sync/Sync/Projection2.fs | 120 +++++++++++++++++-------------- 1 file changed, 68 insertions(+), 52 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 7534dca14..e7746aa73 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -158,18 +158,49 @@ module Buffer = let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 { write = writePos; queue = Span.merge (defaultArg writePos 0L) items; isMalformed = s1.isMalformed || s2.isMalformed } + type Streams() = + let states = Dictionary() + let merge stream (state : StreamState) = + match states.TryGetValue stream with + | false, _ -> + states.Add(stream, state) + | true, current -> + let updated = StreamState.combine current state + states.[stream] <- updated + + member __.StreamCount = states.Count + member __.Items : seq>= states :> _ + + member __.Merge(items : StreamItem seq) = + for item in items do + merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = [| item.event |] } |] } + member __.Merge(other: Streams) = + for x in other.Items do + merge x.Key x.Value + + member __.Dump(categorize, log : ILogger) = + let mutable waiting, waitingB = 0, 0L + let waitingCats, waitingStreams = CatStats(), CatStats() + for KeyValue (stream,state) in states do + let sz = int64 state.Size + waitingCats.Ingest(categorize stream) + waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) + waiting <- waiting + 1 + waitingB <- waitingB + sz + if waiting <> 0 then log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) + if waitingCats.Any then log.Information("Waiting Categories, events {@readyCats}", Seq.truncate 5 waitingCats.StatsDescending) + if waitingCats.Any then log.Information("Waiting Streams, KB {@readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) + module Scheduling = open Buffer type StreamStates() = - let mutable streams = Set.empty let states = Dictionary() let update stream (state : StreamState) = match states.TryGetValue stream with | false, _ -> states.Add(stream, state) - streams <- streams.Add stream stream, state | true, current -> let updated = StreamState.combine current state @@ -177,6 +208,9 @@ module Scheduling = stream, updated let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore + let mergeBuffered (buffer : Buffer.Streams) = + for x in buffer.Items do + update x.Key x.Value |> ignore let busy = HashSet() let pending trySlipstreamed (requestedOrder : string seq) = seq { @@ -198,8 +232,7 @@ module Scheduling = // This enables the (potentially multiple) Ingesters to determine streams (for which they potentially have successor events) that are in play // Ingesters then supply these 'preview events' in advance of the processing being scheduled // This enables the projection logic to roll future work into the current work in the interests of medium term throughput - member __.All = streams - member __.InternalMerge(stream, state) = update stream state |> ignore + member __.InternalMerge buffer = mergeBuffered buffer member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } member __.Add(stream, index, event, ?isMalformed) = updateWritePos stream (defaultArg isMalformed false) None [| { index = index; events = [| event |] } |] @@ -254,8 +287,8 @@ module Scheduling = /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners [] type InternalMessage<'R> = - /// Submit new data pertaining to a stream that has commenced processing - | Merge of KeyValuePair[] + /// Periodic submission of events as they are read, grouped by stream for efficient merging into the StreamState + | Merge of Streams /// Stats per submitted batch for stats listeners to aggregate | Added of streams: int * skip: int * events: int /// Result of processing on stream - result (with basic stats) or the `exn` encountered @@ -276,8 +309,8 @@ module Scheduling = batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function - | Merge items -> - mergedStreams := !mergedStreams + items.Length + | Merge buffer -> + mergedStreams := !mergedStreams + buffer.StreamCount incr merges | Added (streams, skipped, events) -> incr batchesPended @@ -334,22 +367,24 @@ module Scheduling = type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams) = let sleepIntervalMs = 1 let cts = new CancellationTokenSource() - let work = ConcurrentStack>() // dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better + let work = ConcurrentStack>()// dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better + let slipstreamed = ResizeArray() // pulled from `work` and kept aside for processing at the right time as they are encountered let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering let streams = StreamStates() let progressState = Progress.State() + // ingest information to be gleaned from processing the results into `streams` static let workLocalBuffer = Array.zeroCreate 1024 let tryDrainResults feedStats = let mutable worked, more = false, true while more do let c = work.TryPopRange(workLocalBuffer) - if c = 0 && work.IsEmpty then more <- false else worked <- true + if c = 0 then more <- false else worked <- true for i in 0..c-1 do let x = workLocalBuffer.[i] match x with - | Added _ -> () // Only processed in Stats (and actually never enters this queue) - | Merge events -> for e in events do streams.InternalMerge(e.Key,e.Value) + | Added _ -> () // Only processed in Stats (and actually never enters this queue) + | Merge buffer -> slipstreamed.Add buffer // put aside as a) they can be done more efficiently in bulk b) we only want to pay the tax at the right time | Result (stream,res) -> match interpretProgress streams stream res with | None -> streams.MarkFailed stream @@ -358,6 +393,18 @@ module Scheduling = streams.MarkCompleted(stream,index) feedStats x worked + // We periodically process streamwise submissions of events from Ingesters in advance of them entering the processing queue as pending batches + // This allows events that are not yet a requirement for a given batch to complete to be included in work before it becomes due, smoothing throughput + let ingestSlipstreamed () = + match slipstreamed.ToArray() with + | [||] -> () + | [| one |] -> streams.InternalMerge one + | many -> + let combined = many.[0] + for x in Array.skip 1 many do combined.Merge x + streams.InternalMerge combined + slipstreamed.Clear() + // On ech iteration, we try to fill the in-flight queue, taking the oldest and/or heaviest streams first let tryFillDispatcher includeSlipstreamed = async { let mutable hasCapacity, dispatched = dispatcher.HasCapacity, false if hasCapacity then @@ -370,6 +417,7 @@ module Scheduling = dispatched <- dispatched || succeeded // if we added any request, we also don't sleep hasCapacity <- succeeded return hasCapacity, dispatched } + // Take an incoming batch of events, correlating it against our known stream state to yield a set of remaining work let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = let inline validVsSkip (streamState : StreamState) (item : StreamItem) = match streamState.write, item.index + 1L with @@ -403,12 +451,13 @@ module Scheduling = | Idle when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue match pending.TryDequeue() with | true, batch -> ingestPendingBatch stats.Handle batch - | false,_ -> dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters + | false,_ -> ingestSlipstreamed (); dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters | Idle -> dispatcherState <- Full; finished <- true | Slipstreaming -> finished <- true | _ -> () // This loop can take a long time; attempt logging of stats per iteration stats.DumpStats(dispatcher.State,pending.Count) + ingestSlipstreamed () // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently if not (stats.TryDumpState(dispatcherState,dumpStreams streams)) && not idle then // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) @@ -426,11 +475,9 @@ module Scheduling = member __.Submit(markCompleted: (unit -> unit), items: StreamItem[]) = pending.Enqueue (markCompleted, items) - member __.AddOpenStreamData(events) = + member __.SubmitStreamBuffers(events) = work.Push <| Merge events - member __.AllStreams = streams.All - member __.Stop() = cts.Cancel() @@ -457,38 +504,6 @@ module Ingestion = //| StreamSegment of span: StreamSpan | EndOfSeries of seriesIndex: int - type private Streams() = - let states = Dictionary() - let merge stream (state : Buffer.StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - | true, current -> - let updated = Buffer.StreamState.combine current state - states.[stream] <- updated - - member __.Merge(items : StreamItem seq) = - for item in items do - merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = [| item.event |] } |] } - - member __.Take(processingContains) = - let forward = [| for x in states do if processingContains x.Key then yield x |] - for x in forward do states.Remove x.Key |> ignore - forward - - member __.Dump(categorize, log : ILogger) = - let mutable waiting, waitingB = 0, 0L - let waitingCats, waitingStreams = CatStats(), CatStats() - for KeyValue (stream,state) in states do - let sz = int64 state.Size - waitingCats.Ingest(categorize stream) - waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) - waiting <- waiting + 1 - waitingB <- waitingB + sz - if waiting <> 0 then log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) - if waitingCats.Any then log.Information("Waiting Categories, events {@readyCats}", Seq.truncate 5 waitingCats.StatsDescending) - if waitingCats.Any then log.Information("Waiting Streams, KB {@readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) - type private Stats(log : ILogger, maxPendingBatches, categorize, statsInterval : TimeSpan) = let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None let progCommitFails, progCommits = ref 0, ref 0 @@ -533,7 +548,7 @@ module Ingestion = pendingBatchCount <- pendingBatches member __.HandleCommitted epoch = comittedEpoch <- epoch - member __.TryDump((available,maxDop),streams : Streams,readingAhead,ready) = + member __.TryDump((available,maxDop),streams : Buffer.Streams,readingAhead,ready) = incr cycles if statsDue () then dumpStats (available,maxDop) (readingAhead,ready) @@ -562,7 +577,8 @@ module Ingestion = let work = ConcurrentQueue() // Queue as need ordering semantically let readMax = new Sem(maxRead) let submissionsMax = new Sem(maxSubmissions) - let streams = Streams() + let mutable streams = Buffer.Streams() + let grabAccumulatedStreams () = let t = streams in streams <- Buffer.Streams(); t let stats = Stats(log, maxRead, categorize, statsInterval) let pending = Queue<_>() let readingAhead, ready = Dictionary>(), Dictionary>() @@ -632,8 +648,8 @@ module Ingestion = stats.HandleCommitted progressWriter.CommittedEpoch // 2. Forward content for any active streams into processor immediately if presubmitInterval () then - let relevantBufferedStreams = streams.Take(fun x -> true (*scheduler.AllStreams.Contains*)) - scheduler.AddOpenStreamData(relevantBufferedStreams) + grabAccumulatedStreams () |> scheduler.SubmitStreamBuffers + // 3. Submit to ingester until read queue, tranche limit or ingester limit exhausted while pending.Count <> 0 && submissionsMax.HasCapacity do // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) From e786da25535ac31a979b728563937cdf69445acf Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 01:04:31 +0100 Subject: [PATCH 293/353] Tidy completed log --- equinox-sync/Sync/CosmosIngester.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index abea0c4e7..3f0cd3bd4 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -78,8 +78,8 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix - log.Information("Completed {completed}r {streams}s {events:n0}e {mb:n0}MB ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", - results, okStreams.Count, events, mb bytes, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + log.Information("Completed {mb:n0}MB {completed:n0}r {streams:n0}s {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", + mb bytes, results, okStreams.Count, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 || badCats.Any then let fails = !rateLimited + !timedOut + !tooLarge + !malformed + !resultExnOther From b8a8259466b4f156f2245cf596841cd41bd846a0 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 11:47:47 +0100 Subject: [PATCH 294/353] Improve full buffers handling --- equinox-sync/Sync/CosmosSource.fs | 2 +- equinox-sync/Sync/Projection2.fs | 41 ++++++++++++++++++++----------- 2 files changed, 28 insertions(+), 15 deletions(-) diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 1b1f95f9d..517ff4704 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -34,7 +34,7 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection categorize createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) + log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortBy fst) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index e7746aa73..d164de44b 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -300,12 +300,12 @@ module Scheduling = let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 let merges, mergedStreams, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) - let dumpStats (used,maxDop) pendingCount = + let dumpStats (used,maxDop) (waitingBatches,pendingMerges) = log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 - log.Information("Batches Pending {pending} Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}b {mergedStreams}s", - pendingCount, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, !mergedStreams) + log.Information("Batches Waiting {batchesWaiting} Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}/{pendingMerges} {mergedStreams}s", + waitingBatches, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingMerges, !mergedStreams) batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 abstract member Handle : InternalMessage<'R> -> unit default __.Handle msg = msg |> function @@ -395,7 +395,7 @@ module Scheduling = worked // We periodically process streamwise submissions of events from Ingesters in advance of them entering the processing queue as pending batches // This allows events that are not yet a requirement for a given batch to complete to be included in work before it becomes due, smoothing throughput - let ingestSlipstreamed () = + let ingestStreamMerges () = match slipstreamed.ToArray() with | [||] -> () | [| one |] -> streams.InternalMerge one @@ -440,24 +440,37 @@ module Scheduling = use _ = dispatcher.Result.Subscribe(Result >> work.Push) Async.Start(dispatcher.Pump(), cts.Token) while not cts.IsCancellationRequested do - let mutable idle, dispatcherState, finished = true, Idle, false - while not finished do + let mutable idle, dispatcherState, remaining = true, Idle, 100 + ingestStreamMerges () + while remaining <> 0 do + remaining <- remaining - 1 // 1. propagate write write outcomes to buffer (can mark batches completed etc) let processedResults = tryDrainResults stats.Handle // 2. top up provisioning of writers queue let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched match dispatcherState with - | Idle when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue + | Idle when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue from what we have match pending.TryDequeue() with - | true, batch -> ingestPendingBatch stats.Handle batch - | false,_ -> ingestSlipstreamed (); dispatcherState <- Slipstreaming // TODO preload extra spans from active submitters - | Idle -> dispatcherState <- Full; finished <- true - | Slipstreaming -> finished <- true - | _ -> () + | true, batch -> + // Periodically merge events in stream-wise to maximize stream write size especially when we are behind + if remaining % 10 = 0 then ingestStreamMerges () + ingestPendingBatch stats.Handle batch + | false,_ -> + // If we're going to fill the write queue with random work, we should bring all read events into the state first + ingestStreamMerges () + dispatcherState <- Slipstreaming + | Idle -> + // If we've achieved full state, spin around the loop to dump stats and ingest reader data + dispatcherState <- Full + remaining <- 0 + | Slipstreaming -> // only do one round of slipstreaming + remaining <- 0 + | Full -> failwith "Not handled here" // This loop can take a long time; attempt logging of stats per iteration - stats.DumpStats(dispatcher.State,pending.Count) - ingestSlipstreamed () + stats.DumpStats(dispatcher.State,(pending.Count,slipstreamed.Count)) + // Do another ingest before a) reporting state to give best picture b) going to sleep in order to get work out of the way + ingestStreamMerges () // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently if not (stats.TryDumpState(dispatcherState,dumpStreams streams)) && not idle then // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) From dc2db73727bfc0174b77e9f03242a724d985ca44 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 12:04:36 +0100 Subject: [PATCH 295/353] Tidy, reinstate IsEmpty check --- equinox-sync/Sync/Projection2.fs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index d164de44b..966224c91 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -294,7 +294,7 @@ module Scheduling = /// Result of processing on stream - result (with basic stats) or the `exn` encountered | Result of stream: string * outcome: Choice<'R,exn> - type BufferState = Idle | Full | Slipstreaming + type BufferState = Idle | Busy | Full | Slipstreaming /// Gathers stats pertaining to the core projection/ingestion activity type Stats<'R>(log : ILogger, statsInterval : TimeSpan, stateInterval : TimeSpan) = let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 @@ -379,7 +379,7 @@ module Scheduling = let mutable worked, more = false, true while more do let c = work.TryPopRange(workLocalBuffer) - if c = 0 then more <- false else worked <- true + if c = 0 && work.IsEmpty then more <- false else worked <- true for i in 0..c-1 do let x = workLocalBuffer.[i] match x with @@ -450,6 +450,8 @@ module Scheduling = let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched match dispatcherState with + | Idle when remaining = 0 -> + dispatcherState <- Busy | Idle when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue from what we have match pending.TryDequeue() with | true, batch -> @@ -466,7 +468,7 @@ module Scheduling = remaining <- 0 | Slipstreaming -> // only do one round of slipstreaming remaining <- 0 - | Full -> failwith "Not handled here" + | Busy | Full -> failwith "Not handled here" // This loop can take a long time; attempt logging of stats per iteration stats.DumpStats(dispatcher.State,(pending.Count,slipstreamed.Count)) // Do another ingest before a) reporting state to give best picture b) going to sleep in order to get work out of the way From d4f877a3136cd405bc022e9002b0dab11a8e692c Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 12:26:11 +0100 Subject: [PATCH 296/353] Take multiple batches in inner loop --- equinox-sync/Sync/Projection2.fs | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 966224c91..092951540 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -440,7 +440,7 @@ module Scheduling = use _ = dispatcher.Result.Subscribe(Result >> work.Push) Async.Start(dispatcher.Pump(), cts.Token) while not cts.IsCancellationRequested do - let mutable idle, dispatcherState, remaining = true, Idle, 100 + let mutable idle, dispatcherState, remaining = true, Idle, 16 ingestStreamMerges () while remaining <> 0 do remaining <- remaining - 1 @@ -450,22 +450,24 @@ module Scheduling = let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) idle <- idle && not processedResults && not dispatched match dispatcherState with - | Idle when remaining = 0 -> - dispatcherState <- Busy - | Idle when hasCapacity -> // need to bring more work into the pool as we can't fill the work queue from what we have - match pending.TryDequeue() with - | true, batch -> - // Periodically merge events in stream-wise to maximize stream write size especially when we are behind - if remaining % 10 = 0 then ingestStreamMerges () - ingestPendingBatch stats.Handle batch - | false,_ -> - // If we're going to fill the write queue with random work, we should bring all read events into the state first - ingestStreamMerges () - dispatcherState <- Slipstreaming - | Idle -> + | Idle when not hasCapacity -> // If we've achieved full state, spin around the loop to dump stats and ingest reader data dispatcherState <- Full remaining <- 0 + | Idle when remaining = 0 -> + dispatcherState <- Busy + | Idle -> // need to bring more work into the pool as we can't fill the work queue from what we have + // If we're going to fill the write queue with random work, we should bring all read events into the state first + // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first + ingestStreamMerges () + let mutable batchesTaken = 0 + while batchesTaken < 10 do + batchesTaken <- batchesTaken + 1 + match pending.TryDequeue() with + | true, batch -> + ingestPendingBatch stats.Handle batch + | false,_ -> + dispatcherState <- Slipstreaming | Slipstreaming -> // only do one round of slipstreaming remaining <- 0 | Busy | Full -> failwith "Not handled here" From 839c54882f69369901e3009cb5dea507806b8f4d Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 12:28:44 +0100 Subject: [PATCH 297/353] Fix exit condition --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 092951540..365c1f175 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -461,7 +461,7 @@ module Scheduling = // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first ingestStreamMerges () let mutable batchesTaken = 0 - while batchesTaken < 10 do + while batchesTaken <> 10 && dispatcherState = Idle do batchesTaken <- batchesTaken + 1 match pending.TryDequeue() with | true, batch -> From 0c9d99e60917efd6a48331e1ba8f5285d2e686c6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 12:44:22 +0100 Subject: [PATCH 298/353] 32 batches --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 365c1f175..909f517a5 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -461,7 +461,7 @@ module Scheduling = // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first ingestStreamMerges () let mutable batchesTaken = 0 - while batchesTaken <> 10 && dispatcherState = Idle do + while batchesTaken < 32 && dispatcherState = Idle do batchesTaken <- batchesTaken + 1 match pending.TryDequeue() with | true, batch -> From 3625747905e17c0838244c68bacaadb7b00f808e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 12:50:11 +0100 Subject: [PATCH 299/353] Remove redundant ingest --- equinox-sync/Sync/Projection2.fs | 1 - 1 file changed, 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 909f517a5..46e60db2e 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -441,7 +441,6 @@ module Scheduling = Async.Start(dispatcher.Pump(), cts.Token) while not cts.IsCancellationRequested do let mutable idle, dispatcherState, remaining = true, Idle, 16 - ingestStreamMerges () while remaining <> 0 do remaining <- remaining - 1 // 1. propagate write write outcomes to buffer (can mark batches completed etc) From ab17e636defe468c86962f81652f960b652169be Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 13:02:35 +0100 Subject: [PATCH 300/353] 64 batches --- equinox-sync/Sync/Projection2.fs | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 46e60db2e..7bdf2de65 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -364,8 +364,9 @@ module Scheduling = /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) /// c) submits work to the supplied Dispatcher (which it triggers pumping of) /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams) = + type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = let sleepIntervalMs = 1 + let maxBatches = defaultArg maxBatches 64 let cts = new CancellationTokenSource() let work = ConcurrentStack>()// dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better let slipstreamed = ResizeArray() // pulled from `work` and kept aside for processing at the right time as they are encountered @@ -459,14 +460,19 @@ module Scheduling = // If we're going to fill the write queue with random work, we should bring all read events into the state first // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first ingestStreamMerges () - let mutable batchesTaken = 0 - while batchesTaken < 32 && dispatcherState = Idle do - batchesTaken <- batchesTaken + 1 + let mutable more, batchesTaken = true, 0 + while more do match pending.TryDequeue() with | true, batch -> ingestPendingBatch stats.Handle batch - | false,_ -> + batchesTaken <- batchesTaken + 1 + more <- batchesTaken < maxBatches + | false,_ when batchesTaken <> 0 -> + more <- false + | false,_ when batchesTaken = 0 -> dispatcherState <- Slipstreaming + more <- false + | false,_ -> () | Slipstreaming -> // only do one round of slipstreaming remaining <- 0 | Busy | Full -> failwith "Not handled here" From 53259850b5b0a44ed8a41402054d6f67eea06bde Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 13:27:13 +0100 Subject: [PATCH 301/353] Change queueWeight to event count --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 7bdf2de65..0a2c01bf4 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -241,7 +241,7 @@ module Scheduling = member __.SetMalformed(stream,isMalformed) = updateWritePos stream isMalformed None [| { index = 0L; events = null } |] member __.QueueWeight(stream) = - states.[stream].queue.[0].events |> Seq.sumBy eventSize + states.[stream].queue.[0].events.Length // HACK |> Seq.sumBy eventSize member __.MarkBusy stream = markBusy stream member __.MarkCompleted(stream, index) = From 18061c1d8689619f2a044addadf0958162d67c09 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 13:46:39 +0100 Subject: [PATCH 302/353] Change max size from 512KB to 1MB --- equinox-sync/Sync/CosmosIngester.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 3f0cd3bd4..5150e5382 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -120,7 +120,7 @@ let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, categorize, let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 let writerResultLog = log.ForContext() let trim (_currentWritePos : int64 option, batch : StreamSpan) = - let mutable countBudget, bytesBudget = 16384, 512 * 1024 - (*fudge*)4096 + let mutable countBudget, bytesBudget = 16384, 1024 * 1024 - (*fudge*)4096 let mutable count = 0 let withinLimits (y : Equinox.Codec.IEvent) = count <- count + 1 From e563e60a08b80638ba55d318f8ea4040e95b7a7f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 13 May 2019 16:39:24 +0100 Subject: [PATCH 303/353] Roll back to 32 batches --- equinox-sync/Sync/Projection2.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 0a2c01bf4..1b3c913a7 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -241,7 +241,7 @@ module Scheduling = member __.SetMalformed(stream,isMalformed) = updateWritePos stream isMalformed None [| { index = 0L; events = null } |] member __.QueueWeight(stream) = - states.[stream].queue.[0].events.Length // HACK |> Seq.sumBy eventSize + states.[stream].queue.[0].events |> Seq.sumBy eventSize member __.MarkBusy stream = markBusy stream member __.MarkCompleted(stream, index) = @@ -366,7 +366,7 @@ module Scheduling = /// d) periodically reports state (with hooks for ingestion engines to report same) type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = let sleepIntervalMs = 1 - let maxBatches = defaultArg maxBatches 64 + let maxBatches = defaultArg maxBatches 32 let cts = new CancellationTokenSource() let work = ConcurrentStack>()// dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better let slipstreamed = ResizeArray() // pulled from `work` and kept aside for processing at the right time as they are encountered From c31db26d82e93ac91039aa4414082de853d98be6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 00:08:14 +0100 Subject: [PATCH 304/353] Add phase timing logging --- equinox-sync/Sync/Projection2.fs | 35 ++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 1b3c913a7..d8898490f 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -7,6 +7,7 @@ open System.Collections.Concurrent open System.Collections.Generic open System.Diagnostics open System.Threading +open Equinox.Store /// Gathers stats relating to how many items of a given category have been observed type CatStats() = @@ -300,7 +301,11 @@ module Scheduling = let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 let merges, mergedStreams, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) + let mutable dt,ft,it,st,mt = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero let dumpStats (used,maxDop) (waitingBatches,pendingMerges) = + log.Information("Timing Merge {mt:n1}s Ingest {it:n1}s Fill {ft:n1}s Drain {dt:n1}s Stats {st:n1}s", + mt.TotalSeconds,it.TotalSeconds,ft.TotalSeconds,dt.TotalSeconds,st.TotalSeconds) + dt <- TimeSpan.Zero; ft <- TimeSpan.Zero; it <- TimeSpan.Zero; st <- TimeSpan.Zero; mt <- TimeSpan.Zero log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 @@ -326,9 +331,15 @@ module Scheduling = if statsDue () then dumpStats (used,max) pendingCount __.DumpExtraStats() - member __.TryDumpState(state,dump) = + member __.TryDumpState(state,dump,(_dt,_ft,_mt,_it,_st)) = + dt <- dt + _dt + ft <- ft + _ft + mt <- mt + _mt + it <- it + _it + st <- st + _st incr fullCycles states.Ingest(string state) + let due = stateDue () if due then dump log @@ -368,7 +379,7 @@ module Scheduling = let sleepIntervalMs = 1 let maxBatches = defaultArg maxBatches 32 let cts = new CancellationTokenSource() - let work = ConcurrentStack>()// dont need so complexity of Queue is unwarranted and usage is cross thread so Bag is not better + let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better let slipstreamed = ResizeArray() // pulled from `work` and kept aside for processing at the right time as they are encountered let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering let streams = StreamStates() @@ -442,12 +453,18 @@ module Scheduling = Async.Start(dispatcher.Pump(), cts.Token) while not cts.IsCancellationRequested do let mutable idle, dispatcherState, remaining = true, Idle, 16 + let mutable dt,ft,mt,it,st = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero + let inline accStopwatch (f : unit -> 't) at = + let t,r = f |> Stopwatch.Time + at t.Elapsed + r while remaining <> 0 do remaining <- remaining - 1 // 1. propagate write write outcomes to buffer (can mark batches completed etc) - let processedResults = tryDrainResults stats.Handle + let processedResults = (fun () -> tryDrainResults stats.Handle) |> accStopwatch <| fun x -> dt <- dt + x // 2. top up provisioning of writers queue - let! hasCapacity, dispatched = tryFillDispatcher (dispatcherState = Slipstreaming) + let! _ft,(hasCapacity, dispatched) = tryFillDispatcher (dispatcherState = Slipstreaming) |> Stopwatch.Time + ft <- ft + _ft.Elapsed idle <- idle && not processedResults && not dispatched match dispatcherState with | Idle when not hasCapacity -> @@ -459,12 +476,12 @@ module Scheduling = | Idle -> // need to bring more work into the pool as we can't fill the work queue from what we have // If we're going to fill the write queue with random work, we should bring all read events into the state first // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first - ingestStreamMerges () + ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t let mutable more, batchesTaken = true, 0 while more do match pending.TryDequeue() with | true, batch -> - ingestPendingBatch stats.Handle batch + (fun () -> ingestPendingBatch stats.Handle batch) |> accStopwatch <| fun t -> it <- it + t batchesTaken <- batchesTaken + 1 more <- batchesTaken < maxBatches | false,_ when batchesTaken <> 0 -> @@ -477,11 +494,11 @@ module Scheduling = remaining <- 0 | Busy | Full -> failwith "Not handled here" // This loop can take a long time; attempt logging of stats per iteration - stats.DumpStats(dispatcher.State,(pending.Count,slipstreamed.Count)) + (fun () -> stats.DumpStats(dispatcher.State,(pending.Count,slipstreamed.Count))) |> accStopwatch <| fun t -> st <- st + t // Do another ingest before a) reporting state to give best picture b) going to sleep in order to get work out of the way - ingestStreamMerges () + ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently - if not (stats.TryDumpState(dispatcherState,dumpStreams streams)) && not idle then + if not (stats.TryDumpState(dispatcherState,dumpStreams streams,(dt,ft,mt,it,st))) && not idle then // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) do! Async.Sleep sleepIntervalMs } From 8f7203d1b8d2882f8487eb455fac5b4cdd2237fc Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 00:12:53 +0100 Subject: [PATCH 305/353] Remove GSW --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index d8898490f..3e22d6cfd 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -74,7 +74,7 @@ module Progress = batch <- batch + 1 for s in x.streamToRequiredIndex.Keys do if streams.Add s then - yield s,(batch,getStreamWeight s) } + yield s,(batch,0(*getStreamWeight s*)) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst /// Manages writing of progress From c2af38e3a05607b94c37d4e9fdaad1662723c7e6 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 00:24:18 +0100 Subject: [PATCH 306/353] Remove async from dispatch --- equinox-sync/Sync/Projection2.fs | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 3e22d6cfd..9b65d33c6 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -45,6 +45,8 @@ module private Impl = member __.Await() = inner.Await() |> Async.Ignore /// Wait for the specified timeout to acquire (or return false instantly) member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) + /// Dont use + member __.TryWaitWithoutCancellationForPerf() = inner.Wait(0) member __.HasCapacity = inner.CurrentCount > 0 module Progress = @@ -361,11 +363,8 @@ module Scheduling = [] member __.Result = result.Publish member __.HasCapacity = dop.HasCapacity member __.State = dop.State - member __.TryAdd(item,?timeout) = async { - let! got = dop.TryAwait(?timeout=timeout) - if got then - work.Add(item) - return got } + member __.TryAdd(item) = + if dop.TryWaitWithoutCancellationForPerf() then work.Add(item); true else false member __.Pump () = async { let! ct = Async.CancellationToken for item in work.GetConsumingEnumerable ct do @@ -417,18 +416,18 @@ module Scheduling = streams.InternalMerge combined slipstreamed.Clear() // On ech iteration, we try to fill the in-flight queue, taking the oldest and/or heaviest streams first - let tryFillDispatcher includeSlipstreamed = async { + let tryFillDispatcher includeSlipstreamed = let mutable hasCapacity, dispatched = dispatcher.HasCapacity, false if hasCapacity then let potential = streams.Pending(includeSlipstreamed, progressState.InScheduledOrder streams.QueueWeight) let xs = potential.GetEnumerator() while xs.MoveNext() && hasCapacity do let (_,{stream = s} : StreamSpan) as item = xs.Current - let! succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) + let succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) if succeeded then streams.MarkBusy s dispatched <- dispatched || succeeded // if we added any request, we also don't sleep hasCapacity <- succeeded - return hasCapacity, dispatched } + hasCapacity, dispatched // Take an incoming batch of events, correlating it against our known stream state to yield a set of remaining work let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = let inline validVsSkip (streamState : StreamState) (item : StreamItem) = From 2c53849134ed4c2bc21b8701ec70b487d9da220e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 00:28:38 +0100 Subject: [PATCH 307/353] Remove more Async --- equinox-sync/Sync/Projection2.fs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 9b65d33c6..2f7d42c51 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -462,8 +462,7 @@ module Scheduling = // 1. propagate write write outcomes to buffer (can mark batches completed etc) let processedResults = (fun () -> tryDrainResults stats.Handle) |> accStopwatch <| fun x -> dt <- dt + x // 2. top up provisioning of writers queue - let! _ft,(hasCapacity, dispatched) = tryFillDispatcher (dispatcherState = Slipstreaming) |> Stopwatch.Time - ft <- ft + _ft.Elapsed + let hasCapacity, dispatched = (fun () -> tryFillDispatcher (dispatcherState = Slipstreaming)) |> accStopwatch <| fun x -> ft <- ft + x idle <- idle && not processedResults && not dispatched match dispatcherState with | Idle when not hasCapacity -> @@ -499,7 +498,7 @@ module Scheduling = // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently if not (stats.TryDumpState(dispatcherState,dumpStreams streams,(dt,ft,mt,it,st))) && not idle then // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) - do! Async.Sleep sleepIntervalMs } + Thread.Sleep sleepIntervalMs } // Not Async.Sleep so we don't give up the thread static member Start<'R>(stats, projectorDop, project, interpretProgress, dumpStreams) = let dispatcher = Dispatcher(projectorDop) From f31cdf4e0b0f5991b756e75e759f25dcc5db339b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 00:32:03 +0100 Subject: [PATCH 308/353] Remove IsEmpty --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 2f7d42c51..1448f5378 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -390,7 +390,7 @@ module Scheduling = let mutable worked, more = false, true while more do let c = work.TryPopRange(workLocalBuffer) - if c = 0 && work.IsEmpty then more <- false else worked <- true + if c = 0 then more <- false else worked <- true for i in 0..c-1 do let x = workLocalBuffer.[i] match x with From dcdb156890caf952cfc6bab9f02425daf69ebf96 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 00:33:41 +0100 Subject: [PATCH 309/353] reinstate weight --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 1448f5378..5539e4885 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -76,7 +76,7 @@ module Progress = batch <- batch + 1 for s in x.streamToRequiredIndex.Keys do if streams.Add s then - yield s,(batch,0(*getStreamWeight s*)) } + yield s,(batch,getStreamWeight s) } raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst /// Manages writing of progress From aeb8551fbf217ea8445737919b06e5467d7767c9 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 01:04:00 +0100 Subject: [PATCH 310/353] Fix sort perf --- equinox-sync/Sync/Projection2.fs | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 5539e4885..a3d4baad1 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -69,15 +69,16 @@ module Progress = | _, _ -> () trim () member __.InScheduledOrder getStreamWeight = - let raw = seq { - let streams = HashSet() - let mutable batch = 0 - for x in pending do - batch <- batch + 1 - for s in x.streamToRequiredIndex.Keys do - if streams.Add s then - yield s,(batch,getStreamWeight s) } - raw |> Seq.sortBy (fun (_s,(b,l)) -> b,-l) |> Seq.map fst + let streams = HashSet() + let tmp = ResizeArray(16384) + let mutable batch = 0 + for x in pending do + batch <- batch + 1 + for s in x.streamToRequiredIndex.Keys do + if streams.Add s then + tmp.Add(struct (s,struct (batch,-getStreamWeight s))) + tmp.Sort(fun (struct (_,a)) (struct (_,b)) -> a.CompareTo(b) ) + seq { for s,_ in tmp -> s } /// Manages writing of progress /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) From 38f70118d1835d1065efbf93f61e10c4778164de Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 01:08:46 +0100 Subject: [PATCH 311/353] Fix warning --- equinox-sync/Sync/Projection2.fs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index a3d4baad1..a1edbeecb 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -49,6 +49,8 @@ module private Impl = member __.TryWaitWithoutCancellationForPerf() = inner.Wait(0) member __.HasCapacity = inner.CurrentCount > 0 +#nowarn "52" // see tmp.Sort + module Progress = type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } From 1d402ce31b007fd898a3cb267f5045c119fa26fb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 01:16:46 +0100 Subject: [PATCH 312/353] Final fix --- equinox-sync/Sync/Projection2.fs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index a1edbeecb..b52f75798 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -79,8 +79,9 @@ module Progress = for s in x.streamToRequiredIndex.Keys do if streams.Add s then tmp.Add(struct (s,struct (batch,-getStreamWeight s))) - tmp.Sort(fun (struct (_,a)) (struct (_,b)) -> a.CompareTo(b) ) - seq { for s,_ in tmp -> s } + let a = tmp.ToArray() + Array.sortInPlaceBy(fun (struct(_a,o)) -> o) a + seq { for s,_ in a -> s } /// Manages writing of progress /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) From 7e760c40907749e183f939bce5cf6599a09dab9a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 01:20:07 +0100 Subject: [PATCH 313/353] Final indignity --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index b52f75798..405944927 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -81,7 +81,7 @@ module Progress = tmp.Add(struct (s,struct (batch,-getStreamWeight s))) let a = tmp.ToArray() Array.sortInPlaceBy(fun (struct(_a,o)) -> o) a - seq { for s,_ in a -> s } + a |> Seq.map (fun (struct(s,_)) -> s) /// Manages writing of progress /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) From 4771597150dc22e0d1531872200acadb08b3c940 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 01:24:12 +0100 Subject: [PATCH 314/353] No tmp --- equinox-sync/Sync/Projection2.fs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 405944927..bf832da0a 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -79,9 +79,8 @@ module Progress = for s in x.streamToRequiredIndex.Keys do if streams.Add s then tmp.Add(struct (s,struct (batch,-getStreamWeight s))) - let a = tmp.ToArray() - Array.sortInPlaceBy(fun (struct(_a,o)) -> o) a - a |> Seq.map (fun (struct(s,_)) -> s) + tmp.Sort(fun (struct(_,_a)) (struct(_,_b)) -> _a.CompareTo(_b)) + tmp |> Seq.map (fun (struct(s,_)) -> s) /// Manages writing of progress /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) From 9008cd363b7731b619fb0fc0280fb57aaaf7c099 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 01:31:57 +0100 Subject: [PATCH 315/353] Reduce maxBatches a lot --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index bf832da0a..65822a1ee 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -379,7 +379,7 @@ module Scheduling = /// d) periodically reports state (with hooks for ingestion engines to report same) type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = let sleepIntervalMs = 1 - let maxBatches = defaultArg maxBatches 32 + let maxBatches = defaultArg maxBatches 4 let cts = new CancellationTokenSource() let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better let slipstreamed = ResizeArray() // pulled from `work` and kept aside for processing at the right time as they are encountered From 015a2b95d106db68f9c8fad65403b79634841dcf Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 02:32:22 +0100 Subject: [PATCH 316/353] Background stream merge --- equinox-sync/Sync/Projection2.fs | 47 ++++++++++++++++++++------------ 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 65822a1ee..81da935f4 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -45,8 +45,10 @@ module private Impl = member __.Await() = inner.Await() |> Async.Ignore /// Wait for the specified timeout to acquire (or return false instantly) member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) - /// Dont use + /// Dont use without profiling proving it helps as it doesnt help correctness or legibility member __.TryWaitWithoutCancellationForPerf() = inner.Wait(0) + /// Only use where you're interested in intenionally busywaiting on a thread - i.e. when you have proven its critical + member __.SpinWaitWithoutCancellationForPerf() = inner.Wait(Timeout.Infinite) |> ignore member __.HasCapacity = inner.CurrentCount > 0 #nowarn "52" // see tmp.Sort @@ -154,7 +156,7 @@ module Buffer = | None, _ -> true | _ -> false module StreamState = - let inline optionCombine f (r1: int64 option) (r2: int64 option) = + let inline optionCombine f (r1: 'a option) (r2: 'a option) = match r1, r2 with | Some x, Some y -> f x y |> Some | None, None -> None @@ -382,11 +384,32 @@ module Scheduling = let maxBatches = defaultArg maxBatches 4 let cts = new CancellationTokenSource() let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better - let slipstreamed = ResizeArray() // pulled from `work` and kept aside for processing at the right time as they are encountered let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering let streams = StreamStates() let progressState = Progress.State() + // Arguably could be a bag, which would be more efficient, but sequencing in order of submission yields cheaper merges + let streamsPending = ConcurrentQueue() // pulled from `work` and kept aside for processing at the right time as they are encountered + let mutable streamsMerged : Streams option = None + let slipstreamsCoalescing = Sem(1) + let tryGetStream () = match streamsPending.TryDequeue() with true,x -> Some x | false,_-> None + // We periodically process streamwise submissions of events from Ingesters in advance of them entering the processing queue as pending batches + // This allows events that are not yet a requirement for a given batch to complete to be included in work before it becomes due, smoothing throughput + let continuouslyCompactStreamMerges () = async { + let! ct = Async.CancellationToken + while not ct.IsCancellationRequested do + do! slipstreamsCoalescing.Await() + streamsMerged <- (streamsMerged,tryGetStream()) ||> StreamState.optionCombine (fun x y -> x.Merge y; x) + slipstreamsCoalescing.Release() + do! Async.Sleep 5 } // ms // needs to be long enough for ingestStreamMerges to be able to grab + let ingestStreamMerges () = + slipstreamsCoalescing.SpinWaitWithoutCancellationForPerf() + match streamsMerged with + | None -> () + | Some ready -> + streamsMerged <- None + streams.InternalMerge ready + slipstreamsCoalescing.Release() // ingest information to be gleaned from processing the results into `streams` static let workLocalBuffer = Array.zeroCreate 1024 let tryDrainResults feedStats = @@ -398,7 +421,7 @@ module Scheduling = let x = workLocalBuffer.[i] match x with | Added _ -> () // Only processed in Stats (and actually never enters this queue) - | Merge buffer -> slipstreamed.Add buffer // put aside as a) they can be done more efficiently in bulk b) we only want to pay the tax at the right time + | Merge buffer -> streamsPending.Enqueue buffer // put aside as a) they can be done more efficiently in bulk b) we only want to pay the tax at the right time | Result (stream,res) -> match interpretProgress streams stream res with | None -> streams.MarkFailed stream @@ -407,17 +430,6 @@ module Scheduling = streams.MarkCompleted(stream,index) feedStats x worked - // We periodically process streamwise submissions of events from Ingesters in advance of them entering the processing queue as pending batches - // This allows events that are not yet a requirement for a given batch to complete to be included in work before it becomes due, smoothing throughput - let ingestStreamMerges () = - match slipstreamed.ToArray() with - | [||] -> () - | [| one |] -> streams.InternalMerge one - | many -> - let combined = many.[0] - for x in Array.skip 1 many do combined.Merge x - streams.InternalMerge combined - slipstreamed.Clear() // On ech iteration, we try to fill the in-flight queue, taking the oldest and/or heaviest streams first let tryFillDispatcher includeSlipstreamed = let mutable hasCapacity, dispatched = dispatcher.HasCapacity, false @@ -453,6 +465,7 @@ module Scheduling = member private __.Pump(stats : Stats<'R>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Push) Async.Start(dispatcher.Pump(), cts.Token) + Async.Start(continuouslyCompactStreamMerges (), cts.Token) while not cts.IsCancellationRequested do let mutable idle, dispatcherState, remaining = true, Idle, 16 let mutable dt,ft,mt,it,st = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero @@ -477,9 +490,9 @@ module Scheduling = | Idle -> // need to bring more work into the pool as we can't fill the work queue from what we have // If we're going to fill the write queue with random work, we should bring all read events into the state first // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first - ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t let mutable more, batchesTaken = true, 0 while more do + ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t match pending.TryDequeue() with | true, batch -> (fun () -> ingestPendingBatch stats.Handle batch) |> accStopwatch <| fun t -> it <- it + t @@ -495,7 +508,7 @@ module Scheduling = remaining <- 0 | Busy | Full -> failwith "Not handled here" // This loop can take a long time; attempt logging of stats per iteration - (fun () -> stats.DumpStats(dispatcher.State,(pending.Count,slipstreamed.Count))) |> accStopwatch <| fun t -> st <- st + t + (fun () -> stats.DumpStats(dispatcher.State,(pending.Count,streamsPending.Count))) |> accStopwatch <| fun t -> st <- st + t // Do another ingest before a) reporting state to give best picture b) going to sleep in order to get work out of the way ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently From 5349f213b5e744e151e4a24b9c65bd9d06e75285 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 02:53:56 +0100 Subject: [PATCH 317/353] Forward merges immediately --- equinox-sync/Sync/Projection2.fs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 81da935f4..70200b9ca 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -401,7 +401,7 @@ module Scheduling = do! slipstreamsCoalescing.Await() streamsMerged <- (streamsMerged,tryGetStream()) ||> StreamState.optionCombine (fun x y -> x.Merge y; x) slipstreamsCoalescing.Release() - do! Async.Sleep 5 } // ms // needs to be long enough for ingestStreamMerges to be able to grab + do! Async.Sleep 1 } // ms // needs to be long enough for ingestStreamMerges to be able to grab let ingestStreamMerges () = slipstreamsCoalescing.SpinWaitWithoutCancellationForPerf() match streamsMerged with @@ -688,7 +688,6 @@ module Ingestion = member private __.Pump() = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) - let presubmitInterval = expiredMs (1000L*10L) while not cts.IsCancellationRequested do try let mutable itemLimit = 4096 while itemLimit > 0 do @@ -699,9 +698,8 @@ module Ingestion = stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch - // 2. Forward content for any active streams into processor immediately - if presubmitInterval () then - grabAccumulatedStreams () |> scheduler.SubmitStreamBuffers + // 2. Forward info grouped by streams into processor immediately + grabAccumulatedStreams () |> scheduler.SubmitStreamBuffers // 3. Submit to ingester until read queue, tranche limit or ingester limit exhausted while pending.Count <> 0 && submissionsMax.HasCapacity do From 2325b5f94a076d861ab3307adcb7e103ec606e04 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 02:58:53 +0100 Subject: [PATCH 318/353] Reduce merge frequency --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 70200b9ca..59801c295 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -491,8 +491,8 @@ module Scheduling = // If we're going to fill the write queue with random work, we should bring all read events into the state first // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first let mutable more, batchesTaken = true, 0 + ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t while more do - ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t match pending.TryDequeue() with | true, batch -> (fun () -> ingestPendingBatch stats.Handle batch) |> accStopwatch <| fun t -> it <- it + t From 3dfe73d33d40020b61df7e65bb4c81ed3915af5e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 03:11:10 +0100 Subject: [PATCH 319/353] Throttle batch submission --- equinox-sync/Sync/Projection2.fs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 59801c295..cc5d761df 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -688,6 +688,7 @@ module Ingestion = member private __.Pump() = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) + let presubmitInterval = expiredMs (1000L*2L) while not cts.IsCancellationRequested do try let mutable itemLimit = 4096 while itemLimit > 0 do @@ -698,8 +699,9 @@ module Ingestion = stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) validatedPos |> Option.iter progressWriter.Post stats.HandleCommitted progressWriter.CommittedEpoch - // 2. Forward info grouped by streams into processor immediately - grabAccumulatedStreams () |> scheduler.SubmitStreamBuffers + // 2. Forward info grouped by streams into processor in small batches + if presubmitInterval () then + grabAccumulatedStreams () |> scheduler.SubmitStreamBuffers // 3. Submit to ingester until read queue, tranche limit or ingester limit exhausted while pending.Count <> 0 && submissionsMax.HasCapacity do From c718336f86bf307af8c76a7c76a41ac1f8324f4f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 03:37:09 +0100 Subject: [PATCH 320/353] 4s submit interval --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index cc5d761df..88d89e315 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -688,7 +688,7 @@ module Ingestion = member private __.Pump() = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) - let presubmitInterval = expiredMs (1000L*2L) + let presubmitInterval = expiredMs (4000L*2L) while not cts.IsCancellationRequested do try let mutable itemLimit = 4096 while itemLimit > 0 do From d8eca451f21c49f25d74a28c153f4ff351be00fc Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 03:45:49 +0100 Subject: [PATCH 321/353] 16 batches --- equinox-sync/Sync/Projection2.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 88d89e315..78f9f89d2 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -381,7 +381,7 @@ module Scheduling = /// d) periodically reports state (with hooks for ingestion engines to report same) type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = let sleepIntervalMs = 1 - let maxBatches = defaultArg maxBatches 4 + let maxBatches = defaultArg maxBatches 16 let cts = new CancellationTokenSource() let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering From 3f3f29b686ba568ccfd1c6478ae9abdefcb288b8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 05:09:57 +0100 Subject: [PATCH 322/353] Grayscale --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2f602e69f..2f43f452e 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -303,7 +303,7 @@ module Logging = let isCfp429b = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.LeaseRenewer").Invoke let isCfp429c = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.PartitionLoadBalancer").Invoke (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isCp x || isWriter x || isCfp429a x || isCfp429b x || isCfp429c x)) - .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) + .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Grayscale, outputTemplate=t) |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) From 6228d423cc64173ef89b4c25a82897f3e94d5e7a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 05:49:31 +0100 Subject: [PATCH 323/353] Rebase on Equinox 2.0.0-preview7 --- equinox-projector/Consumer/Consumer.fsproj | 4 +- equinox-projector/Projector/Program.fs | 52 +++++++++++--------- equinox-projector/Projector/Projector.fsproj | 4 +- equinox-sync/Sync/CosmosIngester.fs | 1 + equinox-sync/Sync/CosmosSource.fs | 6 +-- equinox-sync/Sync/EventStoreSource.fs | 14 +++--- equinox-sync/Sync/Projection2.fs | 27 +++------- equinox-sync/Sync/Sync.fsproj | 8 +-- 8 files changed, 54 insertions(+), 62 deletions(-) diff --git a/equinox-projector/Consumer/Consumer.fsproj b/equinox-projector/Consumer/Consumer.fsproj index 279d4741c..f2629806a 100644 --- a/equinox-projector/Consumer/Consumer.fsproj +++ b/equinox-projector/Consumer/Consumer.fsproj @@ -14,8 +14,8 @@ - - + + diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 3db69e3aa..3021b499b 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -76,8 +76,8 @@ module CmdParser = | [] LeaseCollectionSuffix of string | [] FromTail | [] MaxDocuments of int - | [] MaxPendingBatches of int - | [] ProcessorDop of int + | [] MaxReadAhead of int + | [] MaxWriters of int | [] LagFreqS of float | [] Verbose | [] ChangeFeedVerbose @@ -85,6 +85,8 @@ module CmdParser = (* Kafka Args *) | [] Broker of string | [] Topic of string +//#else + | [] MaxSubmissionsPerRange of int //#endif (* ChangeFeed Args *) | [] Cosmos of ParseResults @@ -95,37 +97,45 @@ module CmdParser = | LeaseCollectionSuffix _ -> "specify Collection Name suffix for Leases collection (default: `-aux`)." | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." | MaxDocuments _ -> "maxiumum document count to supply for the Change Feed query. Default: use response size limit" - | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" - | ProcessorDop _ -> "Maximum number of streams to process concurrently. Default: 64" + | MaxReadAhead _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" + | MaxWriters _ -> "Maximum number of concurrent streams on which to process at any time. Default: 1024" | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | Verbose -> "request Verbose Logging. Default: off" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" //#if kafka | Broker _ -> "specify Kafka Broker, in host:port format. (default: use environment variable EQUINOX_KAFKA_BROKER, if specified)" | Topic _ -> "specify Kafka Topic Id. (default: use environment variable EQUINOX_KAFKA_TOPIC, if specified)" +//#else + | MaxSubmissionsPerRange _ -> "Maximum number of batches to submit to the processor at any time. Default: 32" //#endif | Cosmos _ -> "specify CosmosDb input parameters" and Arguments(args : ParseResults) = member val Cosmos = Cosmos.Arguments(args.GetResult Cosmos) //#if kafka member val Target = TargetInfo args +//#else + member __.MaxSubmissionPerRange = args.GetResult(MaxSubmissionsPerRange,32) //#endif member __.LeaseId = args.GetResult ConsumerGroupName member __.Suffix = args.GetResult(LeaseCollectionSuffix,"-aux") member __.Verbose = args.Contains Verbose member __.ChangeFeedVerbose = args.Contains ChangeFeedVerbose member __.MaxDocuments = args.TryGetResult MaxDocuments - member __.MaxPendingBatches = args.GetResult(MaxPendingBatches,64) - member __.ProcessorDop = args.GetResult(ProcessorDop,64) + member __.MaxReadAhead = args.GetResult(MaxReadAhead,64) + member __.ProcessorDop = args.GetResult(MaxWriters,64) member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.AuxCollectionName = __.Cosmos.Collection + __.Suffix member x.BuildChangeFeedParams() = match x.MaxDocuments with - | None -> Log.Information("Processing {leaseId} in {auxCollName} without document count limit (<= {maxPending} pending) using {dop} processors", x.LeaseId, x.AuxCollectionName, x.MaxPendingBatches, x.ProcessorDop) - | Some lim -> Log.Information("Processing {leaseId} in {auxCollName} with max {changeFeedMaxDocuments} documents (<= {maxPending} pending) using {dop} processors", x.LeaseId, x.AuxCollectionName, x.MaxDocuments, x.MaxPendingBatches, x.ProcessorDop) + | None -> + Log.Information("Processing {leaseId} in {auxCollName} without document count limit (<= {maxPending} pending) using {dop} processors", + x.LeaseId, x.AuxCollectionName, x.MaxReadAhead, x.ProcessorDop) + | Some lim -> + Log.Information("Processing {leaseId} in {auxCollName} with max {changeFeedMaxDocuments} documents (<= {maxPending} pending) using {dop} processors", + x.LeaseId, x.AuxCollectionName, lim, x.MaxReadAhead, x.ProcessorDop) if args.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) - { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, args.Contains FromTail, x.MaxDocuments, x.MaxPendingBatches, x.ProcessorDop, x.LagFrequency + { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, args.Contains FromTail, x.MaxDocuments, (x.MaxReadAhead, x.MaxSubmissionPerRange, x.ProcessorDop), x.LagFrequency //#if kafka and TargetInfo(args : ParseResults) = member __.Broker = Uri(match args.TryGetResult Broker with Some x -> x | None -> envBackstop "Broker" "EQUINOX_KAFKA_BROKER") @@ -148,17 +158,11 @@ let run (log : ILogger) discovery connectionPolicy source let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = ChangeFeedProcessor.Start - ( log, discovery, connectionPolicy, source, aux, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, - createObserver = createRangeProjector, ?cfBatchSize = maybeLimitDocumentCount, ?reportLagAndAwaitNextEstimation = maybeLogLag) + ( log, discovery, connectionPolicy, source, aux, leasePrefix = leaseId, startFromTail = startFromTail, + createObserver = createRangeProjector, ?maxDocuments = maybeLimitDocumentCount, ?reportLagAndAwaitNextEstimation = maybeLogLag) do! Async.AwaitKeyboardInterrupt() } //#if kafka -// TODO remove when using 2.0.0-preview7 -open Equinox.Projection.Codec -module RenderedEvent = - let ofStreamItem (x: Equinox.Projection.StreamItem) : RenderedEvent = - { s = x.stream; i = x.index; c = x.event.EventType; t = x.event.Timestamp; d = x.event.Data; m = x.event.Meta } - let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = CompressionType.Lz4) @@ -172,7 +176,6 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = // TODO handle failure let! _ = producer.ProduceBatch es return! ctx.Checkpoint() |> Stopwatch.Time } - log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) @@ -180,11 +183,11 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = } ChangeFeedObserver.Create(log, ingest, dispose = disposeProducer) //#else -let createRangeHandler (log:ILogger) (maxPendingBatches, processorDop, project) = - let projectionEngine = Projector.Start (log, maxPendingBatches, processorDop, project, TimeSpan.FromMinutes 1.) +let createRangeHandler (log:ILogger) (maxReadAhead, maxSubmissionsPerRange, projectionDop) categorize project = + let scheduler = Projector.Start (log, projectionDop, project, categorize, TimeSpan.FromMinutes 1.) fun () -> let mutable rangeIngester = Unchecked.defaultof - let init rangeLog = async { rangeIngester <- Ingester.Start (rangeLog, projectionEngine, maxPendingBatches, processorDop, TimeSpan.FromMinutes 1.) } + let init rangeLog = async { rangeIngester <- Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) } let ingest epoch checkpoint docs = let items = Seq.collect DocumentParser.enumEvents docs rangeIngester.Submit(epoch, checkpoint, items) @@ -223,19 +226,20 @@ let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ChangeFeedVerbose let discovery, connector, source = args.Cosmos.BuildConnectionDetails() - let aux, leaseId, startFromTail, maxDocuments, maxPendingBatches, processorDop, lagFrequency = args.BuildChangeFeedParams() + let aux, leaseId, startFromTail, maxDocuments, limits, lagFrequency = args.BuildChangeFeedParams() //#if kafka //let targetParams = args.Target.BuildTargetParams() //let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams //#endif - let project (batch : Equinox.Projection.State.StreamSpan) = async { + let project (batch : Equinox.Projection.Buffer.StreamSpan) = async { let r = Random() let ms = r.Next(1,batch.span.events.Length * 10) do! Async.Sleep ms return batch.span.events.Length } + let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] run Log.Logger discovery connector.ConnectionPolicy source (aux, leaseId, startFromTail, maxDocuments, lagFrequency) - (createRangeHandler Log.Logger (maxPendingBatches, processorDop, project)) + (createRangeHandler Log.Logger limits categorize project) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 14ea9d861..6f56ae39c 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -14,9 +14,9 @@ - + - + diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 5150e5382..8c0726b2e 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -2,6 +2,7 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Store +open Equinox.Projection open Equinox.Projection2 open Equinox.Projection2.Buffer open Equinox.Projection2.Scheduling diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 517ff4704..c9a2422b6 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -29,7 +29,7 @@ let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq } ChangeFeedObserver.Create(log, processBatch, assign=init, dispose=dispose) -let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maxDocuments, lagReportFreq : TimeSpan option) +let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connectionPolicy (leaseId, startFromTail, maybeLimitDocuments, lagReportFreq : TimeSpan option) (cosmosContext, maxWriters) categorize createRangeProjector = async { @@ -40,8 +40,8 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) let! _feedEventHost = ChangeFeedProcessor.Start - ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, forceSkipExistingEvents = startFromTail, - createObserver = createRangeProjector cosmosIngester, ?reportLagAndAwaitNextEstimation = maybeLogLag, cfBatchSize = defaultArg maxDocuments 999999, + ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, startFromTail = startFromTail, + createObserver = createRangeProjector cosmosIngester, ?reportLagAndAwaitNextEstimation = maybeLogLag, ?maxDocuments = maybeLimitDocuments, leaseAcquireInterval=TimeSpan.FromSeconds 5., leaseRenewInterval=TimeSpan.FromSeconds 5., leaseTtl=TimeSpan.FromMinutes 1.) do! Async.AwaitKeyboardInterrupt() cosmosIngester.Stop() } diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index cffa467d0..3010001ab 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -1,6 +1,6 @@ module SyncTemplate.EventStoreSource -//open Equinox.Cosmos.Projection +open Equinox.Cosmos.Projection open Equinox.Store // AwaitTaskCorrect open Equinox.Projection open Equinox.Projection2 @@ -10,14 +10,16 @@ open System open System.Collections.Generic open System.Diagnostics open System.Threading -open Equinox.Cosmos.Projection type EventStore.ClientAPI.RecordedEvent with member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) -let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = State.arrayBytes x.Data + State.arrayBytes x.Metadata -let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 -let private mb x = float x / 1024. / 1024. +[] +module private Impl = + let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length + let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata + let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 + let inline mb x = float x / 1024. / 1024. let toIngestionItem (e : RecordedEvent) : StreamItem = let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata @@ -127,7 +129,7 @@ let establishMax (conn : IEventStoreConnection) = async { /// Walks a stream within the specified constraints; used to grab data when writing to a stream for which a prefix is missing /// Can throw (in which case the caller is in charge of retrying, possibly with a smaller batch size) -let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : State.StreamSpan -> Async) = +let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : Buffer.StreamSpan -> Async) = let rec fetchFrom pos limit = async { let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 78f9f89d2..397df7514 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -9,22 +9,6 @@ open System.Diagnostics open System.Threading open Equinox.Store -/// Gathers stats relating to how many items of a given category have been observed -type CatStats() = - let cats = Dictionary() - member __.Ingest(cat,?weight) = - let weight = defaultArg weight 1L - match cats.TryGetValue cat with - | true, catCount -> cats.[cat] <- catCount + weight - | false, _ -> cats.[cat] <- weight - member __.Any = cats.Count <> 0 - member __.Clear() = cats.Clear() -#if NET461 - member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortBy (fun (_,s) -> -s) -#else - member __.StatsDescending = cats |> Seq.map (|KeyValue|) |> Seq.sortByDescending snd -#endif - [] module private Impl = let (|NNA|) xs = if xs = null then Array.empty else xs @@ -37,6 +21,11 @@ module private Impl = let due = timer.ElapsedMilliseconds > ms if due then timer.Restart() due + let inline accStopwatch (f : unit -> 't) at = + let sw = Stopwatch.StartNew() + let r = f () + at sw.Elapsed + r type Sem(max) = let inner = new SemaphoreSlim(max) member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore @@ -202,7 +191,7 @@ module Buffer = module Scheduling = open Buffer - + type StreamStates() = let states = Dictionary() let update stream (state : StreamState) = @@ -469,10 +458,6 @@ module Scheduling = while not cts.IsCancellationRequested do let mutable idle, dispatcherState, remaining = true, Idle, 16 let mutable dt,ft,mt,it,st = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero - let inline accStopwatch (f : unit -> 't) at = - let t,r = f |> Stopwatch.Time - at t.Elapsed - r while remaining <> 0 do remaining <- remaining - 1 // 1. propagate write write outcomes to buffer (can mark batches completed etc) diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 94b803ecc..26875e752 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,7 +4,7 @@ Exe netcoreapp2.1 5 - $(DefineConstants);cosmos + $(DefineConstants);cosmos_ @@ -18,10 +18,10 @@ - + - - + + From 0c04058246506e7dbc971696fcd1636e10fbd0b5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 11:57:09 +0100 Subject: [PATCH 324/353] Drop level on messages that should not be Warnings --- equinox-sync/Sync/EventStoreSource.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 3010001ab..2071afeb3 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -232,7 +232,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, categorize, tryMapEven | Chunk (series, range, batchSize) -> let postBatch pos items = post (Res.Batch (series, pos, items)) use _ = Serilog.Context.LogContext.PushProperty("Tranche", series) - Log.Warning("Commencing tranche, batch size {bs}", batchSize) + Log.Information("Commencing tranche, batch size {bs}", batchSize) let! t, res = pullAll (slicesStats, overallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time match res with | PullResult.Eof pos -> @@ -289,7 +289,7 @@ type Reader(conns : _ [], defaultBatchSize, minBatchSize, categorize, tryMapEven // Jitter is most relevant when processing commences - any commencement of a chunk can trigger significant page faults on server // which we want to attempt to limit the effects of let jitterMs = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) - Log.Warning("Waiting {jitter}ms to jitter reader stripes, {currentCount} further reader stripes awaiting start", jitterMs, currentCount) + Log.Information("Waiting {jitter}ms to jitter reader stripes, {currentCount} further reader stripes awaiting start", jitterMs, currentCount) do! Async.Sleep jitterMs let! _ = Async.StartChild <| async { try let conn = selectConn () From 2df06c6dfd1664e0099806a5562dfb6c599b5273 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 13:26:37 +0100 Subject: [PATCH 325/353] Add Exception stats --- equinox-projector/Projector/Program.fs | 6 ++--- equinox-sync/Sync/CosmosIngester.fs | 32 ++++++++++++++------------ equinox-sync/Sync/Projection2.fs | 31 ++++++++++++------------- 3 files changed, 35 insertions(+), 34 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 3021b499b..0cb746d6f 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -163,7 +163,7 @@ let run (log : ILogger) discovery connectionPolicy source do! Async.AwaitKeyboardInterrupt() } //#if kafka -let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = +let mkRangeProjector (broker, topic) log (_maxPendingBatches,_maxDop,_project) _categorize () = let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = CompressionType.Lz4) let producer = KafkaProducer.Create(Log.Logger, cfg, topic) @@ -181,7 +181,7 @@ let mkRangeProjector log (_maxPendingBatches,_maxDop,_project) (broker, topic) = float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - ChangeFeedObserver.Create(log, ingest, dispose = disposeProducer) + ChangeFeedObserver.Create(log, ingest, dispose=disposeProducer) //#else let createRangeHandler (log:ILogger) (maxReadAhead, maxSubmissionsPerRange, projectionDop) categorize project = let scheduler = Projector.Start (log, projectionDop, project, categorize, TimeSpan.FromMinutes 1.) @@ -229,7 +229,7 @@ let main argv = let aux, leaseId, startFromTail, maxDocuments, limits, lagFrequency = args.BuildChangeFeedParams() //#if kafka //let targetParams = args.Target.BuildTargetParams() - //let createRangeHandler log processingParams () = mkRangeProjector log processingParams targetParams + //let createRangeHandler = mkRangeProjector targetParams //#endif let project (batch : Equinox.Projection.Buffer.StreamSpan) = async { let r = Random() diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index 8c0726b2e..b22424190 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -24,7 +24,7 @@ module Writer = | Duplicate of updatedPos: int64 | PartialDuplicate of overage: Span | PrefixMissing of batch: Span * writePos: int64 - let logTo (log: ILogger) (res : string * Choice<(int*int)*Result,exn>) = + let logTo (log: ILogger) (res : string * Choice<(int*int)*Result,(int*int)*exn>) = match res with | stream, (Choice1Of2 (_, Ok pos)) -> log.Information("Wrote {stream} up to {pos}", stream, pos) @@ -34,7 +34,7 @@ module Writer = log.Information("Requeing {stream} {pos} ({count} events)", stream, overage.index, overage.events.Length) | stream, (Choice1Of2 (_, PrefixMissing (batch,pos))) -> log.Information("Waiting {stream} missing {gap} events ({count} events @ {pos})", stream, batch.index-pos, batch.events.Length, batch.index) - | stream, Choice2Of2 exn -> + | stream, (Choice2Of2 (_, exn)) -> log.Warning(exn,"Writing {stream} failed, retrying", stream) let write (log : ILogger) (ctx : CosmosContext) ({ stream = s; span = { index = i; events = e}} : StreamSpan as batch) = async { @@ -71,21 +71,21 @@ module Writer = | ResultKind.TooLarge | ResultKind.Malformed -> true type Stats(log : ILogger, categorize, statsInterval, statesInterval) = - inherit Stats<(int*int)*Writer.Result>(log, statsInterval, statesInterval) + inherit Stats<(int*int)*Writer.Result,(int*int)*exn>(log, statsInterval, statesInterval) let okStreams, resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = HashSet(), ref 0, ref 0, ref 0, ref 0, ref 0 let badCats, failStreams, rateLimited, timedOut, tooLarge, malformed = CatStats(), HashSet(), ref 0, ref 0, ref 0, ref 0 let rlStreams, toStreams, tlStreams, mfStreams, oStreams = HashSet(), HashSet(), HashSet(), HashSet(), HashSet() - let mutable events, bytes = 0, 0L + let mutable okEvents, okBytes, exnEvents, exnBytes = 0, 0L, 0, 0L override __.DumpExtraStats() = let results = !resultOk + !resultDup + !resultPartialDup + !resultPrefix log.Information("Completed {mb:n0}MB {completed:n0}r {streams:n0}s {events:n0}e ({ok:n0} ok {dup:n0} redundant {partial:n0} partial {prefix:n0} waiting)", - mb bytes, results, okStreams.Count, events, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) - okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; events <- 0; bytes <- 0L + mb okBytes, results, okStreams.Count, okEvents, !resultOk, !resultDup, !resultPartialDup, !resultPrefix) + okStreams.Clear(); resultOk := 0; resultDup := 0; resultPartialDup := 0; resultPrefix := 0; okEvents <- 0; okBytes <- 0L if !rateLimited <> 0 || !timedOut <> 0 || !tooLarge <> 0 || !malformed <> 0 || badCats.Any then let fails = !rateLimited + !timedOut + !tooLarge + !malformed + !resultExnOther - log.Warning("Failures {fails}r {streams:n0}s Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s", - fails, failStreams.Count, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) + log.Warning("Exceptions {mb:n0}MB {fails:n0}r {streams:n0}s {events:n0}e Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s", + mb exnBytes, fails, failStreams.Count, exnEvents, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear() if badCats.Any then log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0}r {@tlStreams} Malformed {malformed:n0}r {@mfStreams} Other {other:n0}r {@oStreams}", @@ -101,15 +101,17 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = | Merge _ | Added _ -> () // Processed by standard logging already; we have nothing to add | Result (stream, Choice1Of2 ((es,bs),r)) -> adds stream okStreams - events <- events + es - bytes <- bytes + int64 bs + okEvents <- okEvents + es + okBytes <- okBytes + int64 bs match r with | Writer.Result.Ok _ -> incr resultOk | Writer.Result.Duplicate _ -> incr resultDup | Writer.Result.PartialDuplicate _ -> incr resultPartialDup | Writer.Result.PrefixMissing _ -> incr resultPrefix - | Result (stream, Choice2Of2 exn) -> + | Result (stream, Choice2Of2 ((es,bs),exn)) -> adds stream failStreams + exnEvents <- exnEvents + es + exnBytes <- exnBytes + int64 bs match Writer.classify exn with | ResultKind.RateLimited -> adds stream rlStreams; incr rateLimited | ResultKind.TimedOut -> adds stream toStreams; incr timedOut @@ -135,21 +137,21 @@ let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, categorize, let trimmed = trim batch let index = Interlocked.Increment(&robin) % cosmosContexts.Length let selectedConnection = cosmosContexts.[index] + let stats = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes try let! res = Writer.write log selectedConnection trimmed - let stats = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes return Choice1Of2 (stats,res) - with e -> return Choice2Of2 e } + with e -> return Choice2Of2 (stats,e) } let interpretWriteResultProgress (streams: Scheduling.StreamStates) stream res = let applyResultToStreamState = function | Choice1Of2 (_stats, Writer.Ok pos) -> streams.InternalUpdate stream pos null | Choice1Of2 (_stats, Writer.Duplicate pos) -> streams.InternalUpdate stream pos null | Choice1Of2 (_stats, Writer.PartialDuplicate overage) -> streams.InternalUpdate stream overage.index [|overage|] | Choice1Of2 (_stats, Writer.PrefixMissing (overage,pos)) -> streams.InternalUpdate stream pos [|overage|] - | Choice2Of2 exn -> + | Choice2Of2 (_stats, exn) -> let malformed = Writer.classify exn |> Writer.isMalformed streams.SetMalformed(stream,malformed) let _stream, { write = wp } = applyResultToStreamState res Writer.logTo writerResultLog (stream,res) wp let projectionAndCosmosStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) - Engine<(int*int)*Writer.Result>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file + Engine<(int*int)*Writer.Result,_>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 397df7514..32eb3b3a8 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -7,7 +7,6 @@ open System.Collections.Concurrent open System.Collections.Generic open System.Diagnostics open System.Threading -open Equinox.Store [] module private Impl = @@ -283,32 +282,32 @@ module Scheduling = /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners [] - type InternalMessage<'R> = + type InternalMessage<'R,'E> = /// Periodic submission of events as they are read, grouped by stream for efficient merging into the StreamState | Merge of Streams /// Stats per submitted batch for stats listeners to aggregate | Added of streams: int * skip: int * events: int /// Result of processing on stream - result (with basic stats) or the `exn` encountered - | Result of stream: string * outcome: Choice<'R,exn> + | Result of stream: string * outcome: Choice<'R,'E> type BufferState = Idle | Busy | Full | Slipstreaming /// Gathers stats pertaining to the core projection/ingestion activity - type Stats<'R>(log : ILogger, statsInterval : TimeSpan, stateInterval : TimeSpan) = + type Stats<'R,'E>(log : ILogger, statsInterval : TimeSpan, stateInterval : TimeSpan) = let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 let merges, mergedStreams, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) let mutable dt,ft,it,st,mt = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero let dumpStats (used,maxDop) (waitingBatches,pendingMerges) = - log.Information("Timing Merge {mt:n1}s Ingest {it:n1}s Fill {ft:n1}s Drain {dt:n1}s Stats {st:n1}s", - mt.TotalSeconds,it.TotalSeconds,ft.TotalSeconds,dt.TotalSeconds,st.TotalSeconds) - dt <- TimeSpan.Zero; ft <- TimeSpan.Zero; it <- TimeSpan.Zero; st <- TimeSpan.Zero; mt <- TimeSpan.Zero log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 log.Information("Batches Waiting {batchesWaiting} Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}/{pendingMerges} {mergedStreams}s", waitingBatches, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingMerges, !mergedStreams) batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 - abstract member Handle : InternalMessage<'R> -> unit + log.Information("Scheduling Merge {mt:n1}s Ingest {it:n1}s Fill {ft:n1}s Drain {dt:n1}s Stats {st:n1}s", + mt.TotalSeconds, it.TotalSeconds, ft.TotalSeconds, dt.TotalSeconds, st.TotalSeconds) + dt <- TimeSpan.Zero; ft <- TimeSpan.Zero; it <- TimeSpan.Zero; st <- TimeSpan.Zero; mt <- TimeSpan.Zero + abstract member Handle : InternalMessage<'R,'E> -> unit default __.Handle msg = msg |> function | Merge buffer -> mergedStreams := !mergedStreams + buffer.StreamCount @@ -368,11 +367,11 @@ module Scheduling = /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) /// c) submits work to the supplied Dispatcher (which it triggers pumping of) /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = + type Engine<'R,'E>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = let sleepIntervalMs = 1 let maxBatches = defaultArg maxBatches 16 let cts = new CancellationTokenSource() - let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better + let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering let streams = StreamStates() let progressState = Progress.State() @@ -451,7 +450,7 @@ module Scheduling = progressState.AppendBatch(markCompleted,reqs) feedStats <| Added (reqs.Count,skipCount,count) - member private __.Pump(stats : Stats<'R>) = async { + member private __.Pump(stats : Stats<'R,'E>) = async { use _ = dispatcher.Result.Subscribe(Result >> work.Push) Async.Start(dispatcher.Pump(), cts.Token) Async.Start(continuouslyCompactStreamMerges (), cts.Token) @@ -503,7 +502,7 @@ module Scheduling = static member Start<'R>(stats, projectorDop, project, interpretProgress, dumpStreams) = let dispatcher = Dispatcher(projectorDop) - let instance = new Engine<'R>(dispatcher, project, interpretProgress, dumpStreams) + let instance = new Engine<'R,'E>(dispatcher, project, interpretProgress, dumpStreams) Async.Start <| instance.Pump(stats) instance @@ -532,7 +531,7 @@ type Projector = let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.), defaultArg statesInterval (TimeSpan.FromMinutes 5.)) //let category (streamName : string) = streamName.Split([|'-';'_'|],2).[0] let dumpStreams (streams: Scheduling.StreamStates) log = streams.Dump(log, categorize) - Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress, dumpStreams) + Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress, dumpStreams) module Ingestion = @@ -609,7 +608,7 @@ module Ingestion = | false, _ -> None /// Holds batches away from Core processing to limit in-flight processing - type Engine<'R>(log : ILogger, scheduler: Scheduling.Engine<'R>, maxRead, maxSubmissions, initialSeriesIndex, categorize, statsInterval : TimeSpan, ?pumpDelayMs) = + type Engine<'R,'E>(log : ILogger, scheduler: Scheduling.Engine<'R,'E>, maxRead, maxSubmissions, initialSeriesIndex, categorize, statsInterval : TimeSpan, ?pumpDelayMs) = let cts = new CancellationTokenSource() let pumpDelayMs = defaultArg pumpDelayMs 5 let work = ConcurrentQueue() // Queue as need ordering semantically @@ -700,7 +699,7 @@ module Ingestion = /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval) = - let instance = new Engine<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval = statsInterval) + let instance = new Engine<'R,'E>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval = statsInterval) Async.Start <| instance.Pump() instance @@ -724,7 +723,7 @@ type Ingester = /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing static member Start<'R>(log, scheduler, maxRead, maxSubmissions, categorize, ?statsInterval) = let singleSeriesIndex = 0 - let instance = Ingestion.Engine<'R>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + let instance = Ingestion.Engine<'R,_>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) { new IIngester with member __.Submit(epoch, markCompleted, items) : Async = instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) From 03303962d0b5a88e1e379e317296a3d2dc6baecd Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 15:53:58 +0100 Subject: [PATCH 326/353] Reset exception counters --- equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/EventStoreSource.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index b22424190..ffc323b76 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -86,7 +86,7 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = let fails = !rateLimited + !timedOut + !tooLarge + !malformed + !resultExnOther log.Warning("Exceptions {mb:n0}MB {fails:n0}r {streams:n0}s {events:n0}e Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s", mb exnBytes, fails, failStreams.Count, exnEvents, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) - rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear() + rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear(); exnBytes := 0L; exnEvents := 0 if badCats.Any then log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0}r {@tlStreams} Malformed {malformed:n0}r {@mfStreams} Other {other:n0}r {@oStreams}", badCats.StatsDescending, !tooLarge, tlStreams, !malformed, mfStreams, !resultExnOther, oStreams) diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index 2071afeb3..dac1ae20f 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -384,7 +384,7 @@ let run (log : Serilog.ILogger) (connect, spec, categorize, tryMapEvent) maxRead return startPos } let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) let initialSeriesId, conns, dop = - log.Information("Tailing every every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) + log.Information("Tailing every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with | Some factor -> log.Information("Commencing Gorging with {stripes} $all reader stripes covering a 256MB chunk each", factor) From 46ff26c844cfe4d60daaf944cbcbacc76d08de00 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 14 May 2019 15:58:58 +0100 Subject: [PATCH 327/353] Rebadge timings --- equinox-sync/Sync/CosmosIngester.fs | 2 +- equinox-sync/Sync/Projection2.fs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosIngester.fs index ffc323b76..9373c9a1d 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosIngester.fs @@ -86,7 +86,7 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = let fails = !rateLimited + !timedOut + !tooLarge + !malformed + !resultExnOther log.Warning("Exceptions {mb:n0}MB {fails:n0}r {streams:n0}s {events:n0}e Rate-limited {rateLimited:n0}r {rlStreams:n0}s Timed out {toCount:n0}r {toStreams:n0}s", mb exnBytes, fails, failStreams.Count, exnEvents, !rateLimited, rlStreams.Count, !timedOut, toStreams.Count) - rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear(); exnBytes := 0L; exnEvents := 0 + rateLimited := 0; timedOut := 0; resultExnOther := 0; failStreams.Clear(); rlStreams.Clear(); toStreams.Clear(); exnBytes <- 0L; exnEvents <- 0 if badCats.Any then log.Warning("Malformed cats {@badCats} Too large {tooLarge:n0}r {@tlStreams} Malformed {malformed:n0}r {@mfStreams} Other {other:n0}r {@oStreams}", badCats.StatsDescending, !tooLarge, tlStreams, !malformed, mfStreams, !resultExnOther, oStreams) diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 32eb3b3a8..3ae91d714 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -304,7 +304,7 @@ module Scheduling = log.Information("Batches Waiting {batchesWaiting} Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}/{pendingMerges} {mergedStreams}s", waitingBatches, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingMerges, !mergedStreams) batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 - log.Information("Scheduling Merge {mt:n1}s Ingest {it:n1}s Fill {ft:n1}s Drain {dt:n1}s Stats {st:n1}s", + log.Information("Scheduling Streams {mt:n1}s Batches {it:n1}s Dispatch {ft:n1}s Results {dt:n1}s Stats {st:n1}s", mt.TotalSeconds, it.TotalSeconds, ft.TotalSeconds, dt.TotalSeconds, st.TotalSeconds) dt <- TimeSpan.Zero; ft <- TimeSpan.Zero; it <- TimeSpan.Zero; st <- TimeSpan.Zero; mt <- TimeSpan.Zero abstract member Handle : InternalMessage<'R,'E> -> unit From fcb9f3d6f2f997483c08517f2bcac379dcf546eb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 15 May 2019 21:44:19 +0100 Subject: [PATCH 328/353] Kafka projector rewrite --- equinox-projector/Projector/KafkaSink.fs | 179 ++++++++++++++++++ equinox-projector/Projector/Program.fs | 104 ++++------ equinox-projector/Projector/Projector.fsproj | 7 +- .../Sync/{CosmosIngester.fs => CosmosSink.fs} | 67 +++---- equinox-sync/Sync/CosmosSource.fs | 26 +-- equinox-sync/Sync/EventStoreSource.fs | 9 +- equinox-sync/Sync/Infrastructure.fs | 2 +- equinox-sync/Sync/Program.fs | 4 +- equinox-sync/Sync/Projection2.fs | 85 +++++---- equinox-sync/Sync/ProjectorSink.fs | 29 +++ equinox-sync/Sync/Sync.fsproj | 3 +- 11 files changed, 352 insertions(+), 163 deletions(-) create mode 100644 equinox-projector/Projector/KafkaSink.fs rename equinox-sync/Sync/{CosmosIngester.fs => CosmosSink.fs} (72%) create mode 100644 equinox-sync/Sync/ProjectorSink.fs diff --git a/equinox-projector/Projector/KafkaSink.fs b/equinox-projector/Projector/KafkaSink.fs new file mode 100644 index 000000000..cb30207a8 --- /dev/null +++ b/equinox-projector/Projector/KafkaSink.fs @@ -0,0 +1,179 @@ +module ProjectorTemplate.KafkaSink + +open Confluent.Kafka +open Equinox.Projection2 +open Equinox.Projection.Codec +open Equinox.Store +open Jet.ConfluentKafka.FSharp +open Newtonsoft.Json +open System + +open Serilog +open System.Threading.Tasks +open System.Threading +type KafkaProducer private (log: ILogger, inner : IProducer, topic : string) = + member __.Topic = topic + + interface IDisposable with member __.Dispose() = inner.Dispose() + + /// Produces a single item, yielding a response upon completion/failure of the ack + /// + /// There's no assurance of ordering [without dropping `maxInFlight` down to `1` and annihilating throughput]. + /// Thus its critical to ensure you don't submit another message for the same key until you've had a success / failure response from the call. + member __.ProduceAsync(key, value) : Async>= async { + return! inner.ProduceAsync(topic, Message<_,_>(Key=key, Value=value)) |> Async.AwaitTaskCorrect } + + /// Produces a batch of supplied key/value messages. Results are returned in order of writing (which may vary from order of submission). + /// + /// 1. if there is an immediate local config issue + /// 2. upon receipt of the first failed `DeliveryReport` (NB without waiting for any further reports) + /// + /// Note that the delivery and/or write order may vary from the supplied order (unless you drop `maxInFlight` down to 1, massively constraining throughput). + /// Thus it's important to note that supplying >1 item into the queue bearing the same key risks them being written to the topic out of order. + member __.ProduceBatch(keyValueBatch : (string * string)[]) = async { + if Array.isEmpty keyValueBatch then return [||] else + + let! ct = Async.CancellationToken + + let tcs = new TaskCompletionSource[]>() + let numMessages = keyValueBatch.Length + let results = Array.zeroCreate> numMessages + let numCompleted = ref 0 + + use _ = ct.Register(fun _ -> tcs.TrySetCanceled() |> ignore) + + let handler (m : DeliveryReport) = + if m.Error.IsError then + let errorMsg = exn (sprintf "Error on message topic=%s code=%O reason=%s" m.Topic m.Error.Code m.Error.Reason) + tcs.TrySetException errorMsg |> ignore + else + let i = Interlocked.Increment numCompleted + results.[i - 1] <- m + if i = numMessages then tcs.TrySetResult results |> ignore + for key,value in keyValueBatch do + inner.Produce(topic, Message<_,_>(Key=key, Value=value), deliveryHandler = handler) + inner.Flush(ct) + log.Debug("Produced {count}",!numCompleted) + return! Async.AwaitTaskCorrect tcs.Task } + + static member Create(log : ILogger, config : KafkaProducerConfig, topic : string) = + if String.IsNullOrEmpty topic then nullArg "topic" + log.Information("Producing... {broker} / {topic} compression={compression} acks={acks}", config.Broker, topic, config.Compression, config.Acks) + let producer = + ProducerBuilder(config.Kvps) + .SetLogHandler(fun _p m -> log.Information("{message} level={level} name={name} facility={facility}", m.Message, m.Level, m.Name, m.Facility)) + .SetErrorHandler(fun _p e -> log.Error("{reason} code={code} isBrokerError={isBrokerError}", e.Reason, e.Code, e.IsBrokerError)) + .Build() + new KafkaProducer(log, producer, topic) + +module Codec = + /// Rendition of an event when being projected as Spans to Kafka + type [] RenderedSpanEvent = + { /// Event Type associated with event data in `d` + c: string + + /// Timestamp of original write + t: DateTimeOffset // ISO 8601 + + /// Event body, as UTF-8 encoded json ready to be injected directly into the Json being rendered + [)>] + d: byte[] // required + + /// Optional metadata, as UTF-8 encoded json, ready to emit directly (entire field is not written if value is null) + [)>] + [] + m: byte[] } + + /// A 'normal' (frozen, not Tip) Batch of Events (without any Unfolds) + type [] + RenderedSpan = + { /// base 'i' value for the Events held herein + i: int64 + + /// The Events comprising this span + e: RenderedSpanEvent[] } + + module RenderedSpan = + let ofStreamSpan (x: Buffer.StreamSpan) : RenderedSpan = + { i = x.span.index; e = [| for x in x.span.events -> { c = x.EventType; t = x.Timestamp; d = x.Data; m = x.Meta }|] } + +open Equinox.Cosmos.Core +open Equinox.Cosmos.Store +open Equinox.Projection +open Equinox.Projection2 +open Equinox.Projection2.Buffer +open Equinox.Projection2.Scheduling +open Serilog +open System.Threading +open System.Collections.Generic + +[] +module private Impl = + let inline mb x = float x / 1024. / 1024. + type OkResult = int64 * (int*int) * DeliveryResult + type FailResult = (int*int) * exn + +type Stats(log : ILogger, categorize, statsInterval, statesInterval) = + inherit Stats(log, statsInterval, statesInterval) + let okStreams, failStreams, badCats, resultOk, resultExnOther = HashSet(), HashSet(), CatStats(), ref 0, ref 0 + let mutable okEvents, okBytes, exnEvents, exnBytes = 0, 0L, 0, 0L + + override __.DumpExtraStats() = + log.Information("Completed {mb:n0}MB {completed:n0}r {streams:n0}s {events:n0}e ({ok:n0} ok)", mb okBytes, !resultOk, okStreams.Count, okEvents, !resultOk) + okStreams.Clear(); failStreams.Clear(); resultOk := 0; resultExnOther := 0; okEvents <- 0; okBytes <- 0L; + if !resultExnOther <> 0 then + log.Warning("Exceptions {mb:n0}MB {fails:n0}r {streams:n0}s {events:n0}e ", mb exnBytes, !resultExnOther, failStreams.Count, exnEvents) + resultExnOther := 0; failStreams.Clear(); exnBytes <- 0L; exnEvents <- 0 + log.Warning("Malformed cats {@badCats}", badCats.StatsDescending) + badCats.Clear() + + override __.Handle message = + let inline adds x (set:HashSet<_>) = set.Add x |> ignore + let inline bads x (set:HashSet<_>) = badCats.Ingest(categorize x); adds x set + base.Handle message + match message with + | Merge _ | Added _ -> () // Processed by standard logging already; we have nothing to add + | Result (stream, Choice1Of2 (_,(es,bs),_r)) -> + adds stream okStreams + okEvents <- okEvents + es + okBytes <- okBytes + int64 bs + incr resultOk + | Result (stream, Choice2Of2 ((es,bs),exn)) -> + bads stream failStreams + exnEvents <- exnEvents + es + exnBytes <- exnBytes + int64 bs + incr resultExnOther + +type Scheduler = + static member Start(log : Serilog.ILogger, clientId, broker, topic, maxInFlightMessages, categorize, (statsInterval, statesInterval)) + : Scheduling.Engine = + let producerConfig = KafkaProducerConfig.Create(clientId, broker, Acks.Leader, compression = CompressionType.Lz4, maxInFlight=1_000_000) + let producer = KafkaProducer.Create(Log.Logger, producerConfig, topic) + let attemptWrite (_writePos,fullBuffer) = async { + let maxEvents, maxBytes = 16384, 1_000_000 - (*fudge*)4096 + let ((eventCount,_) as stats), span' = Span.slice (maxEvents,maxBytes) fullBuffer.span + let trimmed = { fullBuffer with span = span' } + let rendered = Codec.RenderedSpan.ofStreamSpan trimmed + try let! res = producer.ProduceAsync(trimmed.stream,JsonConvert.SerializeObject(rendered)) + return Choice1Of2 (trimmed.span.index + int64 eventCount,stats,res) + with e -> return Choice2Of2 (stats,e) } + let interpretWriteResultProgress _streams _stream = function + | Choice1Of2 (i',_, _) -> Some i' + | Choice2Of2 (_,_) -> None + let projectionAndKafkaStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) + Engine<_,_>.Start(projectionAndKafkaStats, maxInFlightMessages, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) + +//type Ingester = + +// static member Start(log, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize, ?statsInterval) : IIngester = +// let project (batch : Buffer.StreamSpan) = async { +// let r = Random() +// let ms = r.Next(1,batch.span.events.Length * 10) +// do! Async.Sleep ms +// return batch.span.events.Length } +// //let createIngester rangeLog = SyncTemplate.ProjectorSink.Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) +// //let mapContent : Microsoft.Azure.Documents.Document seq -> StreamItem seq = Seq.collect DocumentParser.enumEvents >> Seq.map TODOremove >> Seq.map RenderedEvent.ofStreamItem +// //let disposeProducer = (producer :> IDisposable).Dispose +// //let es = [| for e in events -> e.s, JsonConvert.SerializeObject e |] + +// SyncTemplate.ProjectorSink.Ingester.Start(log, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize, ?statsInterval=statsInterval) \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 0cb746d6f..b0209072d 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -1,22 +1,12 @@ module ProjectorTemplate.Projector.Program -//#if kafka -open Confluent.Kafka -//#endif open Equinox.Cosmos +open Equinox.Store // Stopwatch.Time open Equinox.Cosmos.Projection -//#if kafka -open Equinox.Projection.Codec -open Equinox.Store -open Jet.ConfluentKafka.FSharp -//#else open Equinox.Projection -//#endif +open Equinox.Projection2 open Microsoft.Azure.Documents.ChangeFeedProcessor open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing -//#if kafka -open Newtonsoft.Json -//#endif open Serilog open System open System.Collections.Generic @@ -153,59 +143,34 @@ let run (log : ILogger) discovery connectionPolicy source (aux, leaseId, startFromTail, maybeLimitDocumentCount, lagReportFreq : TimeSpan option) createRangeProjector = async { let logLag (interval : TimeSpan) (remainingWork : (int*int64) seq) = async { - log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortByDescending snd) + log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortBy fst) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag let! _feedEventHost = ChangeFeedProcessor.Start ( log, discovery, connectionPolicy, source, aux, leasePrefix = leaseId, startFromTail = startFromTail, createObserver = createRangeProjector, ?maxDocuments = maybeLimitDocumentCount, ?reportLagAndAwaitNextEstimation = maybeLogLag) - do! Async.AwaitKeyboardInterrupt() } + do! Async.AwaitKeyboardInterrupt() } // exiting will Cancel the child tasks, i.e. the _feedEventHost -//#if kafka -let mkRangeProjector (broker, topic) log (_maxPendingBatches,_maxDop,_project) _categorize () = - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let cfg = KafkaProducerConfig.Create("ProjectorTemplate", broker, Acks.Leader, compression = CompressionType.Lz4) - let producer = KafkaProducer.Create(Log.Logger, cfg, topic) - let disposeProducer = (producer :> IDisposable).Dispose +let createRangeHandler (log : ILogger) (createIngester : ILogger -> IIngester) (mapContent : Microsoft.Azure.Documents.Document seq -> 'item seq) () = + let mutable rangeIngester = Unchecked.defaultof<_> + let init rangeLog = async { rangeIngester <- createIngester rangeLog } + let ingest epoch checkpoint docs = + let items : 'item seq = mapContent docs + rangeIngester.Submit(epoch, checkpoint, items) + let dispose () = rangeIngester.Stop() + let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok let ingest (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let pt,events = (fun () -> docs |> Seq.collect DocumentParser.enumEvents |> Seq.map RenderedEvent.ofStreamItem |> Array.ofSeq) |> Stopwatch.Time - let es = [| for e in events -> e.s, JsonConvert.SerializeObject e |] - let! et,() = async { - // TODO handle failure - let! _ = producer.ProduceBatch es - return! ctx.Checkpoint() |> Stopwatch.Time } - log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Parse {events,5} events {p:n3}s Emit {e:n1}s", - ctx.FeedResponse.ResponseContinuation.Trim[|'"'|], docs.Count, int ctx.FeedResponse.RequestCharge, - float sw.ElapsedMilliseconds / 1000., events.Length, (let e = pt.Elapsed in e.TotalSeconds), (let e = et.Elapsed in e.TotalSeconds)) + let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 + // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved + let! pt, (cur,max) = ingest epoch (ctx.Checkpoint()) docs |> Stopwatch.Time + log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Post {pt:n3}s {cur}/{max}", + epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., + (let e = pt.Elapsed in e.TotalSeconds), cur, max) sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor } - ChangeFeedObserver.Create(log, ingest, dispose=disposeProducer) -//#else -let createRangeHandler (log:ILogger) (maxReadAhead, maxSubmissionsPerRange, projectionDop) categorize project = - let scheduler = Projector.Start (log, projectionDop, project, categorize, TimeSpan.FromMinutes 1.) - fun () -> - let mutable rangeIngester = Unchecked.defaultof - let init rangeLog = async { rangeIngester <- Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) } - let ingest epoch checkpoint docs = - let items = Seq.collect DocumentParser.enumEvents docs - rangeIngester.Submit(epoch, checkpoint, items) - let dispose () = rangeIngester.Stop() - let sw = Stopwatch.StartNew() // we'll end up reporting the warmup/connect time on the first batch, but that's ok - let ingest (log : ILogger) (ctx : IChangeFeedObserverContext) (docs : IReadOnlyList) = async { - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let epoch = ctx.FeedResponse.ResponseContinuation.Trim[|'"'|] |> int64 - // Pass along the function that the coordinator will run to checkpoint past this batch when such progress has been achieved - let checkpoint = async { do! ctx.CheckpointAsync() |> Async.AwaitTaskCorrect } - let! pt, (cur,max) = ingest epoch checkpoint docs |> Stopwatch.Time - log.Information("Read {token,8} {count,4} docs {requestCharge,4}RU {l:n1}s Post {pt:n3}s {cur}/{max}", - epoch, docs.Count, int ctx.FeedResponse.RequestCharge, float sw.ElapsedMilliseconds / 1000., - let e = pt.Elapsed in e.TotalSeconds, cur, max) - sw.Restart() // restart the clock as we handoff back to the ChangeFeedProcessor - } - ChangeFeedObserver.Create(log, ingest, assign=init, dispose=dispose) - //#endif + ChangeFeedObserver.Create(log, ingest, assign=init, dispose=dispose) // Illustrates how to emit direct to the Console using Serilog // Other topographies can be achieved by using various adapters and bridges, e.g., SerilogTarget or Serilog.Sinks.NLog @@ -221,25 +186,40 @@ module Logging = c.WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> fun c -> c.CreateLogger() +let TODOremove (e : Equinox.Projection.StreamItem) = + let e2 = + { new Equinox.Codec.IEvent<_> with + member __.Data = e.event.Data + member __.Meta = e.event.Meta + member __.EventType = e.event.EventType + member __.Timestamp = e.event.Timestamp } + { stream = e.stream; index = e.index; event = e2 } + [] let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ChangeFeedVerbose let discovery, connector, source = args.Cosmos.BuildConnectionDetails() - let aux, leaseId, startFromTail, maxDocuments, limits, lagFrequency = args.BuildChangeFeedParams() -//#if kafka - //let targetParams = args.Target.BuildTargetParams() - //let createRangeHandler = mkRangeProjector targetParams -//#endif - let project (batch : Equinox.Projection.Buffer.StreamSpan) = async { + let aux, leaseId, startFromTail, maxDocuments, (maxReadAhead, maxSubmissionsPerRange, projectionDop), lagFrequency = args.BuildChangeFeedParams() + let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] +#if !nokafka + let (broker,topic) = args.Target.BuildTargetParams() + let scheduler = ProjectorTemplate.KafkaSink.Scheduler.Start(Log.Logger, "ProjectorTemplate", broker, topic, projectionDop, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 5.)) + let createIngester rangeLog = SyncTemplate.ProjectorSink.Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) + let mapContent : Microsoft.Azure.Documents.Document seq -> StreamItem seq = Seq.collect DocumentParser.enumEvents >> Seq.map TODOremove +#else + let project (batch : Buffer.StreamSpan) = async { let r = Random() let ms = r.Next(1,batch.span.events.Length * 10) do! Async.Sleep ms return batch.span.events.Length } - let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] + let scheduler = ProjectorSink.Scheduler.Start (Log.Logger, projectionDop, project, categorize, TimeSpan.FromMinutes 1.) + let createIngester rangeLog = ProjectorSink.Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) + let mapContent : Microsoft.Azure.Documents.Document seq -> StreamItem seq = Seq.collect DocumentParser.enumEvents >> Seq.map TODOremove +#endif run Log.Logger discovery connector.ConnectionPolicy source (aux, leaseId, startFromTail, maxDocuments, lagFrequency) - (createRangeHandler Log.Logger limits categorize project) + (createRangeHandler Log.Logger createIngester mapContent) |> Async.RunSynchronously 0 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 6f56ae39c..64986a9a2 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -8,11 +8,12 @@ + - + @@ -22,4 +23,8 @@ + + + + \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosIngester.fs b/equinox-sync/Sync/CosmosSink.fs similarity index 72% rename from equinox-sync/Sync/CosmosIngester.fs rename to equinox-sync/Sync/CosmosSink.fs index 9373c9a1d..71efe53bd 100644 --- a/equinox-sync/Sync/CosmosIngester.fs +++ b/equinox-sync/Sync/CosmosSink.fs @@ -1,4 +1,4 @@ -module Equinox.Cosmos.Projection.CosmosIngester +module SyncTemplate.CosmosSink open Equinox.Cosmos.Core open Equinox.Cosmos.Store @@ -12,7 +12,6 @@ open System.Collections.Generic [] module private Impl = - let arrayBytes (x:byte[]) = if x = null then 0 else x.Length let inline mb x = float x / 1024. / 1024. [] @@ -119,39 +118,31 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = | ResultKind.Malformed -> bads stream mfStreams; incr malformed | ResultKind.Other -> bads stream oStreams; incr resultExnOther -let start (log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, categorize, (statsInterval, statesInterval)) = - let cosmosPayloadBytes (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 - let writerResultLog = log.ForContext() - let trim (_currentWritePos : int64 option, batch : StreamSpan) = - let mutable countBudget, bytesBudget = 16384, 1024 * 1024 - (*fudge*)4096 - let mutable count = 0 - let withinLimits (y : Equinox.Codec.IEvent) = - count <- count + 1 - countBudget <- countBudget - 1 - bytesBudget <- bytesBudget - cosmosPayloadBytes y - // always send at least one event in order to surface the problem and have the stream marked malformed - count = 1 || (countBudget >= 0 && bytesBudget >= 0) - { stream = batch.stream; span = { index = batch.span.index; events = batch.span.events |> Array.takeWhile withinLimits } } - let mutable robin = 0 - let attemptWrite batch = async { - let trimmed = trim batch - let index = Interlocked.Increment(&robin) % cosmosContexts.Length - let selectedConnection = cosmosContexts.[index] - let stats = trimmed.span.events.Length, trimmed.span.events |> Seq.sumBy cosmosPayloadBytes - try let! res = Writer.write log selectedConnection trimmed - return Choice1Of2 (stats,res) - with e -> return Choice2Of2 (stats,e) } - let interpretWriteResultProgress (streams: Scheduling.StreamStates) stream res = - let applyResultToStreamState = function - | Choice1Of2 (_stats, Writer.Ok pos) -> streams.InternalUpdate stream pos null - | Choice1Of2 (_stats, Writer.Duplicate pos) -> streams.InternalUpdate stream pos null - | Choice1Of2 (_stats, Writer.PartialDuplicate overage) -> streams.InternalUpdate stream overage.index [|overage|] - | Choice1Of2 (_stats, Writer.PrefixMissing (overage,pos)) -> streams.InternalUpdate stream pos [|overage|] - | Choice2Of2 (_stats, exn) -> - let malformed = Writer.classify exn |> Writer.isMalformed - streams.SetMalformed(stream,malformed) - let _stream, { write = wp } = applyResultToStreamState res - Writer.logTo writerResultLog (stream,res) - wp - let projectionAndCosmosStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) - Engine<(int*int)*Writer.Result,_>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file +type Scheduler = + static member Start(log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, categorize, (statsInterval, statesInterval)) + : Scheduling.Engine<(int*int)*Result,(int*int)*exn> = + let writerResultLog = log.ForContext() + let mutable robin = 0 + let attemptWrite (_writePos,batch) = async { + let index = Interlocked.Increment(&robin) % cosmosContexts.Length + let selectedConnection = cosmosContexts.[index] + let maxEvents, maxBytes = 16384, 1024 * 1024 - (*fudge*)4096 + let stats, span' = Span.slice (maxEvents,maxBytes) batch.span + let trimmed = { batch with span = span' } + try let! res = Writer.write log selectedConnection trimmed + return Choice1Of2 (stats,res) + with e -> return Choice2Of2 (stats,e) } + let interpretWriteResultProgress (streams: Scheduling.StreamStates) stream res = + let applyResultToStreamState = function + | Choice1Of2 (_stats, Writer.Ok pos) -> streams.InternalUpdate stream pos null + | Choice1Of2 (_stats, Writer.Duplicate pos) -> streams.InternalUpdate stream pos null + | Choice1Of2 (_stats, Writer.PartialDuplicate overage) -> streams.InternalUpdate stream overage.index [|overage|] + | Choice1Of2 (_stats, Writer.PrefixMissing (overage,pos)) -> streams.InternalUpdate stream pos [|overage|] + | Choice2Of2 (_stats, exn) -> + let malformed = Writer.classify exn |> Writer.isMalformed + streams.SetMalformed(stream,malformed) + let _stream, { write = wp } = applyResultToStreamState res + Writer.logTo writerResultLog (stream,res) + wp + let projectionAndCosmosStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) + Engine<(int*int)*Writer.Result,_>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index c9a2422b6..59c6790a6 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -10,9 +10,9 @@ open Serilog open System open System.Collections.Generic -let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq) (maxReads, maxSubmissions) categorize cosmosIngester () = +let createRangeSyncHandler (log:ILogger) (transform : Document -> StreamItem seq) (maxReads, maxSubmissions) categorize scheduler () = let mutable rangeIngester = Unchecked.defaultof<_> - let init rangeLog = async { rangeIngester <- Ingester.Start(rangeLog, cosmosIngester, maxReads, maxSubmissions, categorize, TimeSpan.FromMinutes 1.) } + let init rangeLog = async { rangeIngester <- ProjectorSink.Ingester.Start(rangeLog, scheduler, maxReads, maxSubmissions, categorize, TimeSpan.FromMinutes 1.) } let ingest epoch checkpoint docs = let events = docs |> Seq.collect transform in rangeIngester.Submit(epoch, checkpoint, events) let dispose () = rangeIngester.Stop () let sw = System.Diagnostics.Stopwatch() // we'll end up reporting the warmup/connect time on the first batch, but that's ok @@ -37,7 +37,7 @@ let run (log : ILogger) (sourceDiscovery, source) (auxDiscovery, aux) connection log.Information("Backlog {backlog:n0} (by range: {@rangeLags})", remainingWork |> Seq.map snd |> Seq.sum, remainingWork |> Seq.sortBy fst) return! Async.Sleep interval } let maybeLogLag = lagReportFreq |> Option.map logLag - let cosmosIngester = CosmosIngester.start (log, cosmosContext, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) + let cosmosIngester = CosmosSink.Scheduler.Start(log, cosmosContext, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 1.)) let! _feedEventHost = ChangeFeedProcessor.Start ( log, sourceDiscovery, connectionPolicy, source, aux, auxDiscovery = auxDiscovery, leasePrefix = leaseId, startFromTail = startFromTail, @@ -87,7 +87,7 @@ module EventV0Parser = /// We assume all Documents represent Events laid out as above let parse (d : Document) : StreamItem = let (StandardCodecEvent e) as x = d.Cast() - { stream = x.s; index = x.i; event = e } : Equinox.Projection.StreamItem + { stream = x.s; index = x.i; event = e } : StreamItem let transformV0 categorize catFilter (v0SchemaDocument: Document) : StreamItem seq = seq { let parsed = EventV0Parser.parse v0SchemaDocument @@ -98,14 +98,14 @@ let transformOrFilter categorize catFilter (changeFeedDocument: Document) : Stre for e in DocumentParser.enumEvents changeFeedDocument do // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level if catFilter (categorize e.stream) then - //let removeBody e = - //let e2 = - // { new Equinox.Codec.IEvent<_> with - // member __.Data = null - // member __.Meta = null - // member __.EventType = e.event.EventType - // member __.Timestamp = e.event.Timestamp } - //yield { e with event = e2 } - yield e + //yield e + // TODO remove this temporary type bridging + let e2 = + { new Equinox.Codec.IEvent<_> with + member __.Data = e.event.Data + member __.Meta = e.event.Meta + member __.EventType = e.event.EventType + member __.Timestamp = e.event.Timestamp } + yield { stream = e.stream; index = e.index; event = e2 } } //#endif \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index dac1ae20f..ea99194d6 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -1,6 +1,5 @@ module SyncTemplate.EventStoreSource -open Equinox.Cosmos.Projection open Equinox.Store // AwaitTaskCorrect open Equinox.Projection open Equinox.Projection2 @@ -382,7 +381,7 @@ let run (log : Serilog.ILogger) (connect, spec, categorize, tryMapEvent) maxRead startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, checkpointFreq.TotalMinutes) return startPos } - let cosmosIngestionEngine = CosmosIngester.start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) + let cosmosIngester = CosmosSink.Scheduler.Start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) let initialSeriesId, conns, dop = log.Information("Tailing every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) match spec.gorge with @@ -393,12 +392,12 @@ let run (log : Serilog.ILogger) (connect, spec, categorize, tryMapEvent) maxRead chunk startPos |> int, conns, (max (conns.Length) (spec.streamReaders+1)) | None -> 0, [|conn|], spec.streamReaders+1 - let trancheEngine = Ingestion.Engine.Start(log.ForContext("Tranche","EventStore"), cosmosIngestionEngine, maxReadAhead, maxReadAhead, initialSeriesId, categorize, TimeSpan.FromMinutes 1.) + let trancheIngester = Ingestion.Engine.Start(log.ForContext("Tranche","EventStore"), cosmosIngester, maxReadAhead, maxReadAhead, initialSeriesId, categorize, TimeSpan.FromMinutes 1.) let post = function - | Res.EndOfChunk seriesId -> trancheEngine.Submit <| Ingestion.EndOfSeries seriesId + | Res.EndOfChunk seriesId -> trancheIngester.Submit <| Ingestion.EndOfSeries seriesId | Res.Batch (seriesId, pos, xs) -> let cp = pos.CommitPosition - trancheEngine.Submit <| Ingestion.Message.Batch(seriesId, cp, checkpoints.Commit cp, xs) + trancheIngester.Submit <| Ingestion.Message.Batch(seriesId, cp, checkpoints.Commit cp, xs) let reader = Reader(conns, spec.batchSize, spec.minBatchSize, categorize, tryMapEvent, post, spec.tailInterval, dop) do! reader.Start (initialSeriesId,startPos) maxPos do! Async.AwaitKeyboardInterrupt() } \ No newline at end of file diff --git a/equinox-sync/Sync/Infrastructure.fs b/equinox-sync/Sync/Infrastructure.fs index 9387ba6e6..be6e96e9d 100644 --- a/equinox-sync/Sync/Infrastructure.fs +++ b/equinox-sync/Sync/Infrastructure.fs @@ -9,7 +9,7 @@ open System.Threading.Tasks #nowarn "40" // re AwaitKeyboardInterrupt type Async with - static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) + //static member Sleep(t : TimeSpan) : Async = Async.Sleep(int t.TotalMilliseconds) /// Asynchronously awaits the next keyboard interrupt event static member AwaitKeyboardInterrupt () : Async = Async.FromContinuations(fun (sc,_,_) -> diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 2f43f452e..62e3326fd 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -289,7 +289,7 @@ module Logging = c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning - c.MinimumLevel.Override(typeof.FullName, generalLevel) + c.MinimumLevel.Override(typeof.FullName, generalLevel) .MinimumLevel.Override(typeof.FullName, LogEventLevel.Information) |> fun c -> let t = "[{Timestamp:HH:mm:ss} {Level:u3}] {partitionKeyRangeId} {Tranche} {Message:lj} {NewLine}{Exception}" let configure (a : Configuration.LoggerSinkConfiguration) : unit = @@ -298,7 +298,7 @@ module Logging = a.Logger(fun l -> let isEqx = Filters.Matching.FromSource().Invoke let isCp = Filters.Matching.FromSource().Invoke - let isWriter = Filters.Matching.FromSource().Invoke + let isWriter = Filters.Matching.FromSource().Invoke let isCfp429a = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.LeaseManagement.DocumentServiceLeaseUpdater").Invoke let isCfp429b = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.LeaseRenewer").Invoke let isCfp429c = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.PartitionLoadBalancer").Invoke diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 3ae91d714..93295049c 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -8,6 +8,18 @@ open System.Collections.Generic open System.Diagnostics open System.Threading +/// Item from a reader as supplied to the `IIngester` +type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } + +/// Core interface for projection system, representing the complete contract a feed consumer uses to deliver batches of work for projection +type IIngester<'Epoch,'Item> = + /// Passes a (lazy) batch of items into the Ingestion Engine; the batch will then be materialized out of band and submitted to the Scheduler + /// Admission is Async in order that the Projector and Ingester can together contrive to force backpressure on the producer of the batches by delaying conclusion of the Async computation + /// Returns the ephemeral position of this entry in the queue of uncheckpointed batches at time of posting, together with current max number of items permissible + abstract member Submit: progressEpoch: 'Epoch * markCompleted: Async * items: 'Item seq -> Async + /// Requests cancellation of ingestion processing as soon as practicable (typically this is in reaction to a lease being revoked) + abstract member Stop: unit -> unit + [] module private Impl = let (|NNA|) xs = if xs = null then Array.empty else xs @@ -31,8 +43,6 @@ module private Impl = member __.State = max-inner.CurrentCount,max /// Wait infinitely to get the semaphore member __.Await() = inner.Await() |> Async.Ignore - /// Wait for the specified timeout to acquire (or return false instantly) - member __.TryAwait(?timeout) = inner.Await(defaultArg timeout TimeSpan.Zero) /// Dont use without profiling proving it helps as it doesnt help correctness or legibility member __.TryWaitWithoutCancellationForPerf() = inner.Wait(0) /// Only use where you're interested in intenionally busywaiting on a thread - i.e. when you have proven its critical @@ -101,7 +111,7 @@ module Buffer = type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } module Span = let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength - let trim min : Span -> Span = function + let dropBeforeIndex min : Span -> Span = function | x when x.index >= min -> x // don't adjust if min not within | End n when n < min -> { index = min; events = [||] } // throw away if before min #if NET461 @@ -112,7 +122,7 @@ module Buffer = let merge min (xs : Span seq) = let xs = seq { for x in xs -> { x with events = (|NNA|) x.events } } - |> Seq.map (trim min) + |> Seq.map (dropBeforeIndex min) |> Seq.filter (fun x -> x.events.Length <> 0) |> Seq.sortBy (fun x -> x.index) let buffer = ResizeArray() @@ -128,9 +138,26 @@ module Buffer = curr <- Some x // Overlapping, join | Some (End nextIndex as c), x -> - curr <- Some { c with events = Array.append c.events (trim nextIndex x).events } + curr <- Some { c with events = Array.append c.events (dropBeforeIndex nextIndex x).events } curr |> Option.iter buffer.Add if buffer.Count = 0 then null else buffer.ToArray() + let slice (maxEvents,maxBytes) (x: Span) = + let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length + // TODO tests etc + let inline estimateBytesAsJsonUtf8 (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 + let mutable count,bytes = 0, 0 + let mutable countBudget, bytesBudget = maxEvents,maxBytes + let withinLimits (y : Equinox.Codec.IEvent) = + countBudget <- countBudget - 1 + let eventBytes = estimateBytesAsJsonUtf8 y + bytesBudget <- bytesBudget - eventBytes + // always send at least one event in order to surface the problem and have the stream marked malformed + let res = count = 1 || (countBudget >= 0 && bytesBudget >= 0) + if res then count <- count + 1; bytes <- bytes + eventBytes + res + let trimmed = { x with events = x.events |> Array.takeWhile withinLimits } + let stats = trimmed.events.Length, trimmed.events |> Seq.sumBy estimateBytesAsJsonUtf8 + stats, trimmed type [] StreamSpan = { stream: string; span: Span } type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with member __.Size = @@ -204,12 +231,12 @@ module Scheduling = stream, updated let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore - let mergeBuffered (buffer : Buffer.Streams) = + let mergeBuffered (buffer : Streams) = for x in buffer.Items do update x.Key x.Value |> ignore let busy = HashSet() - let pending trySlipstreamed (requestedOrder : string seq) = seq { + let pending trySlipstreamed (requestedOrder : string seq) : seq = seq { let proposed = HashSet() for s in requestedOrder do let state = states.[s] @@ -357,11 +384,15 @@ module Scheduling = member __.HasCapacity = dop.HasCapacity member __.State = dop.State member __.TryAdd(item) = - if dop.TryWaitWithoutCancellationForPerf() then work.Add(item); true else false + if dop.TryWaitWithoutCancellationForPerf() then + work.Add(item) + true + else false member __.Pump () = async { let! ct = Async.CancellationToken for item in work.GetConsumingEnumerable ct do Async.Start(dispatch item) } + /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order /// a) does not itself perform any reading activities /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) @@ -428,7 +459,7 @@ module Scheduling = let (_,{stream = s} : StreamSpan) as item = xs.Current let succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) if succeeded then streams.MarkBusy s - dispatched <- dispatched || succeeded // if we added any request, we also don't sleep + dispatched <- dispatched || succeeded // if we added any request, we'll skip sleeping hasCapacity <- succeeded hasCapacity, dispatched // Take an incoming batch of events, correlating it against our known stream state to yield a set of remaining work @@ -518,21 +549,6 @@ module Scheduling = member __.Stop() = cts.Cancel() -type Projector = - - static member Start(log, projectorDop, project : Buffer.StreamSpan -> Async, categorize, ?statsInterval, ?statesInterval) = - let project (_maybeWritePos, batch) = async { - try let! count = project batch - return Choice1Of2 (batch.span.index + int64 count) - with e -> return Choice2Of2 e } - let interpretProgress _streams _stream = function - | Choice1Of2 index -> Some index - | Choice2Of2 _ -> None - let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.), defaultArg statesInterval (TimeSpan.FromMinutes 5.)) - //let category (streamName : string) = streamName.Split([|'-';'_'|],2).[0] - let dumpStreams (streams: Scheduling.StreamStates) log = streams.Dump(log, categorize) - Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress, dumpStreams) - module Ingestion = [] @@ -600,7 +616,7 @@ module Ingestion = | CloseSeries of seriesIndex: int | ActivateSeries of seriesIndex: int - let tryRemove key (dict: Dictionary<_,_>) = + let tryTake key (dict: Dictionary<_,_>) = match dict.TryGetValue key with | true, value -> dict.Remove key |> ignore @@ -644,7 +660,7 @@ module Ingestion = log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) work.Enqueue <| ActivateSeries (activeSeries + 1) else - match readingAhead |> tryRemove seriesIndex with + match readingAhead |> tryTake seriesIndex with | Some batchesRead -> ready.[seriesIndex] <- batchesRead log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) @@ -654,13 +670,13 @@ module Ingestion = | ActivateSeries newActiveSeries -> activeSeries <- newActiveSeries let buffered = - match ready |> tryRemove newActiveSeries with + match ready |> tryTake newActiveSeries with | Some completedChunkBatches -> completedChunkBatches |> Seq.iter pending.Enqueue work.Enqueue <| ActivateSeries (newActiveSeries + 1) completedChunkBatches.Count | None -> - match readingAhead |> tryRemove newActiveSeries with + match readingAhead |> tryTake newActiveSeries with | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count | None -> 0 log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", @@ -716,15 +732,4 @@ module Ingestion = return readMax.State } /// As range assignments get revoked, a user is expected to `Stop `the active processing thread for the Ingester before releasing references to it - member __.Stop() = cts.Cancel() - -type Ingester = - - /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing - static member Start<'R>(log, scheduler, maxRead, maxSubmissions, categorize, ?statsInterval) = - let singleSeriesIndex = 0 - let instance = Ingestion.Engine<'R,_>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) - { new IIngester with - member __.Submit(epoch, markCompleted, items) : Async = - instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) - member __.Stop() = __.Stop() } \ No newline at end of file + member __.Stop() = cts.Cancel() \ No newline at end of file diff --git a/equinox-sync/Sync/ProjectorSink.fs b/equinox-sync/Sync/ProjectorSink.fs new file mode 100644 index 000000000..ff3b9e505 --- /dev/null +++ b/equinox-sync/Sync/ProjectorSink.fs @@ -0,0 +1,29 @@ +module SyncTemplate.ProjectorSink + +open Equinox.Projection2 +open System + +type Scheduler = + + static member Start(log, projectorDop, project : Buffer.StreamSpan -> Async, categorize, ?statsInterval, ?statesInterval) = + let project (_maybeWritePos, batch) = async { + try let! count = project batch + return Choice1Of2 (batch.span.index + int64 count) + with e -> return Choice2Of2 e } + let interpretProgress _streams _stream = function + | Choice1Of2 index -> Some index + | Choice2Of2 _ -> None + let stats = Scheduling.Stats(log, defaultArg statsInterval (TimeSpan.FromMinutes 1.), defaultArg statesInterval (TimeSpan.FromMinutes 5.)) + let dumpStreams (streams: Scheduling.StreamStates) log = streams.Dump(log, categorize) + Scheduling.Engine.Start(stats, projectorDop, project, interpretProgress, dumpStreams) + +type Ingester = + + /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing + static member Start<'R,'E>(log, scheduler, maxRead, maxSubmissions, categorize, ?statsInterval) : IIngester = + let singleSeriesIndex = 0 + let instance = Ingestion.Engine<'R,'E>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + { new IIngester with + member __.Submit(epoch, markCompleted, items) : Async = + instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) + member __.Stop() = __.Stop() } \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 26875e752..a3e29786e 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -11,7 +11,8 @@ - + + From 36da6349561238d0888d3d68f8225cbedc1c6b8a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 16 May 2019 02:13:18 +0100 Subject: [PATCH 329/353] Kafka fixes --- equinox-projector/Projector/KafkaSink.fs | 22 ++++------------------ equinox-projector/Projector/Program.fs | 23 ++++++++++++----------- equinox-sync/Sync/Projection2.fs | 4 ++-- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/equinox-projector/Projector/KafkaSink.fs b/equinox-projector/Projector/KafkaSink.fs index cb30207a8..0520e1729 100644 --- a/equinox-projector/Projector/KafkaSink.fs +++ b/equinox-projector/Projector/KafkaSink.fs @@ -120,9 +120,9 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = override __.DumpExtraStats() = log.Information("Completed {mb:n0}MB {completed:n0}r {streams:n0}s {events:n0}e ({ok:n0} ok)", mb okBytes, !resultOk, okStreams.Count, okEvents, !resultOk) - okStreams.Clear(); failStreams.Clear(); resultOk := 0; resultExnOther := 0; okEvents <- 0; okBytes <- 0L; + okStreams.Clear(); resultOk := 0; okEvents <- 0; okBytes <- 0L if !resultExnOther <> 0 then - log.Warning("Exceptions {mb:n0}MB {fails:n0}r {streams:n0}s {events:n0}e ", mb exnBytes, !resultExnOther, failStreams.Count, exnEvents) + log.Warning("Exceptions {mb:n0}MB {fails:n0}r {streams:n0}s {events:n0}e", mb exnBytes, !resultExnOther, failStreams.Count, exnEvents) resultExnOther := 0; failStreams.Clear(); exnBytes <- 0L; exnEvents <- 0 log.Warning("Malformed cats {@badCats}", badCats.StatsDescending) badCats.Clear() @@ -143,6 +143,7 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = exnEvents <- exnEvents + es exnBytes <- exnBytes + int64 bs incr resultExnOther + log.Warning(exn,"Could not write {b:n0} bytes {e:n0}e in stream {stream}", bs, es, stream) type Scheduler = static member Start(log : Serilog.ILogger, clientId, broker, topic, maxInFlightMessages, categorize, (statsInterval, statesInterval)) @@ -161,19 +162,4 @@ type Scheduler = | Choice1Of2 (i',_, _) -> Some i' | Choice2Of2 (_,_) -> None let projectionAndKafkaStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) - Engine<_,_>.Start(projectionAndKafkaStats, maxInFlightMessages, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) - -//type Ingester = - -// static member Start(log, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize, ?statsInterval) : IIngester = -// let project (batch : Buffer.StreamSpan) = async { -// let r = Random() -// let ms = r.Next(1,batch.span.events.Length * 10) -// do! Async.Sleep ms -// return batch.span.events.Length } -// //let createIngester rangeLog = SyncTemplate.ProjectorSink.Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) -// //let mapContent : Microsoft.Azure.Documents.Document seq -> StreamItem seq = Seq.collect DocumentParser.enumEvents >> Seq.map TODOremove >> Seq.map RenderedEvent.ofStreamItem -// //let disposeProducer = (producer :> IDisposable).Dispose -// //let es = [| for e in events -> e.s, JsonConvert.SerializeObject e |] - -// SyncTemplate.ProjectorSink.Ingester.Start(log, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize, ?statsInterval=statsInterval) \ No newline at end of file + Engine<_,_>.Start(projectionAndKafkaStats, maxInFlightMessages, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index b0209072d..ee2facc77 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -86,9 +86,9 @@ module CmdParser = | ConsumerGroupName _ -> "Projector consumer group name." | LeaseCollectionSuffix _ -> "specify Collection Name suffix for Leases collection (default: `-aux`)." | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." - | MaxDocuments _ -> "maxiumum document count to supply for the Change Feed query. Default: use response size limit" - | MaxReadAhead _ -> "Maximum number of batches to let processing get ahead of completion. Default: 64" - | MaxWriters _ -> "Maximum number of concurrent streams on which to process at any time. Default: 1024" + | MaxDocuments _ -> "maximum document count to supply for the Change Feed query. Default: use response size limit" + | MaxReadAhead _ -> "maximum number of batches to let processing get ahead of completion. Default: 64" + | MaxWriters _ -> "maximum number of concurrent streams on which to process at any time. Default: 1024" | LagFreqS _ -> "specify frequency to dump lag stats. Default: off" | Verbose -> "request Verbose Logging. Default: off" | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" @@ -112,20 +112,21 @@ module CmdParser = member __.ChangeFeedVerbose = args.Contains ChangeFeedVerbose member __.MaxDocuments = args.TryGetResult MaxDocuments member __.MaxReadAhead = args.GetResult(MaxReadAhead,64) - member __.ProcessorDop = args.GetResult(MaxWriters,64) + member __.ConcurrentStreamProcessors = args.GetResult(MaxWriters,64) member __.LagFrequency = args.TryGetResult LagFreqS |> Option.map TimeSpan.FromSeconds member __.AuxCollectionName = __.Cosmos.Collection + __.Suffix member x.BuildChangeFeedParams() = match x.MaxDocuments with | None -> Log.Information("Processing {leaseId} in {auxCollName} without document count limit (<= {maxPending} pending) using {dop} processors", - x.LeaseId, x.AuxCollectionName, x.MaxReadAhead, x.ProcessorDop) + x.LeaseId, x.AuxCollectionName, x.MaxReadAhead, x.ConcurrentStreamProcessors) | Some lim -> Log.Information("Processing {leaseId} in {auxCollName} with max {changeFeedMaxDocuments} documents (<= {maxPending} pending) using {dop} processors", - x.LeaseId, x.AuxCollectionName, lim, x.MaxReadAhead, x.ProcessorDop) + x.LeaseId, x.AuxCollectionName, lim, x.MaxReadAhead, x.ConcurrentStreamProcessors) if args.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) - { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, args.Contains FromTail, x.MaxDocuments, (x.MaxReadAhead, x.MaxSubmissionPerRange, x.ProcessorDop), x.LagFrequency + { database = x.Cosmos.Database; collection = x.AuxCollectionName}, x.LeaseId, args.Contains FromTail, x.MaxDocuments, x.LagFrequency, + (x.MaxReadAhead, x.MaxSubmissionPerRange, x.ConcurrentStreamProcessors) //#if kafka and TargetInfo(args : ParseResults) = member __.Broker = Uri(match args.TryGetResult Broker with Some x -> x | None -> envBackstop "Broker" "EQUINOX_KAFKA_BROKER") @@ -189,7 +190,7 @@ module Logging = let TODOremove (e : Equinox.Projection.StreamItem) = let e2 = { new Equinox.Codec.IEvent<_> with - member __.Data = e.event.Data + member __.Data = if e.event.Data.Length > 900_000 then null else e.event.Data member __.Meta = e.event.Meta member __.EventType = e.event.EventType member __.Timestamp = e.event.Timestamp } @@ -200,12 +201,12 @@ let main argv = try let args = CmdParser.parse argv Logging.initialize args.Verbose args.ChangeFeedVerbose let discovery, connector, source = args.Cosmos.BuildConnectionDetails() - let aux, leaseId, startFromTail, maxDocuments, (maxReadAhead, maxSubmissionsPerRange, projectionDop), lagFrequency = args.BuildChangeFeedParams() + let aux, leaseId, startFromTail, maxDocuments, lagFrequency, (maxReadAhead, maxSubmissionsPerRange, maxConcurrentStreams) = args.BuildChangeFeedParams() let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] #if !nokafka let (broker,topic) = args.Target.BuildTargetParams() - let scheduler = ProjectorTemplate.KafkaSink.Scheduler.Start(Log.Logger, "ProjectorTemplate", broker, topic, projectionDop, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 5.)) - let createIngester rangeLog = SyncTemplate.ProjectorSink.Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize) + let scheduler = ProjectorTemplate.KafkaSink.Scheduler.Start(Log.Logger, "ProjectorTemplate", broker, topic, maxConcurrentStreams, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 5.)) + let createIngester rangeLog = SyncTemplate.ProjectorSink.Ingester.Start (rangeLog, scheduler, maxReadAhead, maxSubmissionsPerRange, categorize, TimeSpan.FromMinutes 10.) let mapContent : Microsoft.Azure.Documents.Document seq -> StreamItem seq = Seq.collect DocumentParser.enumEvents >> Seq.map TODOremove #else let project (batch : Buffer.StreamSpan) = async { diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 93295049c..0440a29a7 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -152,7 +152,7 @@ module Buffer = let eventBytes = estimateBytesAsJsonUtf8 y bytesBudget <- bytesBudget - eventBytes // always send at least one event in order to surface the problem and have the stream marked malformed - let res = count = 1 || (countBudget >= 0 && bytesBudget >= 0) + let res = count = 0 || (countBudget >= 0 && bytesBudget >= 0) if res then count <- count + 1; bytes <- bytes + eventBytes res let trimmed = { x with events = x.events |> Array.takeWhile withinLimits } @@ -435,7 +435,7 @@ module Scheduling = let mutable worked, more = false, true while more do let c = work.TryPopRange(workLocalBuffer) - if c = 0 then more <- false else worked <- true + if c = 0 (*&& work.IsEmpty*) then more <- false else worked <- true for i in 0..c-1 do let x = workLocalBuffer.[i] match x with From 81fc65e757404f3fa009bb0bdc123d1521ed5292 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 16 May 2019 21:41:24 +0100 Subject: [PATCH 330/353] WIP --- equinox-projector/Projector/KafkaSink.fs | 58 -------------------- equinox-projector/Projector/Projector.fsproj | 2 +- equinox-sync/Sync/CosmosSink.fs | 6 +- equinox-sync/Sync/Projection2.fs | 51 ++++++++--------- equinox-sync/Sync/ProjectorSink.fs | 2 +- 5 files changed, 31 insertions(+), 88 deletions(-) diff --git a/equinox-projector/Projector/KafkaSink.fs b/equinox-projector/Projector/KafkaSink.fs index 0520e1729..fb86a082e 100644 --- a/equinox-projector/Projector/KafkaSink.fs +++ b/equinox-projector/Projector/KafkaSink.fs @@ -8,64 +8,6 @@ open Jet.ConfluentKafka.FSharp open Newtonsoft.Json open System -open Serilog -open System.Threading.Tasks -open System.Threading -type KafkaProducer private (log: ILogger, inner : IProducer, topic : string) = - member __.Topic = topic - - interface IDisposable with member __.Dispose() = inner.Dispose() - - /// Produces a single item, yielding a response upon completion/failure of the ack - /// - /// There's no assurance of ordering [without dropping `maxInFlight` down to `1` and annihilating throughput]. - /// Thus its critical to ensure you don't submit another message for the same key until you've had a success / failure response from the call. - member __.ProduceAsync(key, value) : Async>= async { - return! inner.ProduceAsync(topic, Message<_,_>(Key=key, Value=value)) |> Async.AwaitTaskCorrect } - - /// Produces a batch of supplied key/value messages. Results are returned in order of writing (which may vary from order of submission). - /// - /// 1. if there is an immediate local config issue - /// 2. upon receipt of the first failed `DeliveryReport` (NB without waiting for any further reports) - /// - /// Note that the delivery and/or write order may vary from the supplied order (unless you drop `maxInFlight` down to 1, massively constraining throughput). - /// Thus it's important to note that supplying >1 item into the queue bearing the same key risks them being written to the topic out of order. - member __.ProduceBatch(keyValueBatch : (string * string)[]) = async { - if Array.isEmpty keyValueBatch then return [||] else - - let! ct = Async.CancellationToken - - let tcs = new TaskCompletionSource[]>() - let numMessages = keyValueBatch.Length - let results = Array.zeroCreate> numMessages - let numCompleted = ref 0 - - use _ = ct.Register(fun _ -> tcs.TrySetCanceled() |> ignore) - - let handler (m : DeliveryReport) = - if m.Error.IsError then - let errorMsg = exn (sprintf "Error on message topic=%s code=%O reason=%s" m.Topic m.Error.Code m.Error.Reason) - tcs.TrySetException errorMsg |> ignore - else - let i = Interlocked.Increment numCompleted - results.[i - 1] <- m - if i = numMessages then tcs.TrySetResult results |> ignore - for key,value in keyValueBatch do - inner.Produce(topic, Message<_,_>(Key=key, Value=value), deliveryHandler = handler) - inner.Flush(ct) - log.Debug("Produced {count}",!numCompleted) - return! Async.AwaitTaskCorrect tcs.Task } - - static member Create(log : ILogger, config : KafkaProducerConfig, topic : string) = - if String.IsNullOrEmpty topic then nullArg "topic" - log.Information("Producing... {broker} / {topic} compression={compression} acks={acks}", config.Broker, topic, config.Compression, config.Acks) - let producer = - ProducerBuilder(config.Kvps) - .SetLogHandler(fun _p m -> log.Information("{message} level={level} name={name} facility={facility}", m.Message, m.Level, m.Name, m.Facility)) - .SetErrorHandler(fun _p e -> log.Error("{reason} code={code} isBrokerError={isBrokerError}", e.Reason, e.Code, e.IsBrokerError)) - .Build() - new KafkaProducer(log, producer, topic) - module Codec = /// Rendition of an event when being projected as Spans to Kafka type [] RenderedSpanEvent = diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 64986a9a2..a4e14a9e2 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -18,7 +18,7 @@ - + diff --git a/equinox-sync/Sync/CosmosSink.fs b/equinox-sync/Sync/CosmosSink.fs index 71efe53bd..4a4e23378 100644 --- a/equinox-sync/Sync/CosmosSink.fs +++ b/equinox-sync/Sync/CosmosSink.fs @@ -70,7 +70,7 @@ module Writer = | ResultKind.TooLarge | ResultKind.Malformed -> true type Stats(log : ILogger, categorize, statsInterval, statesInterval) = - inherit Stats<(int*int)*Writer.Result,(int*int)*exn>(log, statsInterval, statesInterval) + inherit Scheduling.Stats<(int*int)*Writer.Result,(int*int)*exn>(log, statsInterval, statesInterval) let okStreams, resultOk, resultDup, resultPartialDup, resultPrefix, resultExnOther = HashSet(), ref 0, ref 0, ref 0, ref 0, ref 0 let badCats, failStreams, rateLimited, timedOut, tooLarge, malformed = CatStats(), HashSet(), ref 0, ref 0, ref 0, ref 0 let rlStreams, toStreams, tlStreams, mfStreams, oStreams = HashSet(), HashSet(), HashSet(), HashSet(), HashSet() @@ -120,7 +120,7 @@ type Stats(log : ILogger, categorize, statsInterval, statesInterval) = type Scheduler = static member Start(log : Serilog.ILogger, cosmosContexts : _ [], maxWriters, categorize, (statsInterval, statesInterval)) - : Scheduling.Engine<(int*int)*Result,(int*int)*exn> = + : ISchedulingEngine = let writerResultLog = log.ForContext() let mutable robin = 0 let attemptWrite (_writePos,batch) = async { @@ -145,4 +145,4 @@ type Scheduler = Writer.logTo writerResultLog (stream,res) wp let projectionAndCosmosStats = Stats(log.ForContext(), categorize, statsInterval, statesInterval) - Engine<(int*int)*Writer.Result,_>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) \ No newline at end of file + Engine<(int*int)*Writer.Result,_>.Start(projectionAndCosmosStats, maxWriters, attemptWrite, interpretWriteResultProgress, fun s l -> s.Dump(l, categorize)) :> _ \ No newline at end of file diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs index 0440a29a7..af7afcbf5 100644 --- a/equinox-sync/Sync/Projection2.fs +++ b/equinox-sync/Sync/Projection2.fs @@ -214,6 +214,14 @@ module Buffer = if waitingCats.Any then log.Information("Waiting Categories, events {@readyCats}", Seq.truncate 5 waitingCats.StatsDescending) if waitingCats.Any then log.Information("Waiting Streams, KB {@readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) +type ISchedulingEngine = + abstract member Stop : unit -> unit + /// Enqueue a batch of items with supplied progress marking function + /// Submission is accepted on trust; they are internally processed in order of submission + /// caller should ensure that (when multiple submitters are in play) no single Range submits more than their fair share + abstract member Submit : markCompleted : (unit -> unit) * items : StreamItem[] -> unit + abstract member SubmitStreamBuffers : Buffer.Streams -> unit + module Scheduling = open Buffer @@ -309,13 +317,13 @@ module Scheduling = /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners [] - type InternalMessage<'R,'E> = + type InternalMessage<'R> = /// Periodic submission of events as they are read, grouped by stream for efficient merging into the StreamState | Merge of Streams /// Stats per submitted batch for stats listeners to aggregate | Added of streams: int * skip: int * events: int /// Result of processing on stream - result (with basic stats) or the `exn` encountered - | Result of stream: string * outcome: Choice<'R,'E> + | Result of stream: string * outcome: 'R type BufferState = Idle | Busy | Full | Slipstreaming /// Gathers stats pertaining to the core projection/ingestion activity @@ -334,7 +342,7 @@ module Scheduling = log.Information("Scheduling Streams {mt:n1}s Batches {it:n1}s Dispatch {ft:n1}s Results {dt:n1}s Stats {st:n1}s", mt.TotalSeconds, it.TotalSeconds, ft.TotalSeconds, dt.TotalSeconds, st.TotalSeconds) dt <- TimeSpan.Zero; ft <- TimeSpan.Zero; it <- TimeSpan.Zero; st <- TimeSpan.Zero; mt <- TimeSpan.Zero - abstract member Handle : InternalMessage<'R,'E> -> unit + abstract member Handle : InternalMessage> -> unit default __.Handle msg = msg |> function | Merge buffer -> mergedStreams := !mergedStreams + buffer.StreamCount @@ -398,11 +406,11 @@ module Scheduling = /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) /// c) submits work to the supplied Dispatcher (which it triggers pumping of) /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R,'E>(dispatcher : Dispatcher<_>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = + type Engine<'R,'E>(dispatcher : Dispatcher<_>, stats : Stats<'R,'E>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = let sleepIntervalMs = 1 let maxBatches = defaultArg maxBatches 16 let cts = new CancellationTokenSource() - let work = ConcurrentStack>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better + let work = ConcurrentStack>>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering let streams = StreamStates() let progressState = Progress.State() @@ -481,7 +489,7 @@ module Scheduling = progressState.AppendBatch(markCompleted,reqs) feedStats <| Added (reqs.Count,skipCount,count) - member private __.Pump(stats : Stats<'R,'E>) = async { + member private __.Pump = async { use _ = dispatcher.Result.Subscribe(Result >> work.Push) Async.Start(dispatcher.Pump(), cts.Token) Async.Start(continuouslyCompactStreamMerges (), cts.Token) @@ -531,23 +539,16 @@ module Scheduling = // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) Thread.Sleep sleepIntervalMs } // Not Async.Sleep so we don't give up the thread - static member Start<'R>(stats, projectorDop, project, interpretProgress, dumpStreams) = + static member Start(stats, projectorDop, project, interpretProgress, dumpStreams) = let dispatcher = Dispatcher(projectorDop) - let instance = new Engine<'R,'E>(dispatcher, project, interpretProgress, dumpStreams) - Async.Start <| instance.Pump(stats) + let instance = new Engine<_,_>(dispatcher, stats, project, interpretProgress, dumpStreams) + Async.Start instance.Pump instance - /// Enqueue a batch of items with supplied progress marking function - /// Submission is accepted on trust; they are internally processed in order of submission - /// caller should ensure that (when multiple submitters are in play) no single Range submits more than their fair share - member __.Submit(markCompleted: (unit -> unit), items: StreamItem[]) = - pending.Enqueue (markCompleted, items) - - member __.SubmitStreamBuffers(events) = - work.Push <| Merge events - - member __.Stop() = - cts.Cancel() + interface ISchedulingEngine with + member __.Stop() = cts.Cancel() + member __.Submit(markCompleted: (unit -> unit), items : StreamItem[]) = pending.Enqueue (markCompleted, items) + member __.SubmitStreamBuffers streams = work.Push <| Merge streams module Ingestion = @@ -624,7 +625,7 @@ module Ingestion = | false, _ -> None /// Holds batches away from Core processing to limit in-flight processing - type Engine<'R,'E>(log : ILogger, scheduler: Scheduling.Engine<'R,'E>, maxRead, maxSubmissions, initialSeriesIndex, categorize, statsInterval : TimeSpan, ?pumpDelayMs) = + type Engine(log : ILogger, scheduler: ISchedulingEngine, maxRead, maxSubmissions, initialSeriesIndex, categorize, statsInterval : TimeSpan, ?pumpDelayMs) = let cts = new CancellationTokenSource() let pumpDelayMs = defaultArg pumpDelayMs 5 let work = ConcurrentQueue() // Queue as need ordering semantically @@ -685,7 +686,7 @@ module Ingestion = | Added _ | ProgressResult _ -> () - member private __.Pump() = async { + member private __.Pump = async { use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) Async.Start(progressWriter.Pump(), cts.Token) let presubmitInterval = expiredMs (4000L*2L) @@ -714,9 +715,9 @@ module Ingestion = with e -> log.Error(e,"Buffer thread exception") } /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes - static member Start<'R>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval) = - let instance = new Engine<'R,'E>(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval = statsInterval) - Async.Start <| instance.Pump() + static member Start(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval) = + let instance = new Engine(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval = statsInterval) + Async.Start instance.Pump instance /// Awaits space in `read` to limit reading ahead - yields (used,maximum) counts from Read Semaphore for logging purposes diff --git a/equinox-sync/Sync/ProjectorSink.fs b/equinox-sync/Sync/ProjectorSink.fs index ff3b9e505..f2dfa7707 100644 --- a/equinox-sync/Sync/ProjectorSink.fs +++ b/equinox-sync/Sync/ProjectorSink.fs @@ -22,7 +22,7 @@ type Ingester = /// Starts an Ingester that will submit up to `maxSubmissions` items at a time to the `scheduler`, blocking on Submits when more than `maxRead` batches have yet to complete processing static member Start<'R,'E>(log, scheduler, maxRead, maxSubmissions, categorize, ?statsInterval) : IIngester = let singleSeriesIndex = 0 - let instance = Ingestion.Engine<'R,'E>.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) + let instance = Ingestion.Engine.Start(log, scheduler, maxRead, maxSubmissions, singleSeriesIndex, categorize, statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 1.)) { new IIngester with member __.Submit(epoch, markCompleted, items) : Async = instance.Submit(Ingestion.Message.Batch(singleSeriesIndex, epoch, markCompleted, items)) From 69a535047d978bea6bf4da99011e3fcbf0620599 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 17 May 2019 04:22:33 +0100 Subject: [PATCH 331/353] Target 2.0.0-preview8 --- .../equinox-projector-consumer.sln | 12 - equinox-sync/Sync/CosmosSink.fs | 5 +- equinox-sync/Sync/CosmosSource.fs | 1 - equinox-sync/Sync/EventStoreSource.fs | 1 - equinox-sync/Sync/Program.fs | 4 +- equinox-sync/Sync/Projection2.fs | 736 ------------------ equinox-sync/Sync/ProjectorSink.fs | 2 +- equinox-sync/Sync/Sync.fsproj | 7 +- 8 files changed, 8 insertions(+), 760 deletions(-) delete mode 100644 equinox-sync/Sync/Projection2.fs diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index fc5d3546d..0a3aa03eb 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -3,15 +3,11 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.28729.10 MinimumVisualStudioVersion = 15.0.26124.0 -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Projector", "Projector\Projector.fsproj", "{6C72C937-ECFC-4DD4-9BA0-7355B237F974}" -EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{518EE7E2-76AF-4DE9-A127-C2DFF709A468}" ProjectSection(SolutionItems) = preProject README.md = README.md EndProjectSection EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consumer.fsproj", "{7ED94D2B-1744-48A0-9B20-94E4777617E9}" -EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Sync", "..\equinox-sync\Sync\Sync.fsproj", "{C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}" EndProject Global @@ -20,14 +16,6 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Debug|Any CPU.Build.0 = Debug|Any CPU - {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Release|Any CPU.ActiveCfg = Release|Any CPU - {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Release|Any CPU.Build.0 = Release|Any CPU - {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.Build.0 = Debug|Any CPU - {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.ActiveCfg = Release|Any CPU - {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.Build.0 = Release|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.ActiveCfg = Debug|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Debug|Any CPU.Build.0 = Debug|Any CPU {C4C70AAA-1978-4886-B3FF-EF14EBCA9DE2}.Release|Any CPU.ActiveCfg = Release|Any CPU diff --git a/equinox-sync/Sync/CosmosSink.fs b/equinox-sync/Sync/CosmosSink.fs index 4a4e23378..dab25f2d6 100644 --- a/equinox-sync/Sync/CosmosSink.fs +++ b/equinox-sync/Sync/CosmosSink.fs @@ -3,9 +3,8 @@ open Equinox.Cosmos.Core open Equinox.Cosmos.Store open Equinox.Projection -open Equinox.Projection2 -open Equinox.Projection2.Buffer -open Equinox.Projection2.Scheduling +open Equinox.Projection.Buffer +open Equinox.Projection.Scheduling open Serilog open System.Threading open System.Collections.Generic diff --git a/equinox-sync/Sync/CosmosSource.fs b/equinox-sync/Sync/CosmosSource.fs index 59c6790a6..1401428f3 100644 --- a/equinox-sync/Sync/CosmosSource.fs +++ b/equinox-sync/Sync/CosmosSource.fs @@ -2,7 +2,6 @@ open Equinox.Cosmos.Projection open Equinox.Projection -open Equinox.Projection2 open Equinox.Store // AwaitTaskCorrect open Microsoft.Azure.Documents open Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs index ea99194d6..cd4927667 100644 --- a/equinox-sync/Sync/EventStoreSource.fs +++ b/equinox-sync/Sync/EventStoreSource.fs @@ -2,7 +2,6 @@ open Equinox.Store // AwaitTaskCorrect open Equinox.Projection -open Equinox.Projection2 open EventStore.ClientAPI open Serilog // NB Needs to shadow ILogger open System diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 62e3326fd..edb82416c 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -286,7 +286,7 @@ module Logging = let cfpLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Warning c.MinimumLevel.Override("Microsoft.Azure.Documents.ChangeFeedProcessor", cfpLevel) |> fun c -> let ingesterLevel = if verboseConsole then LogEventLevel.Debug else LogEventLevel.Information - c.MinimumLevel.Override(typeof.FullName, ingesterLevel) + c.MinimumLevel.Override(typeof.FullName, ingesterLevel) |> fun c -> if verbose then c.MinimumLevel.Debug() else c |> fun c -> let generalLevel = if verbose then LogEventLevel.Information else LogEventLevel.Warning c.MinimumLevel.Override(typeof.FullName, generalLevel) @@ -308,7 +308,7 @@ module Logging = c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) |> fun c -> match maybeSeqEndpoint with None -> c | Some endpoint -> c.WriteTo.Seq(endpoint) |> fun c -> c.CreateLogger() - Log.ForContext(), Log.ForContext() + Log.ForContext(), Log.ForContext() [] let main argv = diff --git a/equinox-sync/Sync/Projection2.fs b/equinox-sync/Sync/Projection2.fs deleted file mode 100644 index af7afcbf5..000000000 --- a/equinox-sync/Sync/Projection2.fs +++ /dev/null @@ -1,736 +0,0 @@ -namespace Equinox.Projection2 - -open Equinox.Projection -open Serilog -open System -open System.Collections.Concurrent -open System.Collections.Generic -open System.Diagnostics -open System.Threading - -/// Item from a reader as supplied to the `IIngester` -type [] StreamItem = { stream: string; index: int64; event: Equinox.Codec.IEvent } - -/// Core interface for projection system, representing the complete contract a feed consumer uses to deliver batches of work for projection -type IIngester<'Epoch,'Item> = - /// Passes a (lazy) batch of items into the Ingestion Engine; the batch will then be materialized out of band and submitted to the Scheduler - /// Admission is Async in order that the Projector and Ingester can together contrive to force backpressure on the producer of the batches by delaying conclusion of the Async computation - /// Returns the ephemeral position of this entry in the queue of uncheckpointed batches at time of posting, together with current max number of items permissible - abstract member Submit: progressEpoch: 'Epoch * markCompleted: Async * items: 'Item seq -> Async - /// Requests cancellation of ingestion processing as soon as practicable (typically this is in reaction to a lease being revoked) - abstract member Stop: unit -> unit - -[] -module private Impl = - let (|NNA|) xs = if xs = null then Array.empty else xs - let arrayBytes (x:byte[]) = if x = null then 0 else x.Length - let inline eventSize (x : Equinox.Codec.IEvent<_>) = arrayBytes x.Data + arrayBytes x.Meta + x.EventType.Length + 16 - let inline mb x = float x / 1024. / 1024. - let expiredMs ms = - let timer = Stopwatch.StartNew() - fun () -> - let due = timer.ElapsedMilliseconds > ms - if due then timer.Restart() - due - let inline accStopwatch (f : unit -> 't) at = - let sw = Stopwatch.StartNew() - let r = f () - at sw.Elapsed - r - type Sem(max) = - let inner = new SemaphoreSlim(max) - member __.Release(?count) = match defaultArg count 1 with 0 -> () | x -> inner.Release x |> ignore - member __.State = max-inner.CurrentCount,max - /// Wait infinitely to get the semaphore - member __.Await() = inner.Await() |> Async.Ignore - /// Dont use without profiling proving it helps as it doesnt help correctness or legibility - member __.TryWaitWithoutCancellationForPerf() = inner.Wait(0) - /// Only use where you're interested in intenionally busywaiting on a thread - i.e. when you have proven its critical - member __.SpinWaitWithoutCancellationForPerf() = inner.Wait(Timeout.Infinite) |> ignore - member __.HasCapacity = inner.CurrentCount > 0 - -#nowarn "52" // see tmp.Sort - -module Progress = - - type [] internal BatchState = { markCompleted: unit -> unit; streamToRequiredIndex : Dictionary } - - type State<'Pos>() = - let pending = Queue<_>() - let trim () = - while pending.Count <> 0 && pending.Peek().streamToRequiredIndex.Count = 0 do - let batch = pending.Dequeue() - batch.markCompleted() - member __.AppendBatch(markCompleted, reqs : Dictionary) = - pending.Enqueue { markCompleted = markCompleted; streamToRequiredIndex = reqs } - trim () - member __.MarkStreamProgress(stream, index) = - for x in pending do - match x.streamToRequiredIndex.TryGetValue stream with - | true, requiredIndex when requiredIndex <= index -> x.streamToRequiredIndex.Remove stream |> ignore - | _, _ -> () - trim () - member __.InScheduledOrder getStreamWeight = - let streams = HashSet() - let tmp = ResizeArray(16384) - let mutable batch = 0 - for x in pending do - batch <- batch + 1 - for s in x.streamToRequiredIndex.Keys do - if streams.Add s then - tmp.Add(struct (s,struct (batch,-getStreamWeight s))) - tmp.Sort(fun (struct(_,_a)) (struct(_,_b)) -> _a.CompareTo(_b)) - tmp |> Seq.map (fun (struct(s,_)) -> s) - - /// Manages writing of progress - /// - Each write attempt is always of the newest token (each update is assumed to also count for all preceding ones) - /// - retries until success or a new item is posted - type Writer<'Res when 'Res: equality>() = - let pumpSleepMs = 100 - let due = expiredMs 5000L - let mutable committedEpoch = None - let mutable validatedPos = None - let result = Event>() - [] member __.Result = result.Publish - member __.Post(version,f) = - Volatile.Write(&validatedPos,Some (version,f)) - member __.CommittedEpoch = Volatile.Read(&committedEpoch) - member __.Pump() = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - match Volatile.Read &validatedPos with - | Some (v,f) when Volatile.Read(&committedEpoch) <> Some v && due () -> - try do! f - Volatile.Write(&committedEpoch, Some v) - result.Trigger (Choice1Of2 v) - with e -> result.Trigger (Choice2Of2 e) - | _ -> do! Async.Sleep pumpSleepMs } - -module Buffer = - - type [] Span = { index: int64; events: Equinox.Codec.IEvent[] } - module Span = - let (|End|) (x : Span) = x.index + if x.events = null then 0L else x.events.LongLength - let dropBeforeIndex min : Span -> Span = function - | x when x.index >= min -> x // don't adjust if min not within - | End n when n < min -> { index = min; events = [||] } // throw away if before min -#if NET461 - | x -> { index = min; events = x.events |> Seq.skip (min - x.index |> int) |> Seq.toArray } -#else - | x -> { index = min; events = x.events |> Array.skip (min - x.index |> int) } // slice -#endif - let merge min (xs : Span seq) = - let xs = - seq { for x in xs -> { x with events = (|NNA|) x.events } } - |> Seq.map (dropBeforeIndex min) - |> Seq.filter (fun x -> x.events.Length <> 0) - |> Seq.sortBy (fun x -> x.index) - let buffer = ResizeArray() - let mutable curr = None - for x in xs do - match curr, x with - // Not overlapping, no data buffered -> buffer - | None, _ -> - curr <- Some x - // Gap - | Some (End nextIndex as c), x when x.index > nextIndex -> - buffer.Add c - curr <- Some x - // Overlapping, join - | Some (End nextIndex as c), x -> - curr <- Some { c with events = Array.append c.events (dropBeforeIndex nextIndex x).events } - curr |> Option.iter buffer.Add - if buffer.Count = 0 then null else buffer.ToArray() - let slice (maxEvents,maxBytes) (x: Span) = - let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length - // TODO tests etc - let inline estimateBytesAsJsonUtf8 (x: Equinox.Codec.IEvent) = arrayBytes x.Data + arrayBytes x.Meta + (x.EventType.Length * 2) + 96 - let mutable count,bytes = 0, 0 - let mutable countBudget, bytesBudget = maxEvents,maxBytes - let withinLimits (y : Equinox.Codec.IEvent) = - countBudget <- countBudget - 1 - let eventBytes = estimateBytesAsJsonUtf8 y - bytesBudget <- bytesBudget - eventBytes - // always send at least one event in order to surface the problem and have the stream marked malformed - let res = count = 0 || (countBudget >= 0 && bytesBudget >= 0) - if res then count <- count + 1; bytes <- bytes + eventBytes - res - let trimmed = { x with events = x.events |> Array.takeWhile withinLimits } - let stats = trimmed.events.Length, trimmed.events |> Seq.sumBy estimateBytesAsJsonUtf8 - stats, trimmed - type [] StreamSpan = { stream: string; span: Span } - type [] StreamState = { isMalformed: bool; write: int64 option; queue: Span[] } with - member __.Size = - if __.queue = null then 0 - else __.queue |> Seq.collect (fun x -> x.events) |> Seq.sumBy eventSize - member __.IsReady = - if __.queue = null || __.isMalformed then false - else - match __.write, Array.tryHead __.queue with - | Some w, Some { index = i } -> i = w - | None, _ -> true - | _ -> false - module StreamState = - let inline optionCombine f (r1: 'a option) (r2: 'a option) = - match r1, r2 with - | Some x, Some y -> f x y |> Some - | None, None -> None - | None, x | x, None -> x - let combine (s1: StreamState) (s2: StreamState) : StreamState = - let writePos = optionCombine max s1.write s2.write - let items = let (NNA q1, NNA q2) = s1.queue, s2.queue in Seq.append q1 q2 - { write = writePos; queue = Span.merge (defaultArg writePos 0L) items; isMalformed = s1.isMalformed || s2.isMalformed } - - type Streams() = - let states = Dictionary() - let merge stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - - member __.StreamCount = states.Count - member __.Items : seq>= states :> _ - - member __.Merge(items : StreamItem seq) = - for item in items do - merge item.stream { isMalformed = false; write = None; queue = [| { index = item.index; events = [| item.event |] } |] } - member __.Merge(other: Streams) = - for x in other.Items do - merge x.Key x.Value - - member __.Dump(categorize, log : ILogger) = - let mutable waiting, waitingB = 0, 0L - let waitingCats, waitingStreams = CatStats(), CatStats() - for KeyValue (stream,state) in states do - let sz = int64 state.Size - waitingCats.Ingest(categorize stream) - waitingStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, (sz + 512L) / 1024L) - waiting <- waiting + 1 - waitingB <- waitingB + sz - if waiting <> 0 then log.Information("Streams Waiting {busy:n0}/{busyMb:n1}MB ", waiting, mb waitingB) - if waitingCats.Any then log.Information("Waiting Categories, events {@readyCats}", Seq.truncate 5 waitingCats.StatsDescending) - if waitingCats.Any then log.Information("Waiting Streams, KB {@readyStreams}", Seq.truncate 5 waitingStreams.StatsDescending) - -type ISchedulingEngine = - abstract member Stop : unit -> unit - /// Enqueue a batch of items with supplied progress marking function - /// Submission is accepted on trust; they are internally processed in order of submission - /// caller should ensure that (when multiple submitters are in play) no single Range submits more than their fair share - abstract member Submit : markCompleted : (unit -> unit) * items : StreamItem[] -> unit - abstract member SubmitStreamBuffers : Buffer.Streams -> unit - -module Scheduling = - - open Buffer - - type StreamStates() = - let states = Dictionary() - let update stream (state : StreamState) = - match states.TryGetValue stream with - | false, _ -> - states.Add(stream, state) - stream, state - | true, current -> - let updated = StreamState.combine current state - states.[stream] <- updated - stream, updated - let updateWritePos stream isMalformed pos span = update stream { isMalformed = isMalformed; write = pos; queue = span } - let markCompleted stream index = updateWritePos stream false (Some index) null |> ignore - let mergeBuffered (buffer : Streams) = - for x in buffer.Items do - update x.Key x.Value |> ignore - - let busy = HashSet() - let pending trySlipstreamed (requestedOrder : string seq) : seq = seq { - let proposed = HashSet() - for s in requestedOrder do - let state = states.[s] - if state.IsReady && not (busy.Contains s) then - proposed.Add s |> ignore - yield state.write, { stream = s; span = state.queue.[0] } - if trySlipstreamed then - // [lazily] Slipstream in futher events that have been posted to streams which we've already visited - for KeyValue(s,v) in states do - if v.IsReady && not (busy.Contains s) && proposed.Add s then - yield v.write, { stream = s; span = v.queue.[0] } } - let markBusy stream = busy.Add stream |> ignore - let markNotBusy stream = busy.Remove stream |> ignore - - // Result is intentionally a thread-safe persisent data structure - // This enables the (potentially multiple) Ingesters to determine streams (for which they potentially have successor events) that are in play - // Ingesters then supply these 'preview events' in advance of the processing being scheduled - // This enables the projection logic to roll future work into the current work in the interests of medium term throughput - member __.InternalMerge buffer = mergeBuffered buffer - member __.InternalUpdate stream pos queue = update stream { isMalformed = false; write = Some pos; queue = queue } - member __.Add(stream, index, event, ?isMalformed) = - updateWritePos stream (defaultArg isMalformed false) None [| { index = index; events = [| event |] } |] - member __.Add(batch: StreamSpan, isMalformed) = - updateWritePos batch.stream isMalformed None [| { index = batch.span.index; events = batch.span.events } |] - member __.SetMalformed(stream,isMalformed) = - updateWritePos stream isMalformed None [| { index = 0L; events = null } |] - member __.QueueWeight(stream) = - states.[stream].queue.[0].events |> Seq.sumBy eventSize - member __.MarkBusy stream = - markBusy stream - member __.MarkCompleted(stream, index) = - markNotBusy stream - markCompleted stream index - member __.MarkFailed stream = - markNotBusy stream - member __.Pending(trySlipstreamed, byQueuedPriority : string seq) : (int64 option * StreamSpan) seq = - pending trySlipstreamed byQueuedPriority - member __.Dump(log : ILogger, categorize) = - let mutable busyCount, busyB, ready, readyB, unprefixed, unprefixedB, malformed, malformedB, synced = 0, 0L, 0, 0L, 0, 0L, 0, 0L, 0 - let busyCats, readyCats, readyStreams, unprefixedStreams, malformedStreams = CatStats(), CatStats(), CatStats(), CatStats(), CatStats() - let kb sz = (sz + 512L) / 1024L - for KeyValue (stream,state) in states do - match int64 state.Size with - | 0L -> - synced <- synced + 1 - | sz when busy.Contains stream -> - busyCats.Ingest(categorize stream) - busyCount <- busyCount + 1 - busyB <- busyB + sz - | sz when state.isMalformed -> - malformedStreams.Ingest(stream, mb sz |> int64) - malformed <- malformed + 1 - malformedB <- malformedB + sz - | sz when not state.IsReady -> - unprefixedStreams.Ingest(stream, mb sz |> int64) - unprefixed <- unprefixed + 1 - unprefixedB <- unprefixedB + sz - | sz -> - readyCats.Ingest(categorize stream) - readyStreams.Ingest(sprintf "%s@%dx%d" stream (defaultArg state.write 0L) state.queue.[0].events.Length, kb sz) - ready <- ready + 1 - readyB <- readyB + sz - log.Information("Streams Synced {synced:n0} Active {busy:n0}/{busyMb:n1}MB Ready {ready:n0}/{readyMb:n1}MB Waiting {waiting}/{waitingMb:n1}MB Malformed {malformed}/{malformedMb:n1}MB", - synced, busyCount, mb busyB, ready, mb readyB, unprefixed, mb unprefixedB, malformed, mb malformedB) - if busyCats.Any then log.Information("Active Categories, events {@busyCats}", Seq.truncate 5 busyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Categories, events {@readyCats}", Seq.truncate 5 readyCats.StatsDescending) - if readyCats.Any then log.Information("Ready Streams, KB {@readyStreams}", Seq.truncate 5 readyStreams.StatsDescending) - if unprefixedStreams.Any then log.Information("Waiting Streams, KB {@missingStreams}", Seq.truncate 3 unprefixedStreams.StatsDescending) - if malformedStreams.Any then log.Information("Malformed Streams, MB {@malformedStreams}", malformedStreams.StatsDescending) - - /// Messages used internally by projector, including synthetic ones for the purposes of the `Stats` listeners - [] - type InternalMessage<'R> = - /// Periodic submission of events as they are read, grouped by stream for efficient merging into the StreamState - | Merge of Streams - /// Stats per submitted batch for stats listeners to aggregate - | Added of streams: int * skip: int * events: int - /// Result of processing on stream - result (with basic stats) or the `exn` encountered - | Result of stream: string * outcome: 'R - - type BufferState = Idle | Busy | Full | Slipstreaming - /// Gathers stats pertaining to the core projection/ingestion activity - type Stats<'R,'E>(log : ILogger, statsInterval : TimeSpan, stateInterval : TimeSpan) = - let states, fullCycles, cycles, resultCompleted, resultExn = CatStats(), ref 0, ref 0, ref 0, ref 0 - let merges, mergedStreams, batchesPended, streamsPended, eventsSkipped, eventsPended = ref 0, ref 0, ref 0, ref 0, ref 0, ref 0 - let statsDue, stateDue = expiredMs (int64 statsInterval.TotalMilliseconds), expiredMs (int64 stateInterval.TotalMilliseconds) - let mutable dt,ft,it,st,mt = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero - let dumpStats (used,maxDop) (waitingBatches,pendingMerges) = - log.Information("Cycles {cycles}/{fullCycles} {@states} Projecting {busy}/{processors} Completed {completed} Exceptions {exns}", - !cycles, !fullCycles, states.StatsDescending, used, maxDop, !resultCompleted, !resultExn) - cycles := 0; fullCycles := 0; states.Clear(); resultCompleted := 0; resultExn:= 0 - log.Information("Batches Waiting {batchesWaiting} Started {batches} ({streams:n0}s {events:n0}-{skipped:n0}e) Merged {merges}/{pendingMerges} {mergedStreams}s", - waitingBatches, !batchesPended, !streamsPended, !eventsSkipped + !eventsPended, !eventsSkipped, !merges, pendingMerges, !mergedStreams) - batchesPended := 0; streamsPended := 0; eventsSkipped := 0; eventsPended := 0; merges := 0; mergedStreams := 0 - log.Information("Scheduling Streams {mt:n1}s Batches {it:n1}s Dispatch {ft:n1}s Results {dt:n1}s Stats {st:n1}s", - mt.TotalSeconds, it.TotalSeconds, ft.TotalSeconds, dt.TotalSeconds, st.TotalSeconds) - dt <- TimeSpan.Zero; ft <- TimeSpan.Zero; it <- TimeSpan.Zero; st <- TimeSpan.Zero; mt <- TimeSpan.Zero - abstract member Handle : InternalMessage> -> unit - default __.Handle msg = msg |> function - | Merge buffer -> - mergedStreams := !mergedStreams + buffer.StreamCount - incr merges - | Added (streams, skipped, events) -> - incr batchesPended - streamsPended := !streamsPended + streams - eventsPended := !eventsPended + events - eventsSkipped := !eventsSkipped + skipped - | Result (_stream, Choice1Of2 _) -> - incr resultCompleted - | Result (_stream, Choice2Of2 _) -> - incr resultExn - member __.DumpStats((used,max), pendingCount) = - incr cycles - if statsDue () then - dumpStats (used,max) pendingCount - __.DumpExtraStats() - member __.TryDumpState(state,dump,(_dt,_ft,_mt,_it,_st)) = - dt <- dt + _dt - ft <- ft + _ft - mt <- mt + _mt - it <- it + _it - st <- st + _st - incr fullCycles - states.Ingest(string state) - - let due = stateDue () - if due then - dump log - due - /// Allows an ingester or projector to wire in custom stats (typically based on data gathered in a `Handle` override) - abstract DumpExtraStats : unit -> unit - default __.DumpExtraStats () = () - - /// Coordinates the dispatching of work and emission of results, subject to the maxDop concurrent processors constraint - type Dispatcher<'R>(maxDop) = - // Using a Queue as a) the ordering is more correct, favoring more important work b) we are adding from many threads so no value in ConcurrentBag'sthread-affinity - let work = new BlockingCollection<_>(ConcurrentQueue<_>()) - let result = Event<'R>() - let dop = new Sem(maxDop) - let dispatch work = async { - let! res = work - result.Trigger res - dop.Release() } - [] member __.Result = result.Publish - member __.HasCapacity = dop.HasCapacity - member __.State = dop.State - member __.TryAdd(item) = - if dop.TryWaitWithoutCancellationForPerf() then - work.Add(item) - true - else false - member __.Pump () = async { - let! ct = Async.CancellationToken - for item in work.GetConsumingEnumerable ct do - Async.Start(dispatch item) } - - /// Consolidates ingested events into streams; coordinates dispatching of these to projector/ingester in the order implied by the submission order - /// a) does not itself perform any reading activities - /// b) triggers synchronous callbacks as batches complete; writing of progress is managed asynchronously by the TrancheEngine(s) - /// c) submits work to the supplied Dispatcher (which it triggers pumping of) - /// d) periodically reports state (with hooks for ingestion engines to report same) - type Engine<'R,'E>(dispatcher : Dispatcher<_>, stats : Stats<'R,'E>, project : int64 option * StreamSpan -> Async>, interpretProgress, dumpStreams, ?maxBatches) = - let sleepIntervalMs = 1 - let maxBatches = defaultArg maxBatches 16 - let cts = new CancellationTokenSource() - let work = ConcurrentStack>>() // dont need them ordered so Queue is unwarranted; usage is cross-thread so Bag is not better - let pending = ConcurrentQueue<_*StreamItem[]>() // Queue as need ordering - let streams = StreamStates() - let progressState = Progress.State() - - // Arguably could be a bag, which would be more efficient, but sequencing in order of submission yields cheaper merges - let streamsPending = ConcurrentQueue() // pulled from `work` and kept aside for processing at the right time as they are encountered - let mutable streamsMerged : Streams option = None - let slipstreamsCoalescing = Sem(1) - let tryGetStream () = match streamsPending.TryDequeue() with true,x -> Some x | false,_-> None - // We periodically process streamwise submissions of events from Ingesters in advance of them entering the processing queue as pending batches - // This allows events that are not yet a requirement for a given batch to complete to be included in work before it becomes due, smoothing throughput - let continuouslyCompactStreamMerges () = async { - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - do! slipstreamsCoalescing.Await() - streamsMerged <- (streamsMerged,tryGetStream()) ||> StreamState.optionCombine (fun x y -> x.Merge y; x) - slipstreamsCoalescing.Release() - do! Async.Sleep 1 } // ms // needs to be long enough for ingestStreamMerges to be able to grab - let ingestStreamMerges () = - slipstreamsCoalescing.SpinWaitWithoutCancellationForPerf() - match streamsMerged with - | None -> () - | Some ready -> - streamsMerged <- None - streams.InternalMerge ready - slipstreamsCoalescing.Release() - // ingest information to be gleaned from processing the results into `streams` - static let workLocalBuffer = Array.zeroCreate 1024 - let tryDrainResults feedStats = - let mutable worked, more = false, true - while more do - let c = work.TryPopRange(workLocalBuffer) - if c = 0 (*&& work.IsEmpty*) then more <- false else worked <- true - for i in 0..c-1 do - let x = workLocalBuffer.[i] - match x with - | Added _ -> () // Only processed in Stats (and actually never enters this queue) - | Merge buffer -> streamsPending.Enqueue buffer // put aside as a) they can be done more efficiently in bulk b) we only want to pay the tax at the right time - | Result (stream,res) -> - match interpretProgress streams stream res with - | None -> streams.MarkFailed stream - | Some index -> - progressState.MarkStreamProgress(stream,index) - streams.MarkCompleted(stream,index) - feedStats x - worked - // On ech iteration, we try to fill the in-flight queue, taking the oldest and/or heaviest streams first - let tryFillDispatcher includeSlipstreamed = - let mutable hasCapacity, dispatched = dispatcher.HasCapacity, false - if hasCapacity then - let potential = streams.Pending(includeSlipstreamed, progressState.InScheduledOrder streams.QueueWeight) - let xs = potential.GetEnumerator() - while xs.MoveNext() && hasCapacity do - let (_,{stream = s} : StreamSpan) as item = xs.Current - let succeeded = dispatcher.TryAdd(async { let! r = project item in return s, r }) - if succeeded then streams.MarkBusy s - dispatched <- dispatched || succeeded // if we added any request, we'll skip sleeping - hasCapacity <- succeeded - hasCapacity, dispatched - // Take an incoming batch of events, correlating it against our known stream state to yield a set of remaining work - let ingestPendingBatch feedStats (markCompleted, items : StreamItem seq) = - let inline validVsSkip (streamState : StreamState) (item : StreamItem) = - match streamState.write, item.index + 1L with - | Some cw, required when cw >= required -> 0, 1 - | _ -> 1, 0 - let reqs = Dictionary() - let mutable count, skipCount = 0, 0 - for item in items do - let stream,streamState = streams.Add(item.stream,item.index,item.event) - match validVsSkip streamState item with - | 0, skip -> - skipCount <- skipCount + skip - | required, _ -> - count <- count + required - reqs.[stream] <- item.index+1L - progressState.AppendBatch(markCompleted,reqs) - feedStats <| Added (reqs.Count,skipCount,count) - - member private __.Pump = async { - use _ = dispatcher.Result.Subscribe(Result >> work.Push) - Async.Start(dispatcher.Pump(), cts.Token) - Async.Start(continuouslyCompactStreamMerges (), cts.Token) - while not cts.IsCancellationRequested do - let mutable idle, dispatcherState, remaining = true, Idle, 16 - let mutable dt,ft,mt,it,st = TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero, TimeSpan.Zero - while remaining <> 0 do - remaining <- remaining - 1 - // 1. propagate write write outcomes to buffer (can mark batches completed etc) - let processedResults = (fun () -> tryDrainResults stats.Handle) |> accStopwatch <| fun x -> dt <- dt + x - // 2. top up provisioning of writers queue - let hasCapacity, dispatched = (fun () -> tryFillDispatcher (dispatcherState = Slipstreaming)) |> accStopwatch <| fun x -> ft <- ft + x - idle <- idle && not processedResults && not dispatched - match dispatcherState with - | Idle when not hasCapacity -> - // If we've achieved full state, spin around the loop to dump stats and ingest reader data - dispatcherState <- Full - remaining <- 0 - | Idle when remaining = 0 -> - dispatcherState <- Busy - | Idle -> // need to bring more work into the pool as we can't fill the work queue from what we have - // If we're going to fill the write queue with random work, we should bring all read events into the state first - // If we're going to bring in lots of batches, that's more efficient when the streamwise merges are carried out first - let mutable more, batchesTaken = true, 0 - ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t - while more do - match pending.TryDequeue() with - | true, batch -> - (fun () -> ingestPendingBatch stats.Handle batch) |> accStopwatch <| fun t -> it <- it + t - batchesTaken <- batchesTaken + 1 - more <- batchesTaken < maxBatches - | false,_ when batchesTaken <> 0 -> - more <- false - | false,_ when batchesTaken = 0 -> - dispatcherState <- Slipstreaming - more <- false - | false,_ -> () - | Slipstreaming -> // only do one round of slipstreaming - remaining <- 0 - | Busy | Full -> failwith "Not handled here" - // This loop can take a long time; attempt logging of stats per iteration - (fun () -> stats.DumpStats(dispatcher.State,(pending.Count,streamsPending.Count))) |> accStopwatch <| fun t -> st <- st + t - // Do another ingest before a) reporting state to give best picture b) going to sleep in order to get work out of the way - ingestStreamMerges |> accStopwatch <| fun t -> mt <- mt + t - // 3. Record completion state once per full iteration; dumping streams is expensive so needs to be done infrequently - if not (stats.TryDumpState(dispatcherState,dumpStreams streams,(dt,ft,mt,it,st))) && not idle then - // 4. Do a minimal sleep so we don't run completely hot when empty (unless we did something non-trivial) - Thread.Sleep sleepIntervalMs } // Not Async.Sleep so we don't give up the thread - - static member Start(stats, projectorDop, project, interpretProgress, dumpStreams) = - let dispatcher = Dispatcher(projectorDop) - let instance = new Engine<_,_>(dispatcher, stats, project, interpretProgress, dumpStreams) - Async.Start instance.Pump - instance - - interface ISchedulingEngine with - member __.Stop() = cts.Cancel() - member __.Submit(markCompleted: (unit -> unit), items : StreamItem[]) = pending.Enqueue (markCompleted, items) - member __.SubmitStreamBuffers streams = work.Push <| Merge streams - -module Ingestion = - - [] - type Message = - | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq - //| StreamSegment of span: StreamSpan - | EndOfSeries of seriesIndex: int - - type private Stats(log : ILogger, maxPendingBatches, categorize, statsInterval : TimeSpan) = - let mutable pendingBatchCount, validatedEpoch, comittedEpoch : int * int64 option * int64 option = 0, None, None - let progCommitFails, progCommits = ref 0, ref 0 - let cycles, batchesPended, streamsPended, eventsPended = ref 0, ref 0, ref 0, ref 0 - let statsDue = expiredMs (int64 statsInterval.TotalMilliseconds) - let dumpStats (available,maxDop) (readingAhead,ready) = - log.Information("Buffering Cycles {cycles} Ingested {batches} ({streams:n0}s {events:n0}e) Submissions {active}/{writers}", - !cycles, !batchesPended, !streamsPended, !eventsPended, available, maxDop) - cycles := 0; batchesPended := 0; streamsPended := 0; eventsPended := 0 - let mutable buffered = 0 - let count (xs : IDictionary>) = seq { for x in xs do buffered <- buffered + x.Value.Count; yield x.Key, x.Value.Count } |> Seq.sortBy fst |> Seq.toArray - let ahead, ready = count readingAhead, count ready - if buffered <> 0 then log.Information("Holding {buffered} Reading {@reading} Ready {@ready}", buffered, ahead, ready) - if !progCommitFails <> 0 || !progCommits <> 0 then - match comittedEpoch with - | None -> - log.Error("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated}; writing failing: {failures} failures ({commits} successful commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, !progCommitFails, !progCommits) - | Some committed when !progCommitFails <> 0 -> - log.Warning("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits, {failures} failures)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits, !progCommitFails) - | Some committed -> - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed}, {commits} commits)", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, committed, !progCommits) - progCommits := 0; progCommitFails := 0 - else - log.Information("Uncommitted {pendingBatches}/{maxPendingBatches} @ {validated} (committed: {committed})", - pendingBatchCount, maxPendingBatches, Option.toNullable validatedEpoch, Option.toNullable comittedEpoch) - member __.Handle : InternalMessage -> unit = function - | Batch _ | ActivateSeries _ | CloseSeries _-> () // stats are managed via Added internal message in same cycle - | ProgressResult (Choice1Of2 epoch) -> - incr progCommits - comittedEpoch <- Some epoch - | ProgressResult (Choice2Of2 (_exn : exn)) -> - incr progCommitFails - | Added (streams,events) -> - incr batchesPended - streamsPended := !streamsPended + streams - eventsPended := !eventsPended + events - member __.HandleValidated(epoch, pendingBatches) = - validatedEpoch <- epoch - pendingBatchCount <- pendingBatches - member __.HandleCommitted epoch = - comittedEpoch <- epoch - member __.TryDump((available,maxDop),streams : Buffer.Streams,readingAhead,ready) = - incr cycles - if statsDue () then - dumpStats (available,maxDop) (readingAhead,ready) - streams.Dump(categorize,log) - - and [] private InternalMessage = - | Batch of seriesIndex: int * epoch: int64 * markCompleted: Async * items: StreamItem seq - /// Result from updating of Progress to backing store - processed up to nominated `epoch` or threw `exn` - | ProgressResult of Choice - /// Internal message for stats purposes - | Added of steams: int * events: int - | CloseSeries of seriesIndex: int - | ActivateSeries of seriesIndex: int - - let tryTake key (dict: Dictionary<_,_>) = - match dict.TryGetValue key with - | true, value -> - dict.Remove key |> ignore - Some value - | false, _ -> None - - /// Holds batches away from Core processing to limit in-flight processing - type Engine(log : ILogger, scheduler: ISchedulingEngine, maxRead, maxSubmissions, initialSeriesIndex, categorize, statsInterval : TimeSpan, ?pumpDelayMs) = - let cts = new CancellationTokenSource() - let pumpDelayMs = defaultArg pumpDelayMs 5 - let work = ConcurrentQueue() // Queue as need ordering semantically - let readMax = new Sem(maxRead) - let submissionsMax = new Sem(maxSubmissions) - let mutable streams = Buffer.Streams() - let grabAccumulatedStreams () = let t = streams in streams <- Buffer.Streams(); t - let stats = Stats(log, maxRead, categorize, statsInterval) - let pending = Queue<_>() - let readingAhead, ready = Dictionary>(), Dictionary>() - let progressWriter = Progress.Writer<_>() - let mutable activeSeries = initialSeriesIndex - let mutable validatedPos = None - - let handle = function - | Batch (seriesId, epoch, checkpoint, items) -> - let batchInfo = - let items = Array.ofSeq items - streams.Merge items - let markCompleted () = - submissionsMax.Release() - readMax.Release() - validatedPos <- Some (epoch,checkpoint) - work.Enqueue(Added (HashSet(seq { for x in items -> x.stream }).Count,items.Length)) - markCompleted, items - if activeSeries = seriesId then pending.Enqueue batchInfo - else - match readingAhead.TryGetValue seriesId with - | false, _ -> readingAhead.[seriesId] <- ResizeArray(Seq.singleton batchInfo) - | true,current -> current.Add(batchInfo) - | CloseSeries seriesIndex -> - if activeSeries = seriesIndex then - log.Information("Completed reading active series {activeSeries}; moving to next", activeSeries) - work.Enqueue <| ActivateSeries (activeSeries + 1) - else - match readingAhead |> tryTake seriesIndex with - | Some batchesRead -> - ready.[seriesIndex] <- batchesRead - log.Information("Completed reading {series}, marking {buffered} buffered items ready", seriesIndex, batchesRead.Count) - | None -> - ready.[seriesIndex] <- ResizeArray() - log.Information("Completed reading {series}, leaving empty batch list", seriesIndex) - | ActivateSeries newActiveSeries -> - activeSeries <- newActiveSeries - let buffered = - match ready |> tryTake newActiveSeries with - | Some completedChunkBatches -> - completedChunkBatches |> Seq.iter pending.Enqueue - work.Enqueue <| ActivateSeries (newActiveSeries + 1) - completedChunkBatches.Count - | None -> - match readingAhead |> tryTake newActiveSeries with - | Some batchesReadToDate -> batchesReadToDate |> Seq.iter pending.Enqueue; batchesReadToDate.Count - | None -> 0 - log.Information("Moving to series {activeChunk}, releasing {buffered} buffered batches, {ready} others ready, {ahead} reading ahead", - newActiveSeries, buffered, ready.Count, readingAhead.Count) - // These events are for stats purposes - | Added _ - | ProgressResult _ -> () - - member private __.Pump = async { - use _ = progressWriter.Result.Subscribe(ProgressResult >> work.Enqueue) - Async.Start(progressWriter.Pump(), cts.Token) - let presubmitInterval = expiredMs (4000L*2L) - while not cts.IsCancellationRequested do - try let mutable itemLimit = 4096 - while itemLimit > 0 do - match work.TryDequeue() with - | true, x -> handle x; stats.Handle x; itemLimit <- itemLimit - 1 - | false, _ -> itemLimit <- 0 - // 1. Update any progress into the stats - stats.HandleValidated(Option.map fst validatedPos, fst submissionsMax.State) - validatedPos |> Option.iter progressWriter.Post - stats.HandleCommitted progressWriter.CommittedEpoch - // 2. Forward info grouped by streams into processor in small batches - if presubmitInterval () then - grabAccumulatedStreams () |> scheduler.SubmitStreamBuffers - - // 3. Submit to ingester until read queue, tranche limit or ingester limit exhausted - while pending.Count <> 0 && submissionsMax.HasCapacity do - // mark off a write as being in progress (there is a race if there are multiple Ingesters, but thats good) - do! submissionsMax.Await() - scheduler.Submit(pending.Dequeue()) - // 4. Periodically emit status info - stats.TryDump(submissionsMax.State,streams,readingAhead,ready) - do! Async.Sleep pumpDelayMs - with e -> log.Error(e,"Buffer thread exception") } - - /// Generalized; normal usage is via Ingester.Start, this is used by the `eqxsync` template to handle striped reading for bulk ingestion purposes - static member Start(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval) = - let instance = new Engine(log, scheduler, maxRead, maxSubmissions, startingSeriesId, categorize, statsInterval = statsInterval) - Async.Start instance.Pump - instance - - /// Awaits space in `read` to limit reading ahead - yields (used,maximum) counts from Read Semaphore for logging purposes - member __.Submit(content : Message) = async { - do! readMax.Await() - match content with - | Message.Batch (seriesId, epoch, markBatchCompleted, events) -> - work.Enqueue <| Batch (seriesId, epoch, markBatchCompleted, events) - // NB readMax.Release() is effected in the Batch handler's MarkCompleted() - | Message.EndOfSeries seriesId -> - work.Enqueue <| CloseSeries seriesId - readMax.Release() - return readMax.State } - - /// As range assignments get revoked, a user is expected to `Stop `the active processing thread for the Ingester before releasing references to it - member __.Stop() = cts.Cancel() \ No newline at end of file diff --git a/equinox-sync/Sync/ProjectorSink.fs b/equinox-sync/Sync/ProjectorSink.fs index f2dfa7707..3470cf51a 100644 --- a/equinox-sync/Sync/ProjectorSink.fs +++ b/equinox-sync/Sync/ProjectorSink.fs @@ -1,6 +1,6 @@ module SyncTemplate.ProjectorSink -open Equinox.Projection2 +open Equinox.Projection open System type Scheduler = diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index a3e29786e..0ebdc0413 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -10,7 +10,6 @@ - @@ -19,10 +18,10 @@ - + - - + + From 05deff3741cadbadac7e8391a2fcdadeaba97d1a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 5 Jun 2019 10:05:52 +0100 Subject: [PATCH 332/353] Replace ifdefs with matches --- equinox-sync/.template.config/template.json | 7 - equinox-sync/Sync/Checkpoint.fs | 110 ----- equinox-sync/Sync/EventStoreSource.fs | 402 ---------------- equinox-sync/Sync/Program.fs | 492 +++++++++++--------- equinox-sync/Sync/Sync.fsproj | 8 +- equinox-sync/equinox-sync.sln | 12 + 6 files changed, 287 insertions(+), 744 deletions(-) delete mode 100644 equinox-sync/Sync/Checkpoint.fs delete mode 100644 equinox-sync/Sync/EventStoreSource.fs diff --git a/equinox-sync/.template.config/template.json b/equinox-sync/.template.config/template.json index 513229764..f5a1fcae5 100644 --- a/equinox-sync/.template.config/template.json +++ b/equinox-sync/.template.config/template.json @@ -20,13 +20,6 @@ "preferNameDirectory": true, "symbols": { - "eventstore": { - "type": "parameter", - "datatype": "bool", - "isRequired": false, - "defaultValue": "false", - "description": "Use EventStore as a source; default: use a CosmosDb ChangeFeedProcessor as the source." - }, "marveleqx": { "type": "parameter", "datatype": "bool", diff --git a/equinox-sync/Sync/Checkpoint.fs b/equinox-sync/Sync/Checkpoint.fs deleted file mode 100644 index 49d1a801d..000000000 --- a/equinox-sync/Sync/Checkpoint.fs +++ /dev/null @@ -1,110 +0,0 @@ -module SyncTemplate.Checkpoint - -open FSharp.UMX -open System // must shadow UMX to use DateTimeOffSet - -type CheckpointSeriesId = string -and [] checkpointSeriesId -module CheckpointSeriesId = let ofGroupName (groupName : string) = UMX.tag groupName - -// NB - these schemas reflect the actual storage formats and hence need to be versioned with care -module Events = - type Checkpoint = { at: DateTimeOffset; nextCheckpointDue: DateTimeOffset; pos: int64 } - type Config = { checkpointFreqS: int } - type Started = { config: Config; origin: Checkpoint } - type Checkpointed = { config: Config; pos: Checkpoint } - type Unfolded = { config: Config; state: Checkpoint } - type Event = - | Started of Started - | Checkpointed of Checkpointed - | Overrode of Checkpointed - | [] - Unfolded of Unfolded - interface TypeShape.UnionContract.IUnionContract - -module Folds = - type State = NotStarted | Running of Events.Unfolded - - let initial : State = NotStarted - let private evolve _ignoreState = function - | Events.Started { config = cfg; origin=originState } -> Running { config = cfg; state = originState } - | Events.Checkpointed e | Events.Overrode e -> Running { config = e.config; state = e.pos } - | Events.Unfolded runningState -> Running runningState - let fold (state: State) = Seq.fold evolve state - let isOrigin _state = true // we can build a state from any of the events and/or an unfold - let unfold state = - match state with - | NotStarted -> failwith "should never produce a NotStarted state" - | Running state -> Events.Unfolded {config = state.config; state=state.state} - - /// We only want to generate a first class event every N minutes, while efficiently writing contingent on the current etag value - //let postProcess events state = - // let checkpointEventIsRedundant (e: Events.Checkpointed) (s: Events.Unfolded) = - // s.state.nextCheckpointDue = e.pos.nextCheckpointDue - // && s.state.pos <> e.pos.pos - // match events, state with - // | [Events.Checkpointed e], (Running state as s) when checkpointEventIsRedundant e state -> - // [],unfold s - // | xs, state -> - // xs,unfold state - -type Command = - | Start of at: DateTimeOffset * checkpointFreq: TimeSpan * pos: int64 - | Override of at: DateTimeOffset * checkpointFreq: TimeSpan * pos: int64 - | Update of at: DateTimeOffset * pos: int64 - -module Commands = - let interpret command (state : Folds.State) = - let mkCheckpoint at next pos = { at=at; nextCheckpointDue = next; pos = pos } : Events.Checkpoint - let mk (at : DateTimeOffset) (interval: TimeSpan) pos : Events.Config * Events.Checkpoint= - let freq = int interval.TotalSeconds - let next = at.AddSeconds(float freq) - { checkpointFreqS = freq }, mkCheckpoint at next pos - match command, state with - | Start (at, freq, pos), Folds.NotStarted -> - let config, checkpoint = mk at freq pos - [Events.Started { config = config; origin = checkpoint}] - | Override (at, freq, pos), Folds.Running _ -> - let config, checkpoint = mk at freq pos - [Events.Overrode { config = config; pos = checkpoint}] - | Update (at,pos), Folds.Running state -> - // Force a write every N seconds regardless of whether the position has actually changed - if state.state.pos = pos && at < state.state.nextCheckpointDue then [] else - let freq = TimeSpan.FromSeconds <| float state.config.checkpointFreqS - let config, checkpoint = mk at freq pos - [Events.Checkpointed { config = config; pos = checkpoint}] - | c, s -> failwithf "Command %A invalid when %A" c s - -type Service(log, resolveStream, ?maxAttempts) = - let (|AggregateId|) (id : CheckpointSeriesId) = Equinox.AggregateId ("Sync", % id) - let (|Stream|) (AggregateId id) = Equinox.Stream(log, resolveStream id, defaultArg maxAttempts 3) - let execute (Stream stream) cmd = stream.Transact(Commands.interpret cmd) - - /// Determines the present state of the CheckpointSequence - member __.Read(Stream stream) = - stream.Query id - - /// Start a checkpointing series with the supplied parameters - /// NB will fail if already existing; caller should select to `Start` or `Override` based on whether Read indicates state is Running Or NotStarted - member __.Start(id, freq: TimeSpan, pos: int64) = - execute id <| Command.Start(DateTimeOffset.UtcNow, freq, pos) - - /// Override a checkpointing series with the supplied parameters - /// NB fails if not already initialized; caller should select to `Start` or `Override` based on whether Read indicates state is Running Or NotStarted - member __.Override(id, freq: TimeSpan, pos: int64) = - execute id <| Command.Override(DateTimeOffset.UtcNow, freq, pos) - - /// Ingest a position update - /// NB fails if not already initialized; caller should ensure correct initialization has taken place via Read -> Start - member __.Commit(id, pos: int64) = - execute id <| Command.Update(DateTimeOffset.UtcNow, pos) - -// General pattern is that an Equinox Service is a singleton and calls pass an inentifier for a stream per call -// This light wrapper means we can adhere to that general pattern yet still end up with lef=gible code while we in practice only maintain a single checkpoint series per running app -type CheckpointSeries(name, log, resolveStream) = - let seriesId = CheckpointSeriesId.ofGroupName name - let inner = Service(log, resolveStream) - member __.Read = inner.Read seriesId - member __.Start(freq, pos) = inner.Start(seriesId, freq, pos) - member __.Override(freq, pos) = inner.Override(seriesId, freq, pos) - member __.Commit(pos) = inner.Commit(seriesId, pos) \ No newline at end of file diff --git a/equinox-sync/Sync/EventStoreSource.fs b/equinox-sync/Sync/EventStoreSource.fs deleted file mode 100644 index cd4927667..000000000 --- a/equinox-sync/Sync/EventStoreSource.fs +++ /dev/null @@ -1,402 +0,0 @@ -module SyncTemplate.EventStoreSource - -open Equinox.Store // AwaitTaskCorrect -open Equinox.Projection -open EventStore.ClientAPI -open Serilog // NB Needs to shadow ILogger -open System -open System.Collections.Generic -open System.Diagnostics -open System.Threading - -type EventStore.ClientAPI.RecordedEvent with - member __.Timestamp = System.DateTimeOffset.FromUnixTimeMilliseconds(__.CreatedEpoch) - -[] -module private Impl = - let inline arrayBytes (x:byte[]) = if x = null then 0 else x.Length - let inline recPayloadBytes (x: EventStore.ClientAPI.RecordedEvent) = arrayBytes x.Data + arrayBytes x.Metadata - let inline payloadBytes (x: EventStore.ClientAPI.ResolvedEvent) = recPayloadBytes x.Event + x.OriginalStreamId.Length * 2 - let inline mb x = float x / 1024. / 1024. - -let toIngestionItem (e : RecordedEvent) : StreamItem = - let meta' = if e.Metadata <> null && e.Metadata.Length = 0 then null else e.Metadata - let data' = if e.Data <> null && e.Data.Length = 0 then null else e.Data - let event : Equinox.Codec.IEvent<_> = Equinox.Codec.Core.EventData.Create(e.EventType, data', meta', e.Timestamp) :> _ - { stream = e.EventStreamId; index = e.EventNumber; event = event} - -/// Maintains ingestion stats (thread safe via lock free data structures so it can be used across multiple overlapping readers) -type OverallStats(?statsInterval) = - let intervalMs = let t = defaultArg statsInterval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let overallStart, progressStart = Stopwatch.StartNew(), Stopwatch.StartNew() - let mutable totalEvents, totalBytes = 0L, 0L - member __.Ingest(batchEvents, batchBytes) = - Interlocked.Add(&totalEvents,batchEvents) |> ignore - Interlocked.Add(&totalBytes,batchBytes) |> ignore - member __.Bytes = totalBytes - member __.Events = totalEvents - member __.DumpIfIntervalExpired(?force) = - if progressStart.ElapsedMilliseconds > intervalMs || force = Some true then - let totalMb = mb totalBytes - if totalEvents <> 0L then - Log.Information("Reader Throughput {events} events {gb:n1}GB {mb:n2}MB/s", - totalEvents, totalMb/1024., totalMb*1000./float overallStart.ElapsedMilliseconds) - progressStart.Restart() - -/// Maintains stats for traversals of $all; Threadsafe [via naive locks] so can be used by multiple stripes reading concurrently -type SliceStatsBuffer(categorize, ?interval) = - let intervalMs = let t = defaultArg interval (TimeSpan.FromMinutes 5.) in t.TotalMilliseconds |> int64 - let recentCats, accStart = Dictionary(), Stopwatch.StartNew() - member __.Ingest(slice: AllEventsSlice) = - lock recentCats <| fun () -> - let mutable batchBytes = 0 - for x in slice.Events do - let cat = categorize x.OriginalStreamId - let eventBytes = payloadBytes x - match recentCats.TryGetValue cat with - | true, (currCount, currSize) -> recentCats.[cat] <- (currCount + 1, currSize+eventBytes) - | false, _ -> recentCats.[cat] <- (1, eventBytes) - batchBytes <- batchBytes + eventBytes - __.DumpIfIntervalExpired() - slice.Events.Length, int64 batchBytes - member __.DumpIfIntervalExpired(?force) = - if accStart.ElapsedMilliseconds > intervalMs || defaultArg force false then - lock recentCats <| fun () -> - let log kind limit xs = - let cats = - [| for KeyValue (s,(c,b)) in xs |> Seq.sortByDescending (fun (KeyValue (_,(_,b))) -> b) -> - mb (int64 b) |> round, s, c |] - if (not << Array.isEmpty) cats then - let mb, events, top = Array.sumBy (fun (mb, _, _) -> mb) cats, Array.sumBy (fun (_, _, c) -> c) cats, Seq.truncate limit cats - Log.Information("Reader {kind} {mb:n0}MB {events:n0} events categories: {@cats} (MB/cat/count)", kind, mb, events, top) - recentCats |> log "Total" 3 - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$" |> not) |> log "payload" 100 - recentCats |> Seq.where (fun x -> x.Key.StartsWith "$") |> log "meta" 100 - recentCats.Clear() - accStart.Restart() - -/// Defines a tranche of a traversal of a stream (or the store as a whole) -type Range(start, sliceEnd : Position option, ?max : Position) = - member val Current = start with get, set - member __.TryNext(pos: Position) = - __.Current <- pos - __.IsCompleted - member __.IsCompleted = - match sliceEnd with - | Some send when __.Current.CommitPosition >= send.CommitPosition -> false - | _ -> true - member __.PositionAsRangePercentage = - match max with - | None -> Double.NaN - | Some max -> - match __.Current.CommitPosition, max.CommitPosition with - | p,m when p > m -> Double.NaN - | p,m -> float p / float m - -(* Logic for computation of chunk offsets; ES writes chunks whose index starts at a multiple of 256MB - to be able to address an arbitrary position as a percentage, we need to consider this aspect as only a valid Position can be supplied to the read call *) - -// @scarvel8: event_global_position = 256 x 1024 x 1024 x chunk_number + chunk_header_size (128) + event_position_offset_in_chunk -let chunk (pos: Position) = uint64 pos.CommitPosition >>> 28 -let posFromChunk (chunk: int) = - let chunkBase = int64 chunk * 1024L * 1024L * 256L - Position(chunkBase,0L) -let posFromChunkAfter (pos: EventStore.ClientAPI.Position) = - let nextChunk = 1 + int (chunk pos) - posFromChunk nextChunk -let posFromPercentage (pct,max : Position) = - let rawPos = Position(float max.CommitPosition * pct / 100. |> int64, 0L) - let chunk = int (chunk rawPos) in posFromChunk chunk // &&& 0xFFFFFFFFE0000000L // rawPos / 256L / 1024L / 1024L * 1024L * 1024L * 256L - -/// Read the current tail position; used to be able to compute and log progress of ingestion -let fetchMax (conn : IEventStoreConnection) = async { - let! lastItemBatch = conn.ReadAllEventsBackwardAsync(Position.End, 1, resolveLinkTos = false) |> Async.AwaitTaskCorrect - let max = lastItemBatch.FromPosition - Log.Information("EventStore Tail Position: @ {pos} ({chunks} chunks, ~{gb:n1}GB)", max.CommitPosition, chunk max, mb max.CommitPosition/1024.) - return max } -/// `fetchMax` wrapped in a retry loop; Sync process is entirely reliant on establishing the max so we have a crude retry loop -let establishMax (conn : IEventStoreConnection) = async { - let mutable max = None - while Option.isNone max do - try let! currentMax = fetchMax conn - max <- Some currentMax - with e -> - Log.Warning(e,"Could not establish max position") - do! Async.Sleep 5000 - return Option.get max } - -/// Walks a stream within the specified constraints; used to grab data when writing to a stream for which a prefix is missing -/// Can throw (in which case the caller is in charge of retrying, possibly with a smaller batch size) -let pullStream (conn : IEventStoreConnection, batchSize) (stream,pos,limit : int option) (postBatch : Buffer.StreamSpan -> Async) = - let rec fetchFrom pos limit = async { - let reqLen = match limit with Some limit -> min limit batchSize | None -> batchSize - let! currentSlice = conn.ReadStreamEventsForwardAsync(stream, pos, reqLen, resolveLinkTos=true) |> Async.AwaitTaskCorrect - let events = - [| for x in currentSlice.Events -> - let e = x.Event - Equinox.Codec.Core.EventData.Create(e.EventType, e.Data, e.Metadata, e.Timestamp) :> Equinox.Codec.IEvent |] - do! postBatch { stream = stream; span = { index = currentSlice.FromEventNumber; events = events } } - match limit with - | None when currentSlice.IsEndOfStream -> return () - | None -> return! fetchFrom currentSlice.NextEventNumber None - | Some limit when events.Length >= limit -> return () - | Some limit -> return! fetchFrom currentSlice.NextEventNumber (Some (limit - events.Length)) } - fetchFrom pos limit - -/// Walks the $all stream, yielding batches together with the associated Position info for the purposes of checkpointing -/// Can throw (in which case the caller is in charge of retrying, possibly with a smaller batch size) -type [] PullResult = Exn of exn: exn | Eof of Position | EndOfTranche -let pullAll (slicesStats : SliceStatsBuffer, overallStats : OverallStats) (conn : IEventStoreConnection, batchSize) - (range:Range, once) (tryMapEvent : ResolvedEvent -> StreamItem option) (postBatch : Position -> StreamItem[] -> Async) = - let sw = Stopwatch.StartNew() // we'll report the warmup/connect time on the first batch - let rec aux () = async { - let! currentSlice = conn.ReadAllEventsForwardAsync(range.Current, batchSize, resolveLinkTos = false) |> Async.AwaitTaskCorrect - sw.Stop() // Stop the clock after ChangeFeedProcessor hands off to us - let postSw = Stopwatch.StartNew() - let batchEvents, batchBytes = slicesStats.Ingest currentSlice in overallStats.Ingest(int64 batchEvents, batchBytes) - let batches = currentSlice.Events |> Seq.choose tryMapEvent |> Array.ofSeq - let streams = batches |> Seq.groupBy (fun b -> b.stream) |> Array.ofSeq - let usedStreams, usedCats = streams.Length, streams |> Seq.map fst |> Seq.distinct |> Seq.length - let! (cur,max) = postBatch currentSlice.NextPosition batches - Log.Information("Read {pos,10} {pct:p1} {ft:n3}s {mb:n1}MB {count,4} {categories,4}c {streams,4}s {events,4}e Post {pt:n3}s {cur}/{max}", - range.Current.CommitPosition, range.PositionAsRangePercentage, (let e = sw.Elapsed in e.TotalSeconds), mb batchBytes, - batchEvents, usedCats, usedStreams, batches.Length, (let e = postSw.Elapsed in e.TotalSeconds), cur, max) - if not (range.TryNext currentSlice.NextPosition && not once && not currentSlice.IsEndOfStream) then - if currentSlice.IsEndOfStream then return Eof currentSlice.NextPosition - else return EndOfTranche - else - sw.Restart() // restart the clock as we hand off back to the Reader - return! aux () } - async { - try return! aux () - with e -> return Exn e } - -/// Specification for work to be performed by a reader thread -[] -type Req = - /// Tail from a given start position, at intervals of the specified timespan (no waiting if catching up) - | Tail of seriesId: int * startPos: Position * max : Position * interval: TimeSpan * batchSize : int - /// Read a given segment of a stream (used when a stream needs to be rolled forward to lay down an event for which the preceding events are missing) - //| StreamPrefix of name: string * pos: int64 * len: int * batchSize: int - /// Read the entirity of a stream in blocks of the specified batchSize (TODO wire to commandline request) - //| Stream of name: string * batchSize: int - /// Read a specific chunk (min-max range), posting batches tagged with that chunk number - | Chunk of seriesId: int * range: Range * batchSize : int - -/// Data with context resulting from a reader thread -[] -type Res = - /// A batch read from a Chunk - | Batch of seriesId: int * pos: Position * items: StreamItem seq - /// Ingestion buffer requires an explicit end of chunk message before next chunk can commence processing - | EndOfChunk of seriesId: int - /// A Batch read from a Stream or StreamPrefix - //| StreamSpan of span: State.StreamSpan - -/// Holds work queue, together with stats relating to the amount and/or categories of data being traversed -/// Processing is driven by external callers running multiple concurrent invocations of `Process` -type Reader(conns : _ [], defaultBatchSize, minBatchSize, categorize, tryMapEvent, post : Res -> Async, tailInterval, dop, ?statsInterval) = - let work = System.Collections.Concurrent.ConcurrentQueue() - let sleepIntervalMs = 100 - let overallStats = OverallStats(?statsInterval=statsInterval) - let slicesStats = SliceStatsBuffer(categorize) - let mutable eofSpottedInChunk = 0 - - /// Invoked by pump to process a tranche of work; can have parallel invocations - let exec conn req = async { - let adjust batchSize = if batchSize > minBatchSize then batchSize - 128 else batchSize - //let postSpan = ReadResult.StreamSpan >> post >> Async.Ignore - match req with - //| StreamPrefix (name,pos,len,batchSize) -> - // use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - // Log.Warning("Reading stream prefix; pos {pos} len {len} batch size {bs}", pos, len, batchSize) - // try let! t,() = pullStream (conn, batchSize) (name, pos, Some len) postSpan |> Stopwatch.Time - // Log.Information("completed stream prefix in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) - // with e -> - // let bs = adjust batchSize - // Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - // __.AddStreamPrefix(name, pos, len, bs) - // return false - //| Stream (name,batchSize) -> - // use _ = Serilog.Context.LogContext.PushProperty("Tranche",name) - // Log.Warning("Reading stream; batch size {bs}", batchSize) - // try let! t,() = pullStream (conn, batchSize) (name,0L,None) postSpan |> Stopwatch.Time - // Log.Information("completed stream in {ms:n3}s", let e = t.Elapsed in e.TotalSeconds) - // with e -> - // let bs = adjust batchSize - // Log.Warning(e,"Could not read stream, retrying with batch size {bs}", bs) - // __.AddStream(name, bs) - // return false - | Chunk (series, range, batchSize) -> - let postBatch pos items = post (Res.Batch (series, pos, items)) - use _ = Serilog.Context.LogContext.PushProperty("Tranche", series) - Log.Information("Commencing tranche, batch size {bs}", batchSize) - let! t, res = pullAll (slicesStats, overallStats) (conn, batchSize) (range, false) tryMapEvent postBatch |> Stopwatch.Time - match res with - | PullResult.Eof pos -> - Log.Warning("completed tranche AND REACHED THE END in {ms:n3}m", let e = t.Elapsed in e.TotalMinutes) - overallStats.DumpIfIntervalExpired(true) - let! _ = post (Res.EndOfChunk series) in () - if 1 = Interlocked.Increment &eofSpottedInChunk then work.Enqueue <| Req.Tail (series+1, pos, pos, tailInterval, defaultBatchSize) - | PullResult.EndOfTranche -> - Log.Information("completed tranche in {ms:n1}m", let e = t.Elapsed in e.TotalMinutes) - let! _ = post (Res.EndOfChunk series) in () - | PullResult.Exn e -> - let abs = adjust batchSize - Log.Warning(e, "Could not read All, retrying with batch size {bs}", abs) - work.Enqueue <| Req.Chunk (series, range, abs) - | Tail (series, pos, max, interval, batchSize) -> - let postBatch pos items = post (Res.Batch (series, pos, items)) - use _ = Serilog.Context.LogContext.PushProperty("Tranche", "Tail") - let mutable count, batchSize, range = 0, batchSize, Range(pos, None, max) - let statsInterval = defaultArg statsInterval (TimeSpan.FromMinutes 5.) - let progressIntervalMs, tailIntervalMs = int64 statsInterval.TotalMilliseconds, int64 interval.TotalMilliseconds - let tailSw = Stopwatch.StartNew() - let awaitInterval = async { - match tailIntervalMs - tailSw.ElapsedMilliseconds with - | waitTimeMs when waitTimeMs > 0L -> do! Async.Sleep (int waitTimeMs) - | _ -> () - tailSw.Restart() } - let slicesStats, stats = SliceStatsBuffer(categorize), OverallStats() - let progressSw = Stopwatch.StartNew() - while true do - let currentPos = range.Current - if progressSw.ElapsedMilliseconds > progressIntervalMs then - Log.Information("Tailed {count} times @ {pos} (chunk {chunk})", - count, currentPos.CommitPosition, chunk currentPos) - progressSw.Restart() - count <- count + 1 - let! res = pullAll (slicesStats,stats) (conn,batchSize) (range,true) tryMapEvent postBatch - do! awaitInterval - match res with - | PullResult.EndOfTranche | PullResult.Eof _ -> () - | PullResult.Exn e -> - batchSize <- adjust batchSize - Log.Warning(e, "Tail $all failed, adjusting batch size to {bs}", batchSize) } - - let pump (initialSeriesId, initialPos) max = async { - let mutable robin = 0 - let selectConn () = - let connIndex = Interlocked.Increment(&robin) % conns.Length - conns.[connIndex] - let dop = new SemaphoreSlim(dop) - let forkRunRelease = - let r = new Random() - fun req -> async { // this is not called in parallel hence no need to lock `r` - let currentCount = dop.CurrentCount - // Jitter is most relevant when processing commences - any commencement of a chunk can trigger significant page faults on server - // which we want to attempt to limit the effects of - let jitterMs = match currentCount with 0 -> 200 | x -> r.Next(1000, 2000) - Log.Information("Waiting {jitter}ms to jitter reader stripes, {currentCount} further reader stripes awaiting start", jitterMs, currentCount) - do! Async.Sleep jitterMs - let! _ = Async.StartChild <| async { - try let conn = selectConn () - do! exec conn req - finally dop.Release() |> ignore } in () } - let mutable seriesId = initialSeriesId - let mutable remainder = - if conns.Length > 1 then - let nextPos = posFromChunkAfter initialPos - work.Enqueue <| Req.Chunk (initialSeriesId, new Range(initialPos, Some nextPos, max), defaultBatchSize) - Some nextPos - else - work.Enqueue <| Req.Tail (initialSeriesId, initialPos, max, tailInterval, defaultBatchSize) - None - let! ct = Async.CancellationToken - while not ct.IsCancellationRequested do - overallStats.DumpIfIntervalExpired() - let! _ = dop.Await() - match work.TryDequeue(), remainder with - | (true, task), _ -> - do! forkRunRelease task - | (false, _), Some nextChunk when eofSpottedInChunk = 0 -> - seriesId <- seriesId + 1 - let nextPos = posFromChunkAfter nextChunk - remainder <- Some nextPos - do! forkRunRelease <| Req.Chunk (seriesId, Range(nextChunk, Some nextPos, max), defaultBatchSize) - | (false, _), Some _ -> - dop.Release() |> ignore - Log.Warning("No further ingestion work to commence, transitioning to tailing...") - // TODO release connections, reduce DOP, implement stream readers - remainder <- None - | (false, _), None -> - dop.Release() |> ignore - do! Async.Sleep sleepIntervalMs } - member __.Start initialPos max = async { - let! _ = Async.StartChild (pump initialPos max) in () } - -type StartPos = Absolute of int64 | Chunk of int | Percentage of float | TailOrCheckpoint | StartOrCheckpoint - -type ReaderSpec = - { /// Identifier for this projection and it's state - groupName: string - /// Indicates user has specified that they wish to restart from the indicated position as opposed to resuming from the checkpoint position - forceRestart: bool - /// Start position from which forward reading is to commence // Assuming no stored position - start: StartPos - checkpointInterval: TimeSpan - /// Delay when reading yields an empty batch - tailInterval: TimeSpan - /// Specify initial phase where interleaved reading stripes a 256MB chunk apart attain a balance between good reading speed and not killing the server - gorge: int option - /// Maximum number of striped readers to permit when tailing; this dictates how many stream readers will be used to perform catchup work on streams - /// that are missing a prefix (e.g. due to not starting from the start of the $all stream, and/or deleting data from the destination store) - streamReaders: int // TODO - /// Initial batch size to use when commencing reading - batchSize: int - /// Smallest batch size to degrade to in the presence of failures - minBatchSize: int } - -type StartMode = Starting | Resuming | Overridding - -let run (log : Serilog.ILogger) (connect, spec, categorize, tryMapEvent) maxReadAhead (cosmosContexts, maxWriters) resolveCheckpointStream = async { - let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) - let conn = connect () - let! maxInParallel = Async.StartChild <| establishMax conn - let! initialCheckpointState = checkpoints.Read - let! maxPos = maxInParallel - let! startPos = async { - let mkPos x = EventStore.ClientAPI.Position(x, 0L) - let requestedStartPos = - match spec.start with - | Absolute p -> mkPos p - | Chunk c -> posFromChunk c - | Percentage pct -> posFromPercentage (pct, maxPos) - | TailOrCheckpoint -> maxPos - | StartOrCheckpoint -> EventStore.ClientAPI.Position.Start - let! startMode, startPos, checkpointFreq = async { - match initialCheckpointState, requestedStartPos with - | Checkpoint.Folds.NotStarted, r -> - if spec.forceRestart then invalidOp "Cannot specify --forceRestart when no progress yet committed" - do! checkpoints.Start(spec.checkpointInterval, r.CommitPosition) - return Starting, r, spec.checkpointInterval - | Checkpoint.Folds.Running s, _ when not spec.forceRestart -> - return Resuming, mkPos s.state.pos, TimeSpan.FromSeconds(float s.config.checkpointFreqS) - | Checkpoint.Folds.Running _, r -> - do! checkpoints.Override(spec.checkpointInterval, r.CommitPosition) - return Overridding, r, spec.checkpointInterval - } - log.Information("Sync {mode} {groupName} @ {pos} (chunk {chunk}, {pct:p1}) checkpointing every {checkpointFreq:n1}m", - startMode, spec.groupName, startPos.CommitPosition, chunk startPos, float startPos.CommitPosition/float maxPos.CommitPosition, - checkpointFreq.TotalMinutes) - return startPos } - let cosmosIngester = CosmosSink.Scheduler.Start (log.ForContext("Tranche","Sync"), cosmosContexts, maxWriters, categorize, (TimeSpan.FromMinutes 1., TimeSpan.FromMinutes 2.)) - let initialSeriesId, conns, dop = - log.Information("Tailing every {intervalS:n1}s TODO with {streamReaders} stream catchup-readers", spec.tailInterval.TotalSeconds, spec.streamReaders) - match spec.gorge with - | Some factor -> - log.Information("Commencing Gorging with {stripes} $all reader stripes covering a 256MB chunk each", factor) - let extraConns = Seq.init (factor-1) (ignore >> connect) - let conns = [| yield conn; yield! extraConns |] - chunk startPos |> int, conns, (max (conns.Length) (spec.streamReaders+1)) - | None -> - 0, [|conn|], spec.streamReaders+1 - let trancheIngester = Ingestion.Engine.Start(log.ForContext("Tranche","EventStore"), cosmosIngester, maxReadAhead, maxReadAhead, initialSeriesId, categorize, TimeSpan.FromMinutes 1.) - let post = function - | Res.EndOfChunk seriesId -> trancheIngester.Submit <| Ingestion.EndOfSeries seriesId - | Res.Batch (seriesId, pos, xs) -> - let cp = pos.CommitPosition - trancheIngester.Submit <| Ingestion.Message.Batch(seriesId, cp, checkpoints.Commit cp, xs) - let reader = Reader(conns, spec.batchSize, spec.minBatchSize, categorize, tryMapEvent, post, spec.tailInterval, dop) - do! reader.Start (initialSeriesId,startPos) maxPos - do! Async.AwaitKeyboardInterrupt() } \ No newline at end of file diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 7aecf1f6e..595ced46a 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -19,207 +19,231 @@ module CmdParser = [] type Parameters = | [] ConsumerGroupName of string - | [] FromTail | [] MaxPendingBatches of int | [] MaxWriters of int - | [] MaxConnections of int - | [] MaxSubmit of int + | [] MaxConnections of int + | [] MaxSubmit of int + | [] LocalSeq | [] Verbose -#if cosmos - | [] MaxDocuments of int - | [] LagFreqM of float - | [] ChangeFeedVerbose - | [] LeaseCollectionSource of string - | [] LeaseCollectionDestination of string - | [] LagFreqS of float -#else | [] VerboseConsole - | [] ForceRestart - | [] BatchSize of int - | [] MinBatchSize of int - | [] Position of int64 - | [] Chunk of int - | [] Percent of float - | [] Gorge of int - | [] StreamReaders of int - | [] Tail of intervalS: float -#endif - | [] Source of ParseResults + + | [] CategoryBlacklist of string + | [] CategoryWhitelist of string + + | [] SrcEs of ParseResults + | [] SrcCosmos of ParseResults interface IArgParserTemplate with member a.Usage = match a with | ConsumerGroupName _ -> "Projector consumer group name." + | MaxPendingBatches _ -> "maximum number of batches to let processing get ahead of completion. Default: 16" | MaxWriters _ -> "maximum number of concurrent writes to target permitted. Default: 512" | MaxConnections _ -> "size of Sink connection pool to maintain. Default: 1" - | MaxPendingBatches _ -> "maximum number of batches to let processing get ahead of completion. Default: 16" | MaxSubmit _ -> "maximum number of batches to submit concurrently. Default: 8" - | MaxCosmosConnections _ -> "size of CosmosDb connection pool to maintain. Default: 1" -#if cosmos - | ChangeFeedVerbose -> "request Verbose Logging from ChangeFeedProcessor. Default: off" - | FromTail _ -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." - | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" - | LagFreqM _ -> "frequency (in minutes) to dump lag stats. Default: off" - | Source _ -> "CosmosDb input parameters." -#else - | MaxPendingBatches _ -> "Maximum number of batches to let processing get ahead of completion. Default: 2048" + + | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" + | Verbose -> "request Verbose Logging. Default: off" | VerboseConsole -> "request Verbose Console Logging. Default: off" - | FromTail -> "Start the processing from the Tail" - | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." - | BatchSize _ -> "maximum item count to request from feed. Default: 4096" - | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | Position _ -> "EventStore $all Stream Position to commence from" - | Chunk _ -> "EventStore $all Chunk to commence from" - | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - | Gorge _ -> "Request Parallel readers phase during initial catchup, running one chunk (256MB) apart. Default: off" - | StreamReaders _ -> "number of concurrent readers that will fetch a missing stream when in tailing mode. Default: 1. TODO: IMPLEMENT!" - | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" - | Source _ -> "EventStore input parameters." -#endif + + | CategoryBlacklist _ -> "category whitelist" + | CategoryWhitelist _ -> "category blacklist" + + | SrcCosmos _ -> "Cosmos input parameters." + | SrcEs _ -> "EventStore input parameters." and Arguments(a : ParseResults) = - member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None - member __.Verbose = a.Contains Verbose - member __.MaxWriters = a.GetResult(MaxWriters,1024) - member __.ConnectionPoolSize = a.GetResult(MaxCosmosConnections,1) -#if cosmos - member __.MaxReadAhead = a.GetResult(MaxPendingBatches,32) - member __.MaxProcessing = a.GetResult(MaxProcessing,16) - member __.ChangeFeedVerbose = a.Contains ChangeFeedVerbose - member __.LeaseId = a.GetResult ConsumerGroupName - member __.MaxDocuments = a.TryGetResult MaxDocuments - member __.LagFrequency = a.TryGetResult LagFreqM |> Option.map TimeSpan.FromMinutes -#else + member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) + member __.MaxWriters = a.GetResult(MaxWriters,1024) + member __.MaxConnections = a.GetResult(MaxConnections,1) + member __.MaybeSeqEndpoint = if a.Contains LocalSeq then Some "http://localhost:5341" else None + member __.MaxSubmit = a.GetResult(MaxSubmit,8) + + member __.Verbose = a.Contains Parameters.Verbose member __.VerboseConsole = a.Contains VerboseConsole - member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.ConsoleMinLevel = if __.VerboseConsole then Serilog.Events.LogEventLevel.Information else Serilog.Events.LogEventLevel.Warning - member __.StartingBatchSize = a.GetResult(BatchSize,4096) - member __.MinBatchSize = a.GetResult(MinBatchSize,512) - member __.Gorge = a.TryGetResult Gorge - member __.StreamReaders = a.GetResult(StreamReaders,1) - member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds - member __.CheckpointInterval = TimeSpan.FromHours 1. - member __.ForceRestart = a.Contains ForceRestart -#endif + member val Source : Choice = + match a.TryGetSubCommand() with + | Some (SrcCosmos cosmos) -> Choice1Of2 (CosmosSourceArguments cosmos) + | Some (SrcEs es) -> Choice2Of2 (EsSourceArguments es) + | _ -> raise (InvalidArguments "Must specify one of cosmos or es for Src") - member val Source : SourceArguments = SourceArguments(a.GetResult Source) - member __.Sink : Choice = __.Source.Sink -#if cosmos - member x.BuildChangeFeedParams() = - let disco, db = - match a.TryGetResult LeaseCollectionSource, a.TryGetResult LeaseCollectionDestination with - | None, None -> x.Source.Discovery, { database = x.Source.Database; collection = x.Source.Collection + "-aux" } - | Some sc, None -> x.Source.Discovery, { database = x.Source.Database; collection = sc } - | None, Some dc -> x.Destination.Discovery, { database = x.Destination.Database; collection = dc } - | Some _, Some _ -> raise (InvalidArguments "LeaseCollectionSource and LeaseCollectionDestination are mutually exclusive - can only store in one database") - Log.Information("Max read backlog: {maxPending}, of which up to {maxProcessing} processing", x.MaxPendingBatches, x.MaxProcessing) - Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} with maximum document count limited to {maxDocuments}", x.LeaseId, db.database, db.collection, x.MaxDocuments) - if a.Contains FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") - x.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) - disco, db, x.LeaseId, a.Contains FromTail, x.MaxDocuments, x.LagFrequency -#else - member x.BuildFeedParams() : EventStoreSource.ReaderSpec = - let startPos = - match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains FromTail with - | Some p, _, _, _ -> EventStoreSource.Absolute p - | _, Some c, _, _ -> EventStoreSource.StartPos.Chunk c - | _, _, Some p, _ -> EventStoreSource.Percentage p - | None, None, None, true -> EventStoreSource.StartPos.TailOrCheckpoint - | None, None, None, _ -> EventStoreSource.StartPos.StartOrCheckpoint - Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", - x.ConsumerGroupName, startPos, x.ForceRestart, x.Destination.Database, x.Destination.Collection) - Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}], reading up to {maxPendingBatches} uncommitted batches ahead", - x.MinBatchSize, x.StartingBatchSize, x.MaxPendingBatches) - { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = x.CheckpointInterval; tailInterval = x.TailInterval; forceRestart = x.ForceRestart - batchSize = x.StartingBatchSize; minBatchSize = x.MinBatchSize; gorge = x.Gorge; streamReaders = x.StreamReaders } -#endif - and [] SourceParameters = -#if cosmos - | [] SourceConnectionMode of Equinox.Cosmos.ConnectionMode - | [] SourceTimeout of float - | [] SourceRetries of int - | [] SourceRetriesWaitTime of int - | [] SourceConnection of string - | [] SourceDatabase of string - | [] SourceCollection of string -#else - | [] VerboseStore - | [] SourceTimeout of float - | [] SourceRetries of int - | [] Host of string + member __.StatsInterval = TimeSpan.FromMinutes 1. + member __.StatesInterval = TimeSpan.FromMinutes 5. + member __.CategoryFilterFunction : string -> bool = + match a.GetResults CategoryBlacklist, a.GetResults CategoryWhitelist with + | [], [] -> Log.Information("Not filtering by category"); fun _ -> true + | bad, [] -> let black = Set.ofList bad in Log.Warning("Excluding categories: {cats}", black); fun x -> not (black.Contains x) + | [], good -> let white = Set.ofList good in Log.Warning("Only copying categories: {cats}", white); fun x -> white.Contains x + | _, _ -> raise (InvalidArguments "BlackList and Whitelist are mutually exclusive; inclusions and exclusions cannot be mixed") + + member __.Sink : Choice = + match __.Source with + | Choice1Of2 cosmos -> cosmos.Sink + | Choice2Of2 es -> Choice1Of2 es.Sink + member x.SourceParams() : Choice<_,_*ReaderSpec> = + match x.Source with + | Choice1Of2 srcC -> + let disco, db = + match srcC.Sink with + | Choice1Of2 dstC -> + match srcC.LeaseCollection, dstC.LeaseCollection with + | None, None -> srcC.Discovery, { database = srcC.Database; collection = srcC.Collection + "-aux" } + | Some sc, None -> srcC.Discovery, { database = srcC.Database; collection = sc } + | None, Some dc -> dstC.Discovery, { database = dstC.Database; collection = dc } + | Some _, Some _ -> raise (InvalidArguments "LeaseCollectionSource and LeaseCollectionDestination are mutually exclusive - can only store in one database") + | Choice2Of2 _dstE -> + let lc = match srcC.LeaseCollection with Some sc -> sc | None -> srcC.Collection + "-aux" + srcC.Discovery, { database = srcC.Database; collection = lc } + Log.Information("Max read backlog: {maxPending}", x.MaxPendingBatches) + Log.Information("Processing Lease {leaseId} in Database {db} Collection {coll} with maximum document count limited to {maxDocuments}", + x.ConsumerGroupName, db.database, db.collection, srcC.MaxDocuments) + if srcC.FromTail then Log.Warning("(If new projector group) Skipping projection of all existing events.") + srcC.LagFrequency |> Option.iter (fun s -> Log.Information("Dumping lag stats at {lagS:n0}s intervals", s.TotalSeconds)) + Choice1Of2 (srcC, disco, db, x.ConsumerGroupName, srcC.FromTail, srcC.MaxDocuments, srcC.LagFrequency) + | Choice2Of2 srcE -> + let startPos = srcE.StartPos + Log.Information("Processing Consumer Group {groupName} from {startPos} (force: {forceRestart}) in Database {db} Collection {coll}", + x.ConsumerGroupName, startPos, srcE.ForceRestart, srcE.Sink.Database, srcE.Sink.Collection) + Log.Information("Ingesting in batches of [{minBatchSize}..{batchSize}], reading up to {maxPendingBatches} uncommitted batches ahead", + srcE.MinBatchSize, srcE.StartingBatchSize, x.MaxPendingBatches) + Choice2Of2 (srcE, + { groupName = x.ConsumerGroupName; start = startPos; checkpointInterval = srcE.CheckpointInterval; tailInterval = srcE.TailInterval + forceRestart = srcE.ForceRestart + batchSize = srcE.StartingBatchSize; minBatchSize = srcE.MinBatchSize; gorge = srcE.Gorge; streamReaders = srcE.StreamReaders }) + and [] CosmosSourceParameters = + | [] FromTail + | [] MaxDocuments of int + | [] LagFreqM of float + | [] LeaseCollection of string + + | [] Connection of string + | [] ConnectionMode of Equinox.Cosmos.ConnectionMode + | [] Database of string + | [] Collection of string + | [] Timeout of float + | [] Retries of int + | [] RetriesWaitTime of int + + | [] DstEs of ParseResults + | [] DstCosmos of ParseResults + interface IArgParserTemplate with + member a.Usage = + match a with + | FromTail -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." + | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" + | LagFreqM _ -> "frequency (in minutes) to dump lag stats. Default: off" + | LeaseCollection _ -> "specify Collection Name for Leases collection (default: `sourcecollection` + `-aux`)." + + | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION)." + | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." + | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE)." + | Collection _ -> "specify a collection name within `SourceDatabase`." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." + | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" + + | DstEs _ -> "EventStore Sink parameters." + | DstCosmos _ -> "CosmosDb Sink parameters." + and CosmosSourceArguments(a : ParseResults) = + member __.FromTail = a.Contains CosmosSourceParameters.FromTail + member __.MaxDocuments = a.TryGetResult MaxDocuments + member __.LagFrequency = a.TryGetResult LagFreqM |> Option.map TimeSpan.FromMinutes + member __.LeaseCollection = a.TryGetResult CosmosSourceParameters.LeaseCollection + + member __.Connection = match a.TryGetResult CosmosSourceParameters.Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" + member __.Discovery = Discovery.FromConnectionString __.Connection + member __.Mode = a.GetResult(CosmosSourceParameters.ConnectionMode, Equinox.Cosmos.ConnectionMode.DirectTcp) + member __.Database = match a.TryGetResult CosmosSourceParameters.Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" + member __.Collection = a.GetResult CosmosSourceParameters.Collection + member __.Timeout = a.GetResult(CosmosSourceParameters.Timeout, 5.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(CosmosSourceParameters.Retries, 1) + member __.MaxRetryWaitTime = a.GetResult(CosmosSourceParameters.RetriesWaitTime, 5) + member x.BuildConnectionDetails() = + let (Discovery.UriAndKey (endpointUri,_)) as discovery = x.Discovery + Log.Information("Source CosmosDb {mode} {endpointUri} Database {database} Collection {collection}", + x.Mode, endpointUri, x.Database, x.Collection) + Log.Information("Source CosmosDb timeout {timeout}s; Throttling retries {retries}, max wait {maxRetryWaitTime}s", + (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) + let c = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) + discovery, { database = x.Database; collection = x.Collection }, c.ConnectionPolicy + + member val Sink = + match a.TryGetSubCommand() with + | Some (DstCosmos cosmos) -> Choice1Of2 (CosmosSinkArguments cosmos) + | Some (DstEs es) -> Choice2Of2 (EsSinkArguments es) + | _ -> raise (InvalidArguments "Must specify one of cosmos or es for Sink") + and [] EsSourceParameters = + | [] FromTail + | [] Gorge of int + | [] StreamReaders of int + | [] Tail of intervalS: float + | [] ForceRestart + | [] BatchSize of int + | [] MinBatchSize of int + | [] Position of int64 + | [] Chunk of int + | [] Percent of float + + | [] Verbose + | [] Timeout of float + | [] Retries of int + | [] HeartbeatTimeout of float + | [] Host of string | [] Port of int | [] Username of string | [] Password of string - | [] HeartbeatTimeout of float -#endif - | [] CategoryBlacklist of string - | [] CategoryWhitelist of string + | [] Es of ParseResults | [] Cosmos of ParseResults interface IArgParserTemplate with member a.Usage = match a with -#if cosmos - | SourceConnection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION)." - | SourceDatabase _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE)." - | SourceCollection _ -> "specify a collection name within `SourceDatabase`." - | SourceTimeout _ -> "specify operation timeout in seconds (default: 5)." - | SourceRetries _ -> "specify operation retries (default: 1)." - | SourceRetriesWaitTime _ ->"specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" - | SourceConnectionMode _ -> "override the connection mode (default: DirectTcp)." -#else - | VerboseStore -> "Include low level Store logging." - | SourceTimeout _ -> "specify operation timeout in seconds (default: 20)." - | SourceRetries _ -> "specify operation retries (default: 3)." + | FromTail -> "Start the processing from the Tail" + | Gorge _ -> "Request Parallel readers phase during initial catchup, running one chunk (256MB) apart. Default: off" + | StreamReaders _ -> "number of concurrent readers that will fetch a missing stream when in tailing mode. Default: 1. TODO: IMPLEMENT!" + | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" + | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." + | BatchSize _ -> "maximum item count to request from feed. Default: 4096" + | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" + | Position _ -> "EventStore $all Stream Position to commence from" + | Chunk _ -> "EventStore $all Chunk to commence from" + | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" + + | Verbose -> "Include low level Store logging." | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." + | Timeout _ -> "specify operation timeout in seconds (default: 20)." + | Retries _ -> "specify operation retries (default: 3)." | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." -#endif - | CategoryBlacklist _ -> "category whitelist" - | CategoryWhitelist _ -> "category blacklist" - | Es _ -> "EventStore Sink parameters." + | Cosmos _ -> "CosmosDb Sink parameters." - and SourceArguments(a : ParseResults) = - member val Sink = - match a.TryGetSubCommand() with - | Some (Cosmos cosmos) -> Choice1Of2 (CosmosSinkArguments cosmos) - | Some (Es es) -> Choice2Of2 (EsSinkArguments es) - | _ -> raise (InvalidArguments "Must specify one of cosmos or es for Sink") - member __.CategoryFilterFunction : string -> bool = - match a.GetResults CategoryBlacklist, a.GetResults CategoryWhitelist with - | [], [] -> Log.Information("Not filtering by category"); fun _ -> true - | bad, [] -> let black = Set.ofList bad in Log.Warning("Excluding categories: {cats}", black); fun x -> not (black.Contains x) - | [], good -> let white = Set.ofList good in Log.Warning("Only copying categories: {cats}", white); fun x -> white.Contains x - | _, _ -> raise (InvalidArguments "BlackList and Whitelist are mutually exclusive; inclusions and exclusions cannot be mixed") -#if cosmos - member __.Mode = a.GetResult(SourceConnectionMode, Equinox.Cosmos.ConnectionMode.DirectTcp) - member __.Discovery = Discovery.FromConnectionString __.Connection - member __.Connection = match a.TryGetResult SourceConnection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" - member __.Database = match a.TryGetResult SourceDatabase with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" - member __.Collection = a.GetResult SourceCollection + | Es _ -> "EventStore Sink parameters." + and EsSourceArguments(a : ParseResults) = + member __.Gorge = a.TryGetResult Gorge + member __.StreamReaders = a.GetResult(StreamReaders,1) + member __.TailInterval = a.GetResult(Tail,1.) |> TimeSpan.FromSeconds + member __.ForceRestart = a.Contains ForceRestart + member __.StartingBatchSize = a.GetResult(BatchSize,4096) + member __.MinBatchSize = a.GetResult(MinBatchSize,512) + member __.StartPos = + match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains FromTail with + | Some p, _, _, _ -> Absolute p + | _, Some c, _, _ -> StartPos.Chunk c + | _, _, Some p, _ -> Percentage p + | None, None, None, true -> StartPos.TailOrCheckpoint + | None, None, None, _ -> StartPos.StartOrCheckpoint - member __.Timeout = a.GetResult(SourceTimeout, 5.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(SourceRetries, 1) - member __.MaxRetryWaitTime = a.GetResult(SourceRetriesWaitTime, 5) - member x.BuildConnectionDetails() = - let (Discovery.UriAndKey (endpointUri,_)) as discovery = x.Discovery - Log.Information("Source CosmosDb {mode} {endpointUri} Database {database} Collection {collection}", - x.Mode, endpointUri, x.Database, x.Collection) - Log.Information("Source CosmosDb timeout {timeout}s; Throttling retries {retries}, max wait {maxRetryWaitTime}s", - (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) - let c = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) - discovery, { database = x.Database; collection = x.Collection }, c.ConnectionPolicy, x.CategoryFilterFunction -#else - member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" - member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int + member __.Host = match a.TryGetResult EsSourceParameters.Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" + member __.Port = match a.TryGetResult EsSourceParameters.Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host - member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" - member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" - member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds - member __.Timeout = a.GetResult(SourceTimeout,20.) |> TimeSpan.FromSeconds - member __.Retries = a.GetResult(SourceRetries,3) + member __.User = match a.TryGetResult EsSourceParameters.Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" + member __.Password = match a.TryGetResult EsSourceParameters.Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" + member __.Timeout = a.GetResult(EsSourceParameters.Timeout,20.) |> TimeSpan.FromSeconds + member __.Retries = a.GetResult(EsSourceParameters.Retries,3) + member __.Heartbeat = a.GetResult(EsSourceParameters.HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds member __.Connect(log: ILogger, storeLog: ILogger, connectionStrategy) = let s (x : TimeSpan) = x.TotalSeconds log.Information("EventStore {host} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", __.Host, s __.Heartbeat, s __.Timeout, __.Retries) @@ -227,36 +251,42 @@ module CmdParser = let tags=["M", Environment.MachineName; "I", Guid.NewGuid() |> string] GesConnector(__.User, __.Password, __.Timeout, __.Retries, log=log, heartbeatTimeout=__.Heartbeat, tags=tags) .Establish("SyncTemplate", __.Discovery, connectionStrategy) |> Async.RunSynchronously -#endif + member __.CheckpointInterval = TimeSpan.FromHours 1. + + member val Sink = + match a.TryGetSubCommand() with + | Some (Cosmos cosmos) -> CosmosSinkArguments cosmos + | _ -> raise (InvalidArguments "Must specify cosmos for Sink if source is `es`") and [] CosmosSinkParameters = | [] Connection of string + | [] ConnectionMode of Equinox.Cosmos.ConnectionMode | [] Database of string | [] Collection of string + | [] LeaseCollection of string | [] Timeout of float | [] Retries of int | [] RetriesWaitTime of int - | [] ConnectionMode of Equinox.Cosmos.ConnectionMode interface IArgParserTemplate with member a.Usage = match a with | Connection _ -> "specify a connection string for a Cosmos account (default: envvar:EQUINOX_COSMOS_CONNECTION)." | Database _ -> "specify a database name for Cosmos account (default: envvar:EQUINOX_COSMOS_DATABASE)." | Collection _ -> "specify a collection name for Cosmos account (default: envvar:EQUINOX_COSMOS_COLLECTION)." + | LeaseCollection _ -> "specify Collection Name for Leases collection (default: `sourcecollection` + `-aux`)." | Timeout _ -> "specify operation timeout in seconds (default: 5)." | Retries _ -> "specify operation retries (default: 0)." | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." and CosmosSinkArguments(a : ParseResults) = + member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" member __.Mode = a.GetResult(ConnectionMode, Equinox.Cosmos.ConnectionMode.DirectTcp) member __.Discovery = Discovery.FromConnectionString __.Connection - member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" member __.Database = match a.TryGetResult Database with Some x -> x | None -> envBackstop "Database" "EQUINOX_COSMOS_DATABASE" member __.Collection = match a.TryGetResult Collection with Some x -> x | None -> envBackstop "Collection" "EQUINOX_COSMOS_COLLECTION" - + member __.LeaseCollection = a.TryGetResult LeaseCollection member __.Timeout = a.GetResult(CosmosSinkParameters.Timeout, 5.) |> TimeSpan.FromSeconds member __.Retries = a.GetResult(CosmosSinkParameters.Retries, 0) member __.MaxRetryWaitTime = a.GetResult(RetriesWaitTime, 5) - /// Connect with the provided parameters and/or environment variables member x.Connect /// Connection/Client identifier for logging purposes @@ -269,34 +299,34 @@ module CmdParser = let c = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) c.Connect(sprintf "App=%s Conn=%d" appName connIndex, discovery) and [] EsSinkParameters = - | [] VerboseStore - | [] Timeout of float - | [] Retries of int - | [] Host of string + | [] Verbose + | [] Host of string | [] Port of int | [] Username of string | [] Password of string - | [] HeartbeatTimeout of float + | [] Timeout of float + | [] Retries of int + | [] HeartbeatTimeout of float interface IArgParserTemplate with member a.Usage = match a with - | VerboseStore -> "Include low level Store logging." - | Timeout _ -> "specify operation timeout in seconds (default: 20)." - | Retries _ -> "specify operation retries (default: 3)." + | Verbose -> "Include low level Store logging." | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." | Password _ -> "specify a password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." + | Timeout _ -> "specify operation timeout in seconds (default: 20)." + | Retries _ -> "specify operation retries (default: 3)." | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." and EsSinkArguments(a : ParseResults) = + member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" member __.Port = match a.TryGetResult Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int - member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host member __.User = match a.TryGetResult Username with Some x -> x | None -> envBackstop "Username" "EQUINOX_ES_USERNAME" member __.Password = match a.TryGetResult Password with Some x -> x | None -> envBackstop "Password" "EQUINOX_ES_PASSWORD" - member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds - member __.Timeout = a.GetResult(Timeout,20.) |> TimeSpan.FromSeconds member __.Retries = a.GetResult(Retries,3) + member __.Timeout = a.GetResult(Timeout,20.) |> TimeSpan.FromSeconds + member __.Heartbeat = a.GetResult(HeartbeatTimeout,1.5) |> TimeSpan.FromSeconds member __.Connect(log: ILogger, storeLog: ILogger, connectionStrategy, appName, connIndex) = let s (x : TimeSpan) = x.TotalSeconds log.Information("EventStore {host} Connection {connId} heartbeat: {heartbeat}s Timeout: {timeout}s Retries {retries}", @@ -341,13 +371,12 @@ module Logging = let isWriterA = Filters.Matching.FromSource().Invoke let isWriterB = Filters.Matching.FromSource().Invoke let isCp = Filters.Matching.FromSource().Invoke - let isWriter = Filters.Matching.FromSource().Invoke let isCfp429a = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.LeaseManagement.DocumentServiceLeaseUpdater").Invoke let isCfp429b = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.LeaseRenewer").Invoke let isCfp429c = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.PartitionManagement.PartitionLoadBalancer").Invoke let isCfp429d = Filters.Matching.FromSource("Microsoft.Azure.Documents.ChangeFeedProcessor.FeedProcessing.PartitionProcessor").Invoke let isCfp x = isCfp429a x || isCfp429b x || isCfp429c x || isCfp429d x - (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isWriterA x || isWriterB x || isCfp x)) + (if verboseConsole then l else l.Filter.ByExcluding(fun x -> isEqx x || isWriterA x || isWriterB x || isCp x || isCfp x)) .WriteTo.Console(theme=Sinks.SystemConsole.Themes.AnsiConsoleTheme.Code, outputTemplate=t) |> ignore) |> ignore c.WriteTo.Async(bufferSize=65536, blockWhenFull=true, configure=Action<_> configure) @@ -406,35 +435,49 @@ let transformOrFilter categorize catFilter (changeFeedDocument: Microsoft.Azure. yield { stream = s; index = i; event = Propulsion.Streams.Internal.EventData.Create(e.EventType, e.Data, e.Meta, e.Timestamp) } } //#endif -[] -let main argv = - try let args = CmdParser.parse argv -#if cosmos - let log,storeLog = Logging.initialize args.Verbose args.ChangeFeedVerbose args.MaybeSeqEndpoint -#else - let log,storeLog = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint -#endif - let destinations = Seq.init args.CosmosConnectionPool (fun i -> args.Destination.Connect (sprintf "%s Pool %d" "SyncTemplate" i)) |> Async.Parallel |> Async.RunSynchronously - let colls = CosmosCollections(args.Destination.Database, args.Destination.Collection) - let targets = destinations |> Array.mapi (fun i x -> Equinox.Cosmos.Core.CosmosContext(x, colls, storeLog.ForContext("PoolId", i))) - let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] -#if cosmos - let discovery, source, connectionPolicy, catFilter = args.Source.BuildConnectionDetails() - let auxDiscovery, aux, leaseId, startFromHere, maxDocuments, lagFrequency = args.BuildChangeFeedParams() +let start (args : CmdParser.Arguments) = + let log,storeLog = Logging.initialize args.Verbose args.VerboseConsole args.MaybeSeqEndpoint + let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] + let maybeDstCosmos, sink = + match args.Sink with + | Choice1Of2 cosmos -> + let colls = CosmosCollections(cosmos.Database, cosmos.Collection) + let connect connIndex = async { + let! c = cosmos.Connect "SyncTemplate" connIndex + let lfc = storeLog.ForContext("ConnId", connIndex) + return c, Equinox.Cosmos.Core.CosmosContext(c, colls, lfc) } + let all = Array.init args.MaxConnections connect |> Async.Parallel |> Async.RunSynchronously + let mainConn, targets = CosmosGateway(fst all.[0], CosmosBatchingPolicy()), Array.map snd all + Some (mainConn,colls), CosmosSink.Start(log, args.MaxPendingBatches, targets, args.MaxWriters, categorize, args.StatsInterval, args.StatesInterval, args.MaxSubmit) + | Choice2Of2 es -> + let connect connIndex = async { + let lfc = storeLog.ForContext("ConnId", connIndex) + let! c = es.Connect(log, lfc, ConnectionStrategy.ClusterSingle NodePreference.Master, "SyncTemplate", connIndex) + return GesGateway(c, GesBatchingPolicy(Int32.MaxValue)) } + let targets = Array.init args.MaxConnections (string >> connect) |> Async.Parallel |> Async.RunSynchronously + None, EventStoreSink.Start(log, storeLog, args.MaxPendingBatches, targets, args.MaxWriters, categorize, args.StatsInterval, args.StatesInterval, args.MaxSubmit) + let catFilter = args.CategoryFilterFunction + match args.SourceParams() with + | Choice1Of2 (srcC, auxDiscovery, aux, leaseId, startFromHere, maxDocuments, lagFrequency) -> + let discovery, source, connectionPolicy = srcC.BuildConnectionDetails() #if marveleqx - let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformV0 categorize catFilter) + let createObserver () = CosmosSource.CreateObserver(log, sink.StartIngester, Seq.collect (transformV0 categorize catFilter)) #else - let createSyncHandler = CosmosSource.createRangeSyncHandler log (CosmosSource.transformOrFilter categorize catFilter) + let createObserver () = CosmosSource.CreateObserver(log, sink.StartIngester, Seq.collect (transformOrFilter categorize args.CategoryFilterFunction)) #endif - CosmosSource.run log (discovery, source) (auxDiscovery, aux) connectionPolicy - (leaseId, startFromHere, maxDocuments, lagFrequency) - (targets, args.MaxWriters) - categorize - (createSyncHandler (args.MaxPendingBatches,args.MaxProcessing) categorize) -#else + let runPipeline = + CosmosSource.Run(log, discovery, connectionPolicy, source, + aux, leaseId, startFromHere, createObserver, + ?maxDocuments=maxDocuments, ?lagReportFreq=lagFrequency, auxDiscovery=auxDiscovery) + sink,runPipeline + | Choice2Of2 (srcE,spec) -> + match maybeDstCosmos with + | None -> failwith "ES->ES checkpointing E_NOTIMPL" + | Some (mainConn,colls) -> + + let connect () = let c = srcE.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection let resolveCheckpointStream = - let gateway = CosmosGateway(destinations.[0], CosmosBatchingPolicy()) - let store = Equinox.Cosmos.CosmosStore(gateway, colls) + let store = Equinox.Cosmos.CosmosStore(mainConn, colls) let settings = Newtonsoft.Json.JsonSerializerSettings() let codec = Equinox.Codec.NewtonsoftJson.Json.Create settings let caching = @@ -442,9 +485,7 @@ let main argv = Equinox.Cosmos.CachingStrategy.SlidingWindow (c, TimeSpan.FromMinutes 20.) let access = Equinox.Cosmos.AccessStrategy.Snapshot (Checkpoint.Folds.isOrigin, Checkpoint.Folds.unfold) Equinox.Cosmos.CosmosResolver(store, codec, Checkpoint.Folds.fold, Checkpoint.Folds.initial, caching, access).Resolve - let connect () = let c = args.Source.Connect(log, log, ConnectionStrategy.ClusterSingle NodePreference.PreferSlave) in c.ReadConnection - let catFilter = args.Source.CategoryFilterFunction - let spec = args.BuildFeedParams() + let checkpoints = Checkpoint.CheckpointSeries(spec.groupName, log.ForContext(), resolveCheckpointStream) let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = match x.Event with | e when not e.IsJson @@ -460,11 +501,20 @@ let main argv = || e.EventStreamId = "SkuFileUpload-778f1efeab214f5bab2860d1f802ef24" || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None - | e -> e |> EventStoreSource.toIngestionItem |> Some - EventStoreSource.run log (connect, spec, categorize, tryMapEvent catFilter) args.MaxPendingBatches (targets, args.MaxWriters) resolveCheckpointStream -#endif - |> Async.RunSynchronously - 0 - with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 - | CmdParser.InvalidArguments msg -> eprintfn "%s" msg; 1 - | e -> eprintfn "%s" e.Message; 1 \ No newline at end of file + | e -> e |> Propulsion.EventStore.Reader.toIngestionItem |> Some + let runPipeline = + EventStoreSource.Run( + log, sink, checkpoints, connect, spec, categorize, tryMapEvent catFilter, + args.MaxPendingBatches, args.StatsInterval) + sink, runPipeline + +[] +let main argv = + try try let sink, runPipeline = CmdParser.parse argv |> start + runPipeline |> Async.Start + sink.AwaitCompletion() |> Async.RunSynchronously + if sink.RanToCompletion then 0 else 2 + with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 + | CmdParser.InvalidArguments msg -> eprintfn "%s" msg; 1 + | e -> eprintfn "%s" e.Message; 1 + finally Log.CloseAndFlush() \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 2d4793476..cd0e9a253 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -8,16 +8,12 @@ - - - - @@ -25,4 +21,8 @@ + + + + \ No newline at end of file diff --git a/equinox-sync/equinox-sync.sln b/equinox-sync/equinox-sync.sln index 0f752a0ff..194d785e1 100644 --- a/equinox-sync/equinox-sync.sln +++ b/equinox-sync/equinox-sync.sln @@ -9,6 +9,10 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\confluent-kafka-fsharp\src\Propulsion\Propulsion.fsproj", "{5187E18B-7659-492D-8797-3B71AA2A4DC9}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.EventStore", "..\..\confluent-kafka-fsharp\src\Propulsion.EventStore\Propulsion.EventStore.fsproj", "{F124F25B-B0E1-483C-9CF4-EAF18820935A}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -19,6 +23,14 @@ Global {EB9D803D-2E0C-437E-9282-8E29E479F066}.Debug|Any CPU.Build.0 = Debug|Any CPU {EB9D803D-2E0C-437E-9282-8E29E479F066}.Release|Any CPU.ActiveCfg = Release|Any CPU {EB9D803D-2E0C-437E-9282-8E29E479F066}.Release|Any CPU.Build.0 = Release|Any CPU + {5187E18B-7659-492D-8797-3B71AA2A4DC9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5187E18B-7659-492D-8797-3B71AA2A4DC9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5187E18B-7659-492D-8797-3B71AA2A4DC9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5187E18B-7659-492D-8797-3B71AA2A4DC9}.Release|Any CPU.Build.0 = Release|Any CPU + {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Debug|Any CPU.Build.0 = Debug|Any CPU + {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Release|Any CPU.ActiveCfg = Release|Any CPU + {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From e37590964e5d21049501d5b6ffe004315d40cb34 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 5 Jun 2019 10:09:49 +0100 Subject: [PATCH 333/353] Add ChangeLog info --- CHANGELOG.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39a5bfbea..e6d8ce3af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,13 +12,14 @@ The `Unreleased` section name is replaced by the expected version of next releas - `eqxprojector -k`'s `Consumer` offers a `StreamSpan`-based API for ordered, de-deduplicated consumption without concurrent executions at stream level [#24](https://github.com/jet/dotnet-templates/pull/24) - `eqxprojector -k -n`'s `Producer` offers a parallel producer mode which runs all projections in parallel without constraints (or need to synthesize streams) [#24](https://github.com/jet/dotnet-templates/pull/24) -- `eqxsync` has EventStore Sink support via `es` commandline option [#23](https://github.com/jet/dotnet-templates/pull/23) +- `eqxsync` has EventStore Sink support via `cosmos` ... `es` commandline option [#23](https://github.com/jet/dotnet-templates/pull/23) +- `eqxsync` has EventStore Source support via `es` ... `cosmos` commandline option [#16](https://github.com/jet/dotnet-templates/pull/16) ### Changed - `eqxtestbed`, `eqxweb`, `eqxwebcs` now target `Equinox 2.0.0-preview9` -- `eqxprojector` `-k` now targets `Jet.ConfluentKafka.FSharp` + `Propulsion.Kafka` v `1.0.1-rc1` [#24](https://github.com/jet/dotnet-templates/pull/24) -- `eqxsync` now targets `Propulsion.Cosmos`,`Propulsion.EventStore` v `1.0.1-rc1` [#24](https://github.com/jet/dotnet-templates/pull/24) +- `eqxprojector` `-k` now targets `Jet.ConfluentKafka.FSharp` + `Propulsion.Kafka` v `1.0.1-rc2` [#24](https://github.com/jet/dotnet-templates/pull/24) +- `eqxsync` now targets `Propulsion.Cosmos`,`Propulsion.EventStore` v `1.0.1-rc2` [#24](https://github.com/jet/dotnet-templates/pull/24) ### Removed ### Fixed From 87ffa03d61481e879049953e79ce8abc5353fc8b Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 5 Jun 2019 10:17:23 +0100 Subject: [PATCH 334/353] Tidy for consistency with Sync --- equinox-projector/Consumer/Infrastructure.fs | 11 ------ equinox-projector/Consumer/Program.fs | 14 ++++---- equinox-projector/Projector/Infrastructure.fs | 18 ---------- equinox-projector/Projector/Program.fs | 36 +++++++++---------- 4 files changed, 25 insertions(+), 54 deletions(-) delete mode 100644 equinox-projector/Projector/Infrastructure.fs diff --git a/equinox-projector/Consumer/Infrastructure.fs b/equinox-projector/Consumer/Infrastructure.fs index bf58e9c66..9e4655fe5 100644 --- a/equinox-projector/Consumer/Infrastructure.fs +++ b/equinox-projector/Consumer/Infrastructure.fs @@ -17,17 +17,6 @@ type FSharp.Control.Async with elif t.IsCompleted then k t.Result else ek(Exception "invalid Task state!")) |> ignore -// static member AwaitTaskCorrect (task : Task) : Async = -// Async.FromContinuations <| fun (k,ek,_) -> -// task.ContinueWith (fun (t:Task) -> -// if t.IsFaulted then -// let e = t.Exception -// if e.InnerExceptions.Count = 1 then ek e.InnerExceptions.[0] -// else ek e -// elif t.IsCanceled then ek (TaskCanceledException("Task wrapped with Async has been cancelled.")) -// elif t.IsCompleted then k () -// else ek(Exception "invalid Task state!")) -// |> ignore type System.Threading.SemaphoreSlim with /// F# friendly semaphore await function diff --git a/equinox-projector/Consumer/Program.fs b/equinox-projector/Consumer/Program.fs index ebd239276..7693f01d0 100644 --- a/equinox-projector/Consumer/Program.fs +++ b/equinox-projector/Consumer/Program.fs @@ -14,13 +14,13 @@ module CmdParser = [] type Parameters = - | [] Group of string - | [] Broker of string - | [] Topic of string - | [] MaxDop of int - | [] MaxInflightGb of float - | [] LagFreqM of float - | [] Verbose + | [] Group of string + | [] Broker of string + | [] Topic of string + | [] MaxDop of int + | [] MaxInflightGb of float + | [] LagFreqM of float + | [] Verbose interface IArgParserTemplate with member a.Usage = a |> function diff --git a/equinox-projector/Projector/Infrastructure.fs b/equinox-projector/Projector/Infrastructure.fs deleted file mode 100644 index 3d38eae0a..000000000 --- a/equinox-projector/Projector/Infrastructure.fs +++ /dev/null @@ -1,18 +0,0 @@ -[] -module private ProjectorTemplate.Projector.Infrastructure - -open System -open System.Threading -open System.Threading.Tasks - -#nowarn "21" // re AwaitKeyboardInterrupt -#nowarn "40" // re AwaitKeyboardInterrupt - -type Async with - /// Asynchronously awaits the next keyboard interrupt event - static member AwaitKeyboardInterrupt () : Async = - Async.FromContinuations(fun (sc,_,_) -> - let isDisposed = ref 0 - let rec callback _ = Task.Run(fun () -> if Interlocked.Increment isDisposed = 1 then d.Dispose() ; sc ()) |> ignore - and d : IDisposable = Console.CancelKeyPress.Subscribe callback - in ()) \ No newline at end of file diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 8672959c4..e2be2a025 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -16,23 +16,23 @@ module CmdParser = module Cosmos = type [] Parameters = - | [] ConnectionMode of Equinox.Cosmos.ConnectionMode - | [] Timeout of float - | [] Retries of int - | [] RetriesWaitTime of int - | [] Connection of string - | [] Database of string - | [] Collection of string + | [] Connection of string + | [] ConnectionMode of Equinox.Cosmos.ConnectionMode + | [] Database of string + | [] Collection of string + | [] Timeout of float + | [] Retries of int + | [] RetriesWaitTime of int interface IArgParserTemplate with member a.Usage = match a with - | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 1)." - | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION, Cosmos Emulator)." | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE, test)." | Collection _ -> "specify a collection name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_COLLECTION, test)." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." + | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" type Arguments(a : ParseResults) = member __.Mode = a.GetResult(ConnectionMode,Equinox.Cosmos.ConnectionMode.DirectTcp) member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" @@ -56,18 +56,18 @@ module CmdParser = type Parameters = (* ChangeFeed Args*) | [] ConsumerGroupName of string - | [] LeaseCollectionSuffix of string - | [] FromTail - | [] MaxDocuments of int + | [] LeaseCollectionSuffix of string + | [] FromTail + | [] MaxDocuments of int | [] MaxPendingBatches of int | [] MaxWriters of int - | [] LagFreqM of float - | [] Verbose - | [] ChangeFeedVerbose + | [] LagFreqM of float + | [] Verbose + | [] ChangeFeedVerbose //#if kafka (* Kafka Args *) - | [] Broker of string - | [] Topic of string + | [] Broker of string + | [] Topic of string //#endif (* ChangeFeed Args *) | [] Cosmos of ParseResults From 4cd9be7340024a4768712e3d875078cc00d11b46 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 5 Jun 2019 14:35:03 +0100 Subject: [PATCH 335/353] Consistency tweaks --- equinox-projector/Projector/Projector.fsproj | 1 - equinox-sync/Sync/Program.fs | 174 +++++++++---------- equinox-sync/Sync/Sync.fsproj | 1 + equinox-sync/equinox-sync.sln | 6 + 4 files changed, 94 insertions(+), 88 deletions(-) diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 4b0bcb487..fa1d7555e 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -7,7 +7,6 @@ - diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 595ced46a..1fed7c1fb 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -34,23 +34,22 @@ module CmdParser = | [] SrcEs of ParseResults | [] SrcCosmos of ParseResults interface IArgParserTemplate with - member a.Usage = - match a with - | ConsumerGroupName _ -> "Projector consumer group name." - | MaxPendingBatches _ -> "maximum number of batches to let processing get ahead of completion. Default: 16" - | MaxWriters _ -> "maximum number of concurrent writes to target permitted. Default: 512" - | MaxConnections _ -> "size of Sink connection pool to maintain. Default: 1" - | MaxSubmit _ -> "maximum number of batches to submit concurrently. Default: 8" - - | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" - | Verbose -> "request Verbose Logging. Default: off" - | VerboseConsole -> "request Verbose Console Logging. Default: off" - - | CategoryBlacklist _ -> "category whitelist" - | CategoryWhitelist _ -> "category blacklist" - - | SrcCosmos _ -> "Cosmos input parameters." - | SrcEs _ -> "EventStore input parameters." + member a.Usage = a |> function + | ConsumerGroupName _ ->"Projector consumer group name." + | MaxPendingBatches _ ->"maximum number of batches to let processing get ahead of completion. Default: 16" + | MaxWriters _ -> "maximum number of concurrent writes to target permitted. Default: 512" + | MaxConnections _ -> "size of Sink connection pool to maintain. Default: 1" + | MaxSubmit _ -> "maximum number of batches to submit concurrently. Default: 8" + + | LocalSeq -> "configures writing to a local Seq endpoint at http://localhost:5341, see https://getseq.net" + | Verbose -> "request Verbose Logging. Default: off" + | VerboseConsole -> "request Verbose Console Logging. Default: off" + + | CategoryBlacklist _ ->"category whitelist" + | CategoryWhitelist _ ->"category blacklist" + + | SrcCosmos _ -> "Cosmos input parameters." + | SrcEs _ -> "EventStore input parameters." and Arguments(a : ParseResults) = member __.ConsumerGroupName = a.GetResult ConsumerGroupName member __.MaxPendingBatches = a.GetResult(MaxPendingBatches,2048) @@ -128,23 +127,22 @@ module CmdParser = | [] DstEs of ParseResults | [] DstCosmos of ParseResults interface IArgParserTemplate with - member a.Usage = - match a with - | FromTail -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." - | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" - | LagFreqM _ -> "frequency (in minutes) to dump lag stats. Default: off" - | LeaseCollection _ -> "specify Collection Name for Leases collection (default: `sourcecollection` + `-aux`)." - - | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION)." - | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." - | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE)." - | Collection _ -> "specify a collection name within `SourceDatabase`." - | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 1)." - | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" - - | DstEs _ -> "EventStore Sink parameters." - | DstCosmos _ -> "CosmosDb Sink parameters." + member a.Usage = a |> function + | FromTail -> "(iff the Consumer Name is fresh) - force skip to present Position. Default: Never skip an event." + | MaxDocuments _ -> "maximum item count to request from feed. Default: unlimited" + | LagFreqM _ -> "frequency (in minutes) to dump lag stats. Default: off" + | LeaseCollection _ -> "specify Collection Name for Leases collection (default: `sourcecollection` + `-aux`)." + + | Connection _ -> "specify a connection string for a Cosmos account (defaults: envvar:EQUINOX_COSMOS_CONNECTION)." + | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." + | Database _ -> "specify a database name for Cosmos account (defaults: envvar:EQUINOX_COSMOS_DATABASE)." + | Collection _ -> "specify a collection name within `SourceDatabase`." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 1)." + | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" + + | DstEs _ -> "EventStore Sink parameters." + | DstCosmos _ -> "CosmosDb Sink parameters." and CosmosSourceArguments(a : ParseResults) = member __.FromTail = a.Contains CosmosSourceParameters.FromTail member __.MaxDocuments = a.TryGetResult MaxDocuments @@ -197,30 +195,29 @@ module CmdParser = | [] Es of ParseResults | [] Cosmos of ParseResults interface IArgParserTemplate with - member a.Usage = - match a with - | FromTail -> "Start the processing from the Tail" - | Gorge _ -> "Request Parallel readers phase during initial catchup, running one chunk (256MB) apart. Default: off" - | StreamReaders _ -> "number of concurrent readers that will fetch a missing stream when in tailing mode. Default: 1. TODO: IMPLEMENT!" - | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" - | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." - | BatchSize _ -> "maximum item count to request from feed. Default: 4096" - | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" - | Position _ -> "EventStore $all Stream Position to commence from" - | Chunk _ -> "EventStore $all Chunk to commence from" - | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" - - | Verbose -> "Include low level Store logging." - | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." - | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." - | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." - | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." - | Timeout _ -> "specify operation timeout in seconds (default: 20)." - | Retries _ -> "specify operation retries (default: 3)." - | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." - - | Cosmos _ -> "CosmosDb Sink parameters." - | Es _ -> "EventStore Sink parameters." + member a.Usage = a |> function + | FromTail -> "Start the processing from the Tail" + | Gorge _ -> "Request Parallel readers phase during initial catchup, running one chunk (256MB) apart. Default: off" + | StreamReaders _ -> "number of concurrent readers that will fetch a missing stream when in tailing mode. Default: 1. TODO: IMPLEMENT!" + | Tail _ -> "attempt to read from tail at specified interval in Seconds. Default: 1" + | ForceRestart _ -> "Forget the current committed position; start from (and commit) specified position. Default: start from specified position or resume from committed." + | BatchSize _ -> "maximum item count to request from feed. Default: 4096" + | MinBatchSize _ -> "minimum item count to drop down to in reaction to read failures. Default: 512" + | Position _ -> "EventStore $all Stream Position to commence from" + | Chunk _ -> "EventStore $all Chunk to commence from" + | Percent _ -> "EventStore $all Stream Position to commence from (as a percentage of current tail position)" + + | Verbose -> "Include low level Store logging." + | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." + | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." + | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." + | Password _ -> "specify a Password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." + | Timeout _ -> "specify operation timeout in seconds (default: 20)." + | Retries _ -> "specify operation retries (default: 3)." + | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." + + | Cosmos _ -> "CosmosDb Sink parameters." + | Es _ -> "EventStore Sink parameters." and EsSourceArguments(a : ParseResults) = member __.Gorge = a.TryGetResult Gorge member __.StreamReaders = a.GetResult(StreamReaders,1) @@ -230,11 +227,11 @@ module CmdParser = member __.MinBatchSize = a.GetResult(MinBatchSize,512) member __.StartPos = match a.TryGetResult Position, a.TryGetResult Chunk, a.TryGetResult Percent, a.Contains FromTail with - | Some p, _, _, _ -> Absolute p - | _, Some c, _, _ -> StartPos.Chunk c - | _, _, Some p, _ -> Percentage p + | Some p, _, _, _ -> Absolute p + | _, Some c, _, _ -> StartPos.Chunk c + | _, _, Some p, _ -> Percentage p | None, None, None, true -> StartPos.TailOrCheckpoint - | None, None, None, _ -> StartPos.StartOrCheckpoint + | None, None, None, _ -> StartPos.StartOrCheckpoint member __.Host = match a.TryGetResult EsSourceParameters.Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" member __.Port = match a.TryGetResult EsSourceParameters.Port with Some x -> Some x | None -> Environment.GetEnvironmentVariable "EQUINOX_ES_PORT" |> Option.ofObj |> Option.map int @@ -267,16 +264,15 @@ module CmdParser = | [] Retries of int | [] RetriesWaitTime of int interface IArgParserTemplate with - member a.Usage = - match a with - | Connection _ -> "specify a connection string for a Cosmos account (default: envvar:EQUINOX_COSMOS_CONNECTION)." - | Database _ -> "specify a database name for Cosmos account (default: envvar:EQUINOX_COSMOS_DATABASE)." - | Collection _ -> "specify a collection name for Cosmos account (default: envvar:EQUINOX_COSMOS_COLLECTION)." - | LeaseCollection _ -> "specify Collection Name for Leases collection (default: `sourcecollection` + `-aux`)." - | Timeout _ -> "specify operation timeout in seconds (default: 5)." - | Retries _ -> "specify operation retries (default: 0)." - | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" - | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." + member a.Usage = a |> function + | Connection _ -> "specify a connection string for a Cosmos account (default: envvar:EQUINOX_COSMOS_CONNECTION)." + | Database _ -> "specify a database name for Cosmos account (default: envvar:EQUINOX_COSMOS_DATABASE)." + | Collection _ -> "specify a collection name for Cosmos account (default: envvar:EQUINOX_COSMOS_COLLECTION)." + | LeaseCollection _ -> "specify Collection Name for Leases collection (default: `sourcecollection` + `-aux`)." + | Timeout _ -> "specify operation timeout in seconds (default: 5)." + | Retries _ -> "specify operation retries (default: 0)." + | RetriesWaitTime _ -> "specify max wait-time for retry when being throttled by Cosmos in seconds (default: 5)" + | ConnectionMode _ -> "override the connection mode (default: DirectTcp)." and CosmosSinkArguments(a : ParseResults) = member __.Connection = match a.TryGetResult Connection with Some x -> x | None -> envBackstop "Connection" "EQUINOX_COSMOS_CONNECTION" member __.Mode = a.GetResult(ConnectionMode, Equinox.Cosmos.ConnectionMode.DirectTcp) @@ -308,16 +304,15 @@ module CmdParser = | [] Retries of int | [] HeartbeatTimeout of float interface IArgParserTemplate with - member a.Usage = - match a with - | Verbose -> "Include low level Store logging." - | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." - | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." - | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." - | Password _ -> "specify a password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." - | Timeout _ -> "specify operation timeout in seconds (default: 20)." - | Retries _ -> "specify operation retries (default: 3)." - | HeartbeatTimeout _ -> "specify heartbeat timeout in seconds (default: 1.5)." + member a.Usage = a |> function + | Verbose -> "Include low level Store logging." + | Host _ -> "specify a DNS query, using Gossip-driven discovery against all A records returned (defaults: envvar:EQUINOX_ES_HOST, localhost)." + | Port _ -> "specify a custom port (default: envvar:EQUINOX_ES_PORT, 30778)." + | Username _ -> "specify a username (defaults: envvar:EQUINOX_ES_USERNAME, admin)." + | Password _ -> "specify a password (defaults: envvar:EQUINOX_ES_PASSWORD, changeit)." + | Timeout _ -> "specify operation timeout in seconds (default: 20)." + | Retries _ -> "specify operation retries (default: 3)." + | HeartbeatTimeout _ ->"specify heartbeat timeout in seconds (default: 1.5)." and EsSinkArguments(a : ParseResults) = member __.Discovery = match __.Port with Some p -> Discovery.GossipDnsCustomPort (__.Host, p) | None -> Discovery.GossipDns __.Host member __.Host = match a.TryGetResult Host with Some x -> x | None -> envBackstop "Host" "EQUINOX_ES_HOST" @@ -415,11 +410,16 @@ module EventV0Parser = tmp.GetPropertyValue<'T>("content") /// Maps fields in an Equinox V0 Event to the interface defined by the Propulsion.Streams library - let (|StandardCodecEvent|) (x: EventV0) = Propulsion.Streams.Internal.EventData.Create(x.t, x.d, timestamp = x.c) + let (|PropulsionEvent|) (x: EventV0) = + { new Propulsion.Streams.IEvent<_> with + member __.EventType = x.t + member __.Data = x.d + member __.Meta = null + member __.Timestamp = x.c } /// We assume all Documents represent Events laid out as above let parse (d : Microsoft.Azure.Documents.Document) : Propulsion.Streams.StreamEvent<_> = - let (StandardCodecEvent e) as x = d.Cast() + let (PropulsionEvent e) as x = d.Cast() { stream = x.s; index = x.i; event = e } : _ let transformV0 categorize catFilter (v0SchemaDocument: Microsoft.Azure.Documents.Document) : Propulsion.Streams.StreamEvent<_> seq = seq { @@ -429,10 +429,10 @@ let transformV0 categorize catFilter (v0SchemaDocument: Microsoft.Azure.Document yield parsed } //#else let transformOrFilter categorize catFilter (changeFeedDocument: Microsoft.Azure.Documents.Document) : Propulsion.Streams.StreamEvent<_> seq = seq { - for { stream = s; index = i; event = e } in DocumentParser.enumEvents changeFeedDocument do + for { stream = s} as e in EquinoxCosmosParser.enumStreamEvents changeFeedDocument do // NB the `index` needs to be contiguous with existing events - IOW filtering needs to be at stream (and not event) level if catFilter (categorize s) then - yield { stream = s; index = i; event = Propulsion.Streams.Internal.EventData.Create(e.EventType, e.Data, e.Meta, e.Timestamp) } } + yield e } //#endif let start (args : CmdParser.Arguments) = @@ -501,7 +501,7 @@ let start (args : CmdParser.Arguments) = || e.EventStreamId = "SkuFileUpload-778f1efeab214f5bab2860d1f802ef24" || e.EventStreamId = "PurchaseOrder-5791" // item too large || not (catFilter e.EventStreamId) -> None - | e -> e |> Propulsion.EventStore.Reader.toIngestionItem |> Some + | PropulsionStreamEvent e -> Some e let runPipeline = EventStoreSource.Run( log, sink, checkpoints, connect, spec, categorize, tryMapEvent catFilter, diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index cd0e9a253..8e356484f 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -22,6 +22,7 @@ + diff --git a/equinox-sync/equinox-sync.sln b/equinox-sync/equinox-sync.sln index 194d785e1..f091cbb99 100644 --- a/equinox-sync/equinox-sync.sln +++ b/equinox-sync/equinox-sync.sln @@ -13,6 +13,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\conflue EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.EventStore", "..\..\confluent-kafka-fsharp\src\Propulsion.EventStore\Propulsion.EventStore.fsproj", "{F124F25B-B0E1-483C-9CF4-EAF18820935A}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Cosmos", "..\..\confluent-kafka-fsharp\src\Propulsion.Cosmos\Propulsion.Cosmos.fsproj", "{AE6E974A-9B0A-4B31-9317-893401B360B3}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -31,6 +33,10 @@ Global {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Debug|Any CPU.Build.0 = Debug|Any CPU {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Release|Any CPU.ActiveCfg = Release|Any CPU {F124F25B-B0E1-483C-9CF4-EAF18820935A}.Release|Any CPU.Build.0 = Release|Any CPU + {AE6E974A-9B0A-4B31-9317-893401B360B3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AE6E974A-9B0A-4B31-9317-893401B360B3}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AE6E974A-9B0A-4B31-9317-893401B360B3}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AE6E974A-9B0A-4B31-9317-893401B360B3}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From f23ac0080f20a171be0974cea6dc722008d56e0a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Wed, 5 Jun 2019 23:52:05 +0100 Subject: [PATCH 336/353] Adjust to propulsion naming --- equinox-sync/Sync/Program.fs | 2 +- equinox-sync/Sync/Sync.fsproj | 4 ++-- equinox-sync/equinox-sync.sln | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 1fed7c1fb..f4e3bf0e7 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -517,4 +517,4 @@ let main argv = with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.InvalidArguments msg -> eprintfn "%s" msg; 1 | e -> eprintfn "%s" e.Message; 1 - finally Log.CloseAndFlush() \ No newline at end of file + finally Log.CloseAndFlush( \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index 8e356484f..7079f0c18 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -22,8 +22,8 @@ - - + + \ No newline at end of file diff --git a/equinox-sync/equinox-sync.sln b/equinox-sync/equinox-sync.sln index f091cbb99..f1465eeb2 100644 --- a/equinox-sync/equinox-sync.sln +++ b/equinox-sync/equinox-sync.sln @@ -9,11 +9,11 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution README.md = README.md EndProjectSection EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\confluent-kafka-fsharp\src\Propulsion\Propulsion.fsproj", "{5187E18B-7659-492D-8797-3B71AA2A4DC9}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\propulsion\src\Propulsion\Propulsion.fsproj", "{5187E18B-7659-492D-8797-3B71AA2A4DC9}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.EventStore", "..\..\confluent-kafka-fsharp\src\Propulsion.EventStore\Propulsion.EventStore.fsproj", "{F124F25B-B0E1-483C-9CF4-EAF18820935A}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.EventStore", "..\..\propulsion\src\Propulsion.EventStore\Propulsion.EventStore.fsproj", "{F124F25B-B0E1-483C-9CF4-EAF18820935A}" EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Cosmos", "..\..\confluent-kafka-fsharp\src\Propulsion.Cosmos\Propulsion.Cosmos.fsproj", "{AE6E974A-9B0A-4B31-9317-893401B360B3}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Cosmos", "..\..\propulsion\src\Propulsion.Cosmos\Propulsion.Cosmos.fsproj", "{AE6E974A-9B0A-4B31-9317-893401B360B3}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution From e7531aa121ce38ecf3ee84df44d15d6f49e567ce Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Thu, 6 Jun 2019 16:57:20 +0100 Subject: [PATCH 337/353] fix typo --- equinox-sync/Sync/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index f4e3bf0e7..1fed7c1fb 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -517,4 +517,4 @@ let main argv = with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.InvalidArguments msg -> eprintfn "%s" msg; 1 | e -> eprintfn "%s" e.Message; 1 - finally Log.CloseAndFlush( \ No newline at end of file + finally Log.CloseAndFlush() \ No newline at end of file From 5197a1368f58da2e5d23c9d529717f0c79276eec Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 7 Jun 2019 14:16:37 +0100 Subject: [PATCH 338/353] Add #serial filtering --- equinox-sync/Sync/Program.fs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index 1fed7c1fb..c0fb3e290 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -489,8 +489,9 @@ let start (args : CmdParser.Arguments) = let tryMapEvent catFilter (x : EventStore.ClientAPI.ResolvedEvent) = match x.Event with | e when not e.IsJson - || e.EventStreamId.StartsWith("$") + || e.EventStreamId.StartsWith "$" || e.EventType.StartsWith("compacted",StringComparison.OrdinalIgnoreCase) + || e.EventStreamId.StartsWith "#serial" || e.EventStreamId.StartsWith "marvel_bookmark" || e.EventStreamId.EndsWith "_checkpoints" || e.EventStreamId.EndsWith "_checkpoint" From d5e91cbfd92c3c52ebd5e5817f6fed29f2e86e20 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 7 Jun 2019 15:50:57 +0100 Subject: [PATCH 339/353] Log --- equinox-sync/Sync/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-sync/Sync/Program.fs b/equinox-sync/Sync/Program.fs index c0fb3e290..5b697bfdb 100644 --- a/equinox-sync/Sync/Program.fs +++ b/equinox-sync/Sync/Program.fs @@ -511,8 +511,8 @@ let start (args : CmdParser.Arguments) = [] let main argv = - try try let sink, runPipeline = CmdParser.parse argv |> start - runPipeline |> Async.Start + try try let sink, runSourcePipeline = CmdParser.parse argv |> start + runSourcePipeline |> Async.Start sink.AwaitCompletion() |> Async.RunSynchronously if sink.RanToCompletion then 0 else 2 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 From 397ae45dc813ec6ad3aff994d8da2ca065c1fab8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 7 Jun 2019 17:04:42 +0100 Subject: [PATCH 340/353] LOG --- equinox-sync/Sync/Properties/launchSettings.json | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 equinox-sync/Sync/Properties/launchSettings.json diff --git a/equinox-sync/Sync/Properties/launchSettings.json b/equinox-sync/Sync/Properties/launchSettings.json new file mode 100644 index 000000000..061e56b1d --- /dev/null +++ b/equinox-sync/Sync/Properties/launchSettings.json @@ -0,0 +1,8 @@ +{ + "profiles": { + "Sync": { + "commandName": "Project", + "commandLineArgs": "MAIN es -o 10 -oh 5 -m 512 -h guardians-shared-00.eventstore.eastus2.qa.jet.network -x 30798 cosmos -c MAIN3 -r 1" + } + } +} \ No newline at end of file From dd17999d1de5f0bd672a5f4b66ff3c917e0a17d5 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Sat, 8 Jun 2019 20:44:47 +0100 Subject: [PATCH 341/353] kafka --- equinox-projector/Projector/Projector.fsproj | 1 + 1 file changed, 1 insertion(+) diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index dda97b563..760f32ef1 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -4,6 +4,7 @@ Exe netcoreapp2.1 5 + kafka From 0045a0db1183de8e78ca3259c2940f05134965bb Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 00:46:08 +0100 Subject: [PATCH 342/353] Ref local --- equinox-projector/Projector/Program.fs | 4 ++-- equinox-projector/Projector/Projector.fsproj | 9 +++++++-- .../equinox-projector-consumer.sln | 18 ++++++++++++++++++ 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 44c7c6dd7..15383b795 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -148,8 +148,8 @@ module Logging = |> fun c -> c.CreateLogger() let replaceLongDataWithNull (x : Propulsion.Streams.IEvent) : Propulsion.Streams.IEvent<_> = - if x.Data.Length < 900_000 then x - else Propulsion.Streams.Internal.EventData.Create(x.EventType,null,x.Meta,x.Timestamp) :> _ + //if x.Data.Length < 900_000 then x + Propulsion.Streams.Internal.EventData.Create(x.EventType,null,x.Meta,x.Timestamp) :> _ let hackDropBigBodies (e : Propulsion.Streams.StreamEvent<_>) : Propulsion.Streams.StreamEvent<_> = { stream = e.stream; index = e.index; event = replaceLongDataWithNull e.event } diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 760f32ef1..710bce119 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -14,11 +14,16 @@ - + - + + + + + + \ No newline at end of file diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index b4dd004e2..fb14bde77 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -12,6 +12,12 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consum EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Projector", "Projector\Projector.fsproj", "{72668C1E-2187-4DAF-BDBE-1637CDA67894}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Kafka", "..\..\propulsion\src\Propulsion.Kafka\Propulsion.Kafka.fsproj", "{B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\propulsion\src\Propulsion\Propulsion.fsproj", "{5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}" +EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Cosmos", "..\..\propulsion\src\Propulsion.Cosmos\Propulsion.Cosmos.fsproj", "{EC6A27BB-DECF-4799-8E2A-85102E7328CF}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -26,6 +32,18 @@ Global {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Debug|Any CPU.Build.0 = Debug|Any CPU {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Release|Any CPU.ActiveCfg = Release|Any CPU {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Release|Any CPU.Build.0 = Release|Any CPU + {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Debug|Any CPU.Build.0 = Debug|Any CPU + {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Release|Any CPU.ActiveCfg = Release|Any CPU + {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Release|Any CPU.Build.0 = Release|Any CPU + {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Debug|Any CPU.Build.0 = Debug|Any CPU + {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Release|Any CPU.ActiveCfg = Release|Any CPU + {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Release|Any CPU.Build.0 = Release|Any CPU + {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Debug|Any CPU.Build.0 = Debug|Any CPU + {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Release|Any CPU.ActiveCfg = Release|Any CPU + {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From d570cb4b8cac79ea4ea7e1e10bba8c1b2c3da8e3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 01:10:11 +0100 Subject: [PATCH 343/353] log exns --- equinox-projector/Projector/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 15383b795..3e33d2d28 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -148,7 +148,7 @@ module Logging = |> fun c -> c.CreateLogger() let replaceLongDataWithNull (x : Propulsion.Streams.IEvent) : Propulsion.Streams.IEvent<_> = - //if x.Data.Length < 900_000 then x + //if x.Data.Length < 900_000 then x else Propulsion.Streams.Internal.EventData.Create(x.EventType,null,x.Meta,x.Timestamp) :> _ let hackDropBigBodies (e : Propulsion.Streams.StreamEvent<_>) : Propulsion.Streams.StreamEvent<_> = @@ -218,5 +218,5 @@ let main argv = if projector.RanToCompletion then 0 else 2 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 - | e -> eprintfn "%s" e.Message; 1 + | e -> eprintfn "%O" e; 1 finally Log.CloseAndFlush() \ No newline at end of file From bdc884e172abb55c24ddabdfdac90f28871c6c0a Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 01:42:36 +0100 Subject: [PATCH 344/353] low compress --- equinox-projector/Projector/Program.fs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 3e33d2d28..7ce729d02 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -186,7 +186,8 @@ let start (args : CmdParser.Arguments) = let projector = Propulsion.Kafka.StreamsProducer.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, - categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 5.) + categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 5., + customize = fun c -> c.CompressionLevel <- Nullable 0(*; c.CompressionType <- Confluent.Kafka.CompressionType.None*)) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) #endif #else @@ -218,5 +219,5 @@ let main argv = if projector.RanToCompletion then 0 else 2 with :? Argu.ArguParseException as e -> eprintfn "%s" e.Message; 1 | CmdParser.MissingArg msg -> eprintfn "%s" msg; 1 - | e -> eprintfn "%O" e; 1 + | e -> eprintfn "%s" e.Message; 1 finally Log.CloseAndFlush() \ No newline at end of file From 1f80df7eab9b1ff06c3cdf80bc59bd79a82afc4f Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 01:49:05 +0100 Subject: [PATCH 345/353] compress off --- equinox-projector/Projector/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 7ce729d02..41e7ec958 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -187,7 +187,7 @@ let start (args : CmdParser.Arguments) = Propulsion.Kafka.StreamsProducer.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 5., - customize = fun c -> c.CompressionLevel <- Nullable 0(*; c.CompressionType <- Confluent.Kafka.CompressionType.None*)) + customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Confluent.Kafka.CompressionType.None) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) #endif #else From ae09642b9c3ba7108e1982141d7282069421c4f8 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 01:49:54 +0100 Subject: [PATCH 346/353] fix --- equinox-projector/Projector/Program.fs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 41e7ec958..ec951974f 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -187,7 +187,7 @@ let start (args : CmdParser.Arguments) = Propulsion.Kafka.StreamsProducer.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 5., - customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Confluent.Kafka.CompressionType.None) + customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) #endif #else From 1dbf5b9205a74b236abba1e31970d6cacbb81f14 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 11:18:45 +0100 Subject: [PATCH 347/353] compress again --- equinox-projector/Projector/Program.fs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index ec951974f..88a335855 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -186,8 +186,8 @@ let start (args : CmdParser.Arguments) = let projector = Propulsion.Kafka.StreamsProducer.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, - categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 5., - customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None) + categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 2.(*, + customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) #endif #else From f81a32fd895255eac36adbe5c9821cb32b6adfff Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 11 Jun 2019 11:40:45 +0100 Subject: [PATCH 348/353] Parameterize producers --- equinox-projector/Projector/Program.fs | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 88a335855..caeda3872 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -68,6 +68,7 @@ module CmdParser = (* Kafka Args *) | [] Broker of string | [] Topic of string + | [] Producers of int //#endif (* ChangeFeed Args *) | [] Cosmos of ParseResults @@ -86,6 +87,7 @@ module CmdParser = //#if kafka | Broker _ -> "specify Kafka Broker, in host:port format. (default: use environment variable EQUINOX_KAFKA_BROKER, if specified)" | Topic _ -> "specify Kafka Topic Id. (default: use environment variable EQUINOX_KAFKA_TOPIC, if specified)" + | Producers _ -> "specify number of Kafka Producer instances to use. Default 1" //#endif | Cosmos _ -> "specify CosmosDb input parameters" and Arguments(args : ParseResults) = @@ -118,7 +120,8 @@ module CmdParser = and TargetInfo(args : ParseResults) = member __.Broker = Uri(match args.TryGetResult Broker with Some x -> x | None -> envBackstop "Broker" "EQUINOX_KAFKA_BROKER") member __.Topic = match args.TryGetResult Topic with Some x -> x | None -> envBackstop "Topic" "EQUINOX_KAFKA_TOPIC" - member x.BuildTargetParams() = x.Broker, x.Topic + member __.Producers = args.GetResult(Producers,1) + member x.BuildTargetParams() = x.Broker, x.Topic, x.Producers //#endif /// Parse the commandline; can throw exceptions in response to missing arguments and/or `-h`/`--help` args @@ -169,7 +172,7 @@ let start (args : CmdParser.Arguments) = let discovery, connector, source = args.Cosmos.BuildConnectionDetails() let aux, leaseId, startFromTail, maxDocuments, lagFrequency, (maxReadAhead, maxConcurrentStreams) = args.BuildChangeFeedParams() #if kafka - let (broker,topic) = args.Target.BuildTargetParams() + let (broker,topic, producers) = args.Target.BuildTargetParams() #if nostreams let render (doc : Microsoft.Azure.Documents.Document) : string * string = let equinoxPartition,documentId = doc.GetPropertyValue "p",doc.Id @@ -186,7 +189,8 @@ let start (args : CmdParser.Arguments) = let projector = Propulsion.Kafka.StreamsProducer.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, - categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 2.(*, + categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 2., + producerParallelism = producers(*, customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) #endif From 98136d91410ae205e88506903a30a1b52795a604 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Fri, 14 Jun 2019 17:45:07 +0100 Subject: [PATCH 349/353] Deal with renamespacings --- equinox-projector/Consumer/Consumer.fsproj | 2 +- equinox-projector/Consumer/Examples.fs | 10 +++++----- equinox-projector/Projector/Program.fs | 6 +++--- equinox-projector/Projector/Projector.fsproj | 9 ++------- 4 files changed, 11 insertions(+), 16 deletions(-) diff --git a/equinox-projector/Consumer/Consumer.fsproj b/equinox-projector/Consumer/Consumer.fsproj index 3e50960f7..aba3fabdb 100644 --- a/equinox-projector/Consumer/Consumer.fsproj +++ b/equinox-projector/Consumer/Consumer.fsproj @@ -16,7 +16,7 @@ - + diff --git a/equinox-projector/Consumer/Examples.fs b/equinox-projector/Consumer/Examples.fs index 6db7111b3..f3659a004 100644 --- a/equinox-projector/Consumer/Examples.fs +++ b/equinox-projector/Consumer/Examples.fs @@ -66,7 +66,7 @@ module EventParser = type Message = Faves of Favorites.Event | Saves of SavedForLater.Event | Category of name : string * count : int | Unclassified of messageKey : string type EquinoxEvent = - static member Parse (x: Propulsion.Kafka.Codec.RenderedEvent) = + static member Parse (x: Propulsion.Codec.NewtonsoftJson.RenderedEvent) = { new Equinox.Codec.IEvent<_> with member __.EventType = x.c member __.Data = x.d @@ -80,7 +80,7 @@ type EquinoxEvent = member __.Timestamp = x.Timestamp } type EquinoxSpan = - static member EnumCodecEvents (x: Propulsion.Kafka.Codec.RenderedSpan) : seq> = + static member EnumCodecEvents (x: Propulsion.Codec.NewtonsoftJson.RenderedSpan) : seq> = x.e |> Seq.map EquinoxEvent.Parse static member EnumCodecEvents (x: Propulsion.Streams.StreamSpan<_>) : seq> = x.events |> Seq.map EquinoxEvent.Parse @@ -102,12 +102,12 @@ type MessageInterpreter() = member __.EnumStreamEvents(KeyValue (streamName : string, spanJson)) : seq> = if streamName.StartsWith("#serial") then Seq.empty else - let span = JsonConvert.DeserializeObject(spanJson) - Propulsion.Kafka.Codec.RenderedSpan.enumStreamEvents span + let span = JsonConvert.DeserializeObject(spanJson) + Propulsion.Codec.NewtonsoftJson.RenderedSpan.enumStreamEvents span /// Handles various category / eventType / payload types as produced by Equinox.Tool member __.TryDecode(streamName, spanJson) = seq { - let span = JsonConvert.DeserializeObject(spanJson) + let span = JsonConvert.DeserializeObject(spanJson) yield! __.Interpret(streamName, EquinoxSpan.EnumCodecEvents span) } type Processor() = diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index caeda3872..08078cc65 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -49,7 +49,7 @@ module CmdParser = x.Mode, endpointUri, x.Database, x.Collection) Log.Information("CosmosDb timeout {timeout}s; Throttling retries {retries}, max wait {maxRetryWaitTime}s", (let t = x.Timeout in t.TotalSeconds), x.Retries, x.MaxRetryWaitTime) - let connector = CosmosConnector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) + let connector = Connector(x.Timeout, x.Retries, x.MaxRetryWaitTime, Log.Logger, mode=x.Mode) discovery, connector, { database = x.Database; collection = x.Collection } [] @@ -87,7 +87,7 @@ module CmdParser = //#if kafka | Broker _ -> "specify Kafka Broker, in host:port format. (default: use environment variable EQUINOX_KAFKA_BROKER, if specified)" | Topic _ -> "specify Kafka Topic Id. (default: use environment variable EQUINOX_KAFKA_TOPIC, if specified)" - | Producers _ -> "specify number of Kafka Producer instances to use. Default 1" + | Producers _ -> "specify number of Kafka Producer instances to use. Default: 1" //#endif | Cosmos _ -> "specify CosmosDb input parameters" and Arguments(args : ParseResults) = @@ -183,7 +183,7 @@ let start (args : CmdParser.Arguments) = let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, fun x -> upcast x) #else let render (stream: string, span: Propulsion.Streams.StreamSpan<_>) = - let rendered = Propulsion.Kafka.Codec.RenderedSpan.ofStreamSpan stream span + let rendered = Propulsion.Codec.NewtonsoftJson.RenderedSpan.ofStreamSpan stream span Newtonsoft.Json.JsonConvert.SerializeObject(rendered) let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] let projector = diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 710bce119..0b7404e0e 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -14,16 +14,11 @@ - + - + - - - - - \ No newline at end of file From c2befdfb5e9f77973864f99cd107aa1139516211 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 17 Jun 2019 11:20:22 +0100 Subject: [PATCH 350/353] Use separated Kafka config --- equinox-projector/Projector/Program.fs | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index 08078cc65..a3bce6296 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -177,21 +177,26 @@ let start (args : CmdParser.Arguments) = let render (doc : Microsoft.Azure.Documents.Document) : string * string = let equinoxPartition,documentId = doc.GetPropertyValue "p",doc.Id equinoxPartition,Newtonsoft.Json.JsonConvert.SerializeObject { Id = documentId } + let producers = + Propulsion.Kafka.Producers( + Log.Logger, "ProjectorTemplate", broker, topic, producerParallelism = producers(*, + customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) let projector = - Propulsion.Kafka.ParallelProducer.Start( - Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, statsInterval=TimeSpan.FromMinutes 1.) + Propulsion.Kafka.ParallelProducer.Start(maxReadAhead, maxConcurrentStreams, render, producers, statsInterval=TimeSpan.FromMinutes 1.) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, fun x -> upcast x) #else let render (stream: string, span: Propulsion.Streams.StreamSpan<_>) = let rendered = Propulsion.Codec.NewtonsoftJson.RenderedSpan.ofStreamSpan stream span Newtonsoft.Json.JsonConvert.SerializeObject(rendered) let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] + let producers = + Propulsion.Kafka.Producers( + Log.Logger, "ProjectorTemplate", broker, topic, producerParallelism = producers(*, + customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) let projector = Propulsion.Kafka.StreamsProducer.Start( - Log.Logger, maxReadAhead, maxConcurrentStreams, "ProjectorTemplate", broker, topic, render, - categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 2., - producerParallelism = producers(*, - customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) + Log.Logger, maxReadAhead, maxConcurrentStreams, render, producers, + categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 2.) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) #endif #else From 16467a891739836603a86e795ea433b83f079531 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 17 Jun 2019 11:22:13 +0100 Subject: [PATCH 351/353] ref local propulsionI --- equinox-projector/Consumer/Consumer.fsproj | 6 +++++- equinox-projector/Projector/Projector.fsproj | 9 +++++++-- equinox-projector/equinox-projector-consumer.sln | 6 ++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/equinox-projector/Consumer/Consumer.fsproj b/equinox-projector/Consumer/Consumer.fsproj index aba3fabdb..f4c74bd97 100644 --- a/equinox-projector/Consumer/Consumer.fsproj +++ b/equinox-projector/Consumer/Consumer.fsproj @@ -16,8 +16,12 @@ - + + + + + \ No newline at end of file diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index 0b7404e0e..eba623837 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -14,11 +14,16 @@ - + - + + + + + + \ No newline at end of file diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index fb14bde77..f51ba9cfb 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -18,6 +18,8 @@ Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\propuls EndProject Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Cosmos", "..\..\propulsion\src\Propulsion.Cosmos\Propulsion.Cosmos.fsproj", "{EC6A27BB-DECF-4799-8E2A-85102E7328CF}" EndProject +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Kafka0", "..\..\propulsion\src\Propulsion.Kafka0\Propulsion.Kafka0.fsproj", "{AD555167-E9EA-45B0-8D19-C313B10643C0}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -44,6 +46,10 @@ Global {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Debug|Any CPU.Build.0 = Debug|Any CPU {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Release|Any CPU.ActiveCfg = Release|Any CPU {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Release|Any CPU.Build.0 = Release|Any CPU + {AD555167-E9EA-45B0-8D19-C313B10643C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {AD555167-E9EA-45B0-8D19-C313B10643C0}.Debug|Any CPU.Build.0 = Debug|Any CPU + {AD555167-E9EA-45B0-8D19-C313B10643C0}.Release|Any CPU.ActiveCfg = Release|Any CPU + {AD555167-E9EA-45B0-8D19-C313B10643C0}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From effe77f3d20c7827f8b9de5c37a5b813ed75bcd3 Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Mon, 17 Jun 2019 17:08:18 +0100 Subject: [PATCH 352/353] Update after Kafka stats reorg --- equinox-projector/Projector/Program.fs | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/equinox-projector/Projector/Program.fs b/equinox-projector/Projector/Program.fs index a3bce6296..25ef7b73a 100644 --- a/equinox-projector/Projector/Program.fs +++ b/equinox-projector/Projector/Program.fs @@ -182,19 +182,19 @@ let start (args : CmdParser.Arguments) = Log.Logger, "ProjectorTemplate", broker, topic, producerParallelism = producers(*, customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) let projector = - Propulsion.Kafka.ParallelProducer.Start(maxReadAhead, maxConcurrentStreams, render, producers, statsInterval=TimeSpan.FromMinutes 1.) + Propulsion.Kafka.ParallelProducerSink.Start(maxReadAhead, maxConcurrentStreams, render, producers, statsInterval=TimeSpan.FromMinutes 1.) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, fun x -> upcast x) #else let render (stream: string, span: Propulsion.Streams.StreamSpan<_>) = let rendered = Propulsion.Codec.NewtonsoftJson.RenderedSpan.ofStreamSpan stream span Newtonsoft.Json.JsonConvert.SerializeObject(rendered) - let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] + let categorize (streamName : string) = streamName.Split([|'-';'_'|], 2, StringSplitOptions.RemoveEmptyEntries).[0] let producers = Propulsion.Kafka.Producers( Log.Logger, "ProjectorTemplate", broker, topic, producerParallelism = producers(*, customize = fun c -> c.CompressionLevel <- Nullable 0; c.CompressionType <- Nullable Confluent.Kafka.CompressionType.None*)) let projector = - Propulsion.Kafka.StreamsProducer.Start( + Propulsion.Kafka.StreamsProducerSink.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, render, producers, categorize, statsInterval=TimeSpan.FromMinutes 1., stateInterval=TimeSpan.FromMinutes 2.) let createObserver () = CosmosSource.CreateObserver(Log.Logger, projector.StartIngester, mapToStreamItems) @@ -203,9 +203,8 @@ let start (args : CmdParser.Arguments) = let project (_stream, span: Propulsion.Streams.StreamSpan<_>) = async { let r = Random() let ms = r.Next(1,span.events.Length) - do! Async.Sleep ms - return span.events.Length } - let categorize (streamName : string) = streamName.Split([|'-';'_'|],2).[0] + do! Async.Sleep ms } + let categorize (streamName : string) = streamName.Split([|'-';'_'|], 2, StringSplitOptions.RemoveEmptyEntries).[0] let projector = Propulsion.Streams.StreamsProjector.Start( Log.Logger, maxReadAhead, maxConcurrentStreams, project, From 3abdb54af9c1237cc912be87175f4274919f8d0e Mon Sep 17 00:00:00 2001 From: Ruben Bartelink Date: Tue, 18 Jun 2019 16:42:20 +0100 Subject: [PATCH 353/353] Remove propuslision refs --- equinox-projector/Consumer/Consumer.fsproj | 4 -- equinox-projector/Projector/Projector.fsproj | 6 --- .../equinox-projector-consumer.sln | 46 +++++-------------- .../Sync/Properties/launchSettings.json | 8 ---- equinox-sync/Sync/Sync.fsproj | 1 - 5 files changed, 11 insertions(+), 54 deletions(-) delete mode 100644 equinox-sync/Sync/Properties/launchSettings.json diff --git a/equinox-projector/Consumer/Consumer.fsproj b/equinox-projector/Consumer/Consumer.fsproj index 6a8043d75..9471af467 100644 --- a/equinox-projector/Consumer/Consumer.fsproj +++ b/equinox-projector/Consumer/Consumer.fsproj @@ -21,8 +21,4 @@ - - - - \ No newline at end of file diff --git a/equinox-projector/Projector/Projector.fsproj b/equinox-projector/Projector/Projector.fsproj index c43467012..64420d5df 100644 --- a/equinox-projector/Projector/Projector.fsproj +++ b/equinox-projector/Projector/Projector.fsproj @@ -4,7 +4,6 @@ Exe netcoreapp2.1 5 - kafka @@ -21,9 +20,4 @@ - - - - - \ No newline at end of file diff --git a/equinox-projector/equinox-projector-consumer.sln b/equinox-projector/equinox-projector-consumer.sln index f51ba9cfb..d7c3ca9e7 100644 --- a/equinox-projector/equinox-projector-consumer.sln +++ b/equinox-projector/equinox-projector-consumer.sln @@ -3,22 +3,14 @@ Microsoft Visual Studio Solution File, Format Version 12.00 # Visual Studio Version 16 VisualStudioVersion = 16.0.28729.10 MinimumVisualStudioVersion = 15.0.26124.0 +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Projector", "Projector\Projector.fsproj", "{6C72C937-ECFC-4DD4-9BA0-7355B237F974}" +EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution Items", "{518EE7E2-76AF-4DE9-A127-C2DFF709A468}" ProjectSection(SolutionItems) = preProject README.md = README.md EndProjectSection EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consumer.fsproj", "{DBADA035-7A86-4AAC-9ECF-96D28F887C40}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Projector", "Projector\Projector.fsproj", "{72668C1E-2187-4DAF-BDBE-1637CDA67894}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Kafka", "..\..\propulsion\src\Propulsion.Kafka\Propulsion.Kafka.fsproj", "{B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion", "..\..\propulsion\src\Propulsion\Propulsion.fsproj", "{5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Cosmos", "..\..\propulsion\src\Propulsion.Cosmos\Propulsion.Cosmos.fsproj", "{EC6A27BB-DECF-4799-8E2A-85102E7328CF}" -EndProject -Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Propulsion.Kafka0", "..\..\propulsion\src\Propulsion.Kafka0\Propulsion.Kafka0.fsproj", "{AD555167-E9EA-45B0-8D19-C313B10643C0}" +Project("{6EC3EE1D-3C4E-46DD-8F32-0CC8E7565705}") = "Consumer", "Consumer\Consumer.fsproj", "{7ED94D2B-1744-48A0-9B20-94E4777617E9}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -26,30 +18,14 @@ Global Release|Any CPU = Release|Any CPU EndGlobalSection GlobalSection(ProjectConfigurationPlatforms) = postSolution - {DBADA035-7A86-4AAC-9ECF-96D28F887C40}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {DBADA035-7A86-4AAC-9ECF-96D28F887C40}.Debug|Any CPU.Build.0 = Debug|Any CPU - {DBADA035-7A86-4AAC-9ECF-96D28F887C40}.Release|Any CPU.ActiveCfg = Release|Any CPU - {DBADA035-7A86-4AAC-9ECF-96D28F887C40}.Release|Any CPU.Build.0 = Release|Any CPU - {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Debug|Any CPU.Build.0 = Debug|Any CPU - {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Release|Any CPU.ActiveCfg = Release|Any CPU - {72668C1E-2187-4DAF-BDBE-1637CDA67894}.Release|Any CPU.Build.0 = Release|Any CPU - {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Debug|Any CPU.Build.0 = Debug|Any CPU - {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Release|Any CPU.ActiveCfg = Release|Any CPU - {B1E538BB-8C5B-4120-9BE9-2D3C1DC28207}.Release|Any CPU.Build.0 = Release|Any CPU - {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Debug|Any CPU.Build.0 = Debug|Any CPU - {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Release|Any CPU.ActiveCfg = Release|Any CPU - {5FA33C5B-9DD2-44BF-B2DF-B790DCED2A76}.Release|Any CPU.Build.0 = Release|Any CPU - {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Debug|Any CPU.Build.0 = Debug|Any CPU - {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Release|Any CPU.ActiveCfg = Release|Any CPU - {EC6A27BB-DECF-4799-8E2A-85102E7328CF}.Release|Any CPU.Build.0 = Release|Any CPU - {AD555167-E9EA-45B0-8D19-C313B10643C0}.Debug|Any CPU.ActiveCfg = Debug|Any CPU - {AD555167-E9EA-45B0-8D19-C313B10643C0}.Debug|Any CPU.Build.0 = Debug|Any CPU - {AD555167-E9EA-45B0-8D19-C313B10643C0}.Release|Any CPU.ActiveCfg = Release|Any CPU - {AD555167-E9EA-45B0-8D19-C313B10643C0}.Release|Any CPU.Build.0 = Release|Any CPU + {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Debug|Any CPU.Build.0 = Debug|Any CPU + {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Release|Any CPU.ActiveCfg = Release|Any CPU + {6C72C937-ECFC-4DD4-9BA0-7355B237F974}.Release|Any CPU.Build.0 = Release|Any CPU + {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Debug|Any CPU.Build.0 = Debug|Any CPU + {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.ActiveCfg = Release|Any CPU + {7ED94D2B-1744-48A0-9B20-94E4777617E9}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/equinox-sync/Sync/Properties/launchSettings.json b/equinox-sync/Sync/Properties/launchSettings.json deleted file mode 100644 index 061e56b1d..000000000 --- a/equinox-sync/Sync/Properties/launchSettings.json +++ /dev/null @@ -1,8 +0,0 @@ -{ - "profiles": { - "Sync": { - "commandName": "Project", - "commandLineArgs": "MAIN es -o 10 -oh 5 -m 512 -h guardians-shared-00.eventstore.eastus2.qa.jet.network -x 30798 cosmos -c MAIN3 -r 1" - } - } -} \ No newline at end of file diff --git a/equinox-sync/Sync/Sync.fsproj b/equinox-sync/Sync/Sync.fsproj index e1c3f1b70..95518b373 100644 --- a/equinox-sync/Sync/Sync.fsproj +++ b/equinox-sync/Sync/Sync.fsproj @@ -4,7 +4,6 @@ Exe netcoreapp2.1 5 - $(DefineConstants)