diff --git a/.gitignore b/.gitignore index b4b8ca4..b1e9612 100644 --- a/.gitignore +++ b/.gitignore @@ -185,5 +185,7 @@ UpgradeLog*.htm # Microsoft Fakes FakesAssemblies/ + *.orig /sample/Sample/logs/ +.idea diff --git a/Build.ps1 b/Build.ps1 index 35005d6..765d2dc 100644 --- a/Build.ps1 +++ b/Build.ps1 @@ -6,7 +6,7 @@ if(Test-Path .\artifacts) { Remove-Item .\artifacts -Force -Recurse } $branch = @{ $true = $env:APPVEYOR_REPO_BRANCH; $false = $(git symbolic-ref --short -q HEAD) }[$env:APPVEYOR_REPO_BRANCH -ne $NULL]; $revision = @{ $true = "{0:00000}" -f [convert]::ToInt32("0" + $env:APPVEYOR_BUILD_NUMBER, 10); $false = "local" }[$env:APPVEYOR_BUILD_NUMBER -ne $NULL]; -$suffix = @{ $true = ""; $false = "$branch-$revision"}[$branch -eq "master" -and $revision -ne "local"] +$suffix = @{ $true = ""; $false = "$branch-$revision"}[$branch -eq "main" -and $revision -ne "local"] foreach ($src in ls src/Serilog.*) { Push-Location $src diff --git a/README.md b/README.md index 8578941..7dee56b 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,8 @@ # Serilog.Sinks.Seq [![Build status](https://ci.appveyor.com/api/projects/status/t7qdv68pej6inukl/branch/master?svg=true)](https://ci.appveyor.com/project/serilog/serilog-sinks-seq/branch/master) [![NuGet](https://img.shields.io/nuget/v/Serilog.Sinks.Seq.svg)](https://nuget.org/packages/serilog.sinks.seq) [![Join the chat at https://gitter.im/serilog/serilog](https://img.shields.io/gitter/room/serilog/serilog.svg)](https://gitter.im/serilog/serilog) -A Serilog sink that writes events to the [Seq](https://getseq.net) structured log server. Supports .NET 4.5+, .NET Core, and platforms compatible with the [.NET Platform Standard](https://github.com/dotnet/corefx/blob/master/Documentation/architecture/net-platform-standard.md) 1.1 including Windows 8 & UWP, Windows Phone and Xamarin. +A Serilog sink that writes events to the [Seq](https://datalust.co/seq) structured log server. Supports .NET 4.5+, .NET Core, and platforms compatible with the [.NET Platform Standard](https://github.com/dotnet/corefx/blob/master/Documentation/architecture/net-platform-standard.md) 1.1 including Windows 8 & UWP, Windows Phone and Xamarin. -[![Package Logo](http://serilog.net/images/serilog-sink-seq-nuget.png)](http://nuget.org/packages/serilog.sinks.seq) +[Package Logo](https://nuget.org/packages/serilog.sinks.seq) ### Getting started @@ -28,15 +28,15 @@ Log.Error("Failed to log on user {ContactId}", contactId); Then query log event properties like `ContactId` from the browser: -![Query in Seq](https://nblumhardt.github.io/images/seq-sink-screenshot.png) +![Query in Seq](https://raw.githubusercontent.com/serilog/serilog-sinks-seq/dev/assets/search-by-property.png) -When the application shuts down, [ensure any buffered events are propertly flushed to Seq](http://blog.merbla.com/2016/07/06/serilog-log-closeandflush/) by disposing the logger or calling `Log.CloseAndFlush()`: +When the application shuts down, [ensure any buffered events are propertly flushed to Seq](https://merbla.com/2016/07/06/serilog-log-closeandflush/) by disposing the logger or calling `Log.CloseAndFlush()`: ```csharp Log.CloseAndFlush(); ``` -The sink can take advantage of Seq's [API keys](http://docs.getseq.net/docs/api-keys) to authenticate clients and dynamically attach properties to events at the server-side. To use an API key, specify it in the `apiKey` parameter of `WriteTo.Seq()`. +The sink can take advantage of Seq's [API keys](https://docs.datalust.co/docs/api-keys) to authenticate clients and dynamically attach properties to events at the server-side. To use an API key, specify it in the `apiKey` parameter of `WriteTo.Seq()`. ### XML `` configuration @@ -130,16 +130,30 @@ The equivalent configuration in XML (Serilog 2.6+) is: ``` -For further information see the [Seq documentation](http://docs.getseq.net/docs/using-serilog#dynamic-level-control). +The equivalent configuration in JSON is: -### Compact event format - -Seq 3.3 accepts Serilog's more efficient [compact JSON format](https://github.com/serilog/serilog-formatting-compact/). To use this, configure the sink with `compact: true`: - -```csharp - .WriteTo.Seq("http://localhost:5341", compact: true) +```json +{ + "Serilog": + { + "LevelSwitches": { "$controlSwitch": "Information" }, + "MinimumLevel": { "ControlledBy": "$controlSwitch" }, + "WriteTo": + [{ + "Name": "Seq", + "Args": + { + "serverUrl": "http://localhost:5341", + "apiKey": "yeEZyL3SMcxEKUijBjN", + "controlLevelSwitch": "$controlSwitch" + } + }] + } +} ``` +For further information see the [Seq documentation](https://docs.datalust.co/docs/using-serilog#dynamic-level-control). + ### Troubleshooting > Nothing showed up, what can I do? @@ -176,5 +190,4 @@ Serilog.Debugging.SelfLog.Enable(message => { * Turn on the Serilog `SelfLog` as described above to check for connectivity problems and other issues on the client side. * Make sure your application calls `Log.CloseAndFlush()`, or disposes the root `Logger`, before it exits - otherwise, buffered events may be lost. * If your app is a Windows console application, it is also important to close the console window by exiting the app; Windows console apps are terminated "hard" if the close button in the title bar is used, so events buffered for sending to Seq may be lost if you use it. - * [Raise an issue](https://github.com/serilog/serilog-sinks-seq/issues), ask for help on the [Seq support forum](http://docs.getseq.net/discuss) or email **support@getseq.net**. - + * [Raise an issue](https://github.com/serilog/serilog-sinks-seq/issues), ask for help on the [Seq support forum](https://docs.datalust.co/discuss) or email **support@datalust.co**. diff --git a/appveyor.yml b/appveyor.yml index 724a4a9..0500585 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,7 +1,6 @@ version: '{build}' skip_tags: true -image: Visual Studio 2017 -configuration: Release +image: Visual Studio 2019 install: - ps: mkdir -Force ".\build\" | Out-Null build_script: @@ -12,15 +11,14 @@ artifacts: deploy: - provider: NuGet api_key: - secure: nvZ/z+pMS91b3kG4DgfES5AcmwwGoBYQxr9kp4XiJHj25SAlgdIxFx++1N0lFH2x + secure: K6TcIFIyxBcDuZ5DL7bJC+fGwDo458q0SDh+ybLujGTddA/voxg2FMUtJQ7rTEbt skip_symbols: true on: - branch: /^(master|dev)$/ + branch: /^(main|dev)$/ - provider: GitHub auth_token: secure: p4LpVhBKxGS5WqucHxFQ5c7C8cP74kbNB0Z8k9Oxx/PMaDQ1+ibmoexNqVU5ZlmX artifact: /Serilog.*\.nupkg/ tag: v$(appveyor_build_version) on: - branch: master - + branch: main diff --git a/assets/icon.png b/assets/icon.png new file mode 100644 index 0000000..ed37092 Binary files /dev/null and b/assets/icon.png differ diff --git a/assets/search-by-property.png b/assets/search-by-property.png new file mode 100644 index 0000000..e609e4a Binary files /dev/null and b/assets/search-by-property.png differ diff --git a/global.json b/global.json new file mode 100644 index 0000000..10c378d --- /dev/null +++ b/global.json @@ -0,0 +1,5 @@ +{ + "sdk": { + "version": "5.0.102" + } +} diff --git a/sample/Sample/Program.cs b/sample/Sample/Program.cs index eea7433..b6fe03d 100644 --- a/sample/Sample/Program.cs +++ b/sample/Sample/Program.cs @@ -7,7 +7,7 @@ namespace Sample { public class Program { - public static void Main(string[] args) + public static void Main() { // By sharing between the Seq sink and logger itself, // Seq API keys can be used to control the level of the whole logging pipeline. diff --git a/sample/Sample/Properties/AssemblyInfo.cs b/sample/Sample/Properties/AssemblyInfo.cs deleted file mode 100644 index eefe616..0000000 --- a/sample/Sample/Properties/AssemblyInfo.cs +++ /dev/null @@ -1,23 +0,0 @@ -using System.Reflection; -using System.Runtime.CompilerServices; -using System.Runtime.InteropServices; - -// General Information about an assembly is controlled through the following -// set of attributes. Change these attribute values to modify the information -// associated with an assembly. -[assembly: AssemblyTitle("Sample")] -[assembly: AssemblyDescription("")] -[assembly: AssemblyConfiguration("")] -[assembly: AssemblyCompany("")] -[assembly: AssemblyProduct("Sample")] -[assembly: AssemblyCopyright("Copyright © 2016")] -[assembly: AssemblyTrademark("")] -[assembly: AssemblyCulture("")] - -// Setting ComVisible to false makes the types in this assembly not visible -// to COM components. If you need to access a type in this assembly from -// COM, set the ComVisible attribute to true on that type. -[assembly: ComVisible(false)] - -// The following GUID is for the ID of the typelib if this project is exposed to COM -[assembly: Guid("17497155-5d94-45df-81d9-bd960e8cf217")] diff --git a/sample/Sample/Sample.csproj b/sample/Sample/Sample.csproj index ca990e4..335c3d0 100644 --- a/sample/Sample/Sample.csproj +++ b/sample/Sample/Sample.csproj @@ -3,7 +3,7 @@ Sample Console Application nblumhardt - netcoreapp2.0;net47 + netcoreapp3.1;net47 Sample Exe Sample @@ -20,14 +20,10 @@ - - - - - + diff --git a/serilog-sinks-seq.sln b/serilog-sinks-seq.sln index 63b63f9..a10a9a7 100644 --- a/serilog-sinks-seq.sln +++ b/serilog-sinks-seq.sln @@ -7,7 +7,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "src", "src", "{037440DE-440 EndProject Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "assets", "assets", "{E9D1B5E1-DEB9-4A04-8BAB-24EC7240ADAF}" ProjectSection(SolutionItems) = preProject - .editorconfig = .editorconfig appveyor.yml = appveyor.yml Build.ps1 = Build.ps1 README.md = README.md diff --git a/serilog-sinks-seq.sln.DotSettings b/serilog-sinks-seq.sln.DotSettings new file mode 100644 index 0000000..6a9f366 --- /dev/null +++ b/serilog-sinks-seq.sln.DotSettings @@ -0,0 +1,13 @@ + + True + True + True + True + True + True + True + True + True + True + True + True \ No newline at end of file diff --git a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs index e8c43be..42826d0 100644 --- a/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs +++ b/src/Serilog.Sinks.Seq/SeqLoggerConfigurationExtensions.cs @@ -18,6 +18,7 @@ using Serilog.Events; using Serilog.Sinks.Seq; using System.Net.Http; +using Serilog.Sinks.PeriodicBatching; using Serilog.Sinks.Seq.Audit; #if DURABLE @@ -57,9 +58,6 @@ public static class SeqLoggerConfigurationExtensions /// A soft limit for the number of bytes to use for storing failed requests. /// The limit is soft in that it can be exceeded by any single error payload, but in that case only that single error /// payload will be retained. - /// Use the compact log event format defined by - /// Serilog.Formatting.Compact. Has no effect on - /// durable log shipping. Requires Seq 3.3+. /// The maximum number of events that will be held in-memory while waiting to ship them to /// Seq. Beyond this limit, events will be dropped. The default is 100,000. Has no effect on /// durable log shipping. @@ -78,7 +76,6 @@ public static LoggerConfiguration Seq( LoggingLevelSwitch controlLevelSwitch = null, HttpMessageHandler messageHandler = null, long? retainedInvalidPayloadsLimitBytes = null, - bool compact = false, int queueSizeLimit = SeqSink.DefaultQueueSizeLimit) { if (loggerSinkConfiguration == null) throw new ArgumentNullException(nameof(loggerSinkConfiguration)); @@ -89,21 +86,27 @@ public static LoggerConfiguration Seq( throw new ArgumentOutOfRangeException(nameof(queueSizeLimit), "Queue size limit must be non-zero."); var defaultedPeriod = period ?? SeqSink.DefaultPeriod; + var controlledSwitch = new ControlledLevelSwitch(controlLevelSwitch); ILogEventSink sink; - + if (bufferBaseFilename == null) { - sink = new SeqSink( + var batchedSink = new SeqSink( serverUrl, apiKey, - batchPostingLimit, - defaultedPeriod, eventBodyLimitBytes, - controlLevelSwitch, - messageHandler, - compact, - queueSizeLimit); + controlledSwitch, + messageHandler); + + var options = new PeriodicBatchingSinkOptions + { + BatchSizeLimit = batchPostingLimit, + Period = defaultedPeriod, + QueueLimit = queueSizeLimit + }; + + sink = new PeriodicBatchingSink(batchedSink, options); } else { @@ -116,7 +119,7 @@ public static LoggerConfiguration Seq( defaultedPeriod, bufferSizeLimitBytes, eventBodyLimitBytes, - controlLevelSwitch, + controlledSwitch, messageHandler, retainedInvalidPayloadsLimitBytes); #else @@ -125,7 +128,9 @@ public static LoggerConfiguration Seq( #endif } - return loggerSinkConfiguration.Sink(sink, restrictedToMinimumLevel); + return loggerSinkConfiguration.Conditional( + controlledSwitch.IsIncluded, + wt => wt.Sink(sink, restrictedToMinimumLevel)); } /// @@ -138,9 +143,6 @@ public static LoggerConfiguration Seq( /// in order to write an event to the sink. /// A Seq API key that authenticates the client to the Seq server. /// Used to construct the HttpClient that will send the log messages to Seq. - /// Use the compact log event format defined by - /// Serilog.Formatting.Compact. Has no effect on - /// durable log shipping. Requires Seq 3.3+. /// Logger configuration, allowing configuration to continue. /// A required parameter is null. public static LoggerConfiguration Seq( @@ -148,14 +150,13 @@ public static LoggerConfiguration Seq( string serverUrl, LogEventLevel restrictedToMinimumLevel = LevelAlias.Minimum, string apiKey = null, - HttpMessageHandler messageHandler = null, - bool compact = false) + HttpMessageHandler messageHandler = null) { if (loggerAuditSinkConfiguration == null) throw new ArgumentNullException(nameof(loggerAuditSinkConfiguration)); if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); return loggerAuditSinkConfiguration.Sink( - new SeqAuditSink(serverUrl, apiKey, messageHandler, compact), + new SeqAuditSink(serverUrl, apiKey, messageHandler), restrictedToMinimumLevel); } } diff --git a/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj b/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj index 00867df..fe8a814 100644 --- a/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj +++ b/src/Serilog.Sinks.Seq/Serilog.Sinks.Seq.csproj @@ -2,29 +2,42 @@ Serilog sink that writes to the Seq log server over HTTP/HTTPS. - 4.0.0 + 5.0.0 Serilog Contributors - Copyright © Serilog Contributors 2013-2017 - netstandard1.1;netstandard1.3;net45 + Copyright © Serilog Contributors + netstandard1.1;netstandard1.3;net45;netstandard2.0;netcoreapp3.1;net5.0 true true - Serilog.Sinks.Seq Serilog ../../assets/Serilog.snk true true - Serilog.Sinks.Seq serilog;seq - https://serilog.net/images/serilog-sink-seq-nuget.png + icon.png https://github.com/serilog/serilog-sinks-seq - http://www.apache.org/licenses/LICENSE-2.0 + Apache-2.0 + https://github.com/serilog/serilog-sinks-seq + git true + 8 $(DefineConstants);DURABLE;THREADING_TIMER + + $(DefineConstants);DURABLE;THREADING_TIMER + + + + $(DefineConstants);DURABLE;THREADING_TIMER + + + + $(DefineConstants);DURABLE;THREADING_TIMER + + $(DefineConstants);DURABLE;THREADING_TIMER;HRESULTS @@ -34,12 +47,16 @@ - - - + + + + + + + - + @@ -47,4 +64,8 @@ + + + + diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs index a88cc65..d61219e 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Audit/SeqAuditSink.cs @@ -29,21 +29,18 @@ sealed class SeqAuditSink : ILogEventSink, IDisposable { readonly string _apiKey; readonly HttpClient _httpClient; - readonly bool _useCompactFormat; - static readonly JsonValueFormatter JsonValueFormatter = new JsonValueFormatter(); + static readonly JsonValueFormatter JsonValueFormatter = new JsonValueFormatter("$type"); public SeqAuditSink( string serverUrl, string apiKey, - HttpMessageHandler messageHandler, - bool useCompactFormat) + HttpMessageHandler messageHandler) { if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); - _apiKey = apiKey; - _useCompactFormat = useCompactFormat; _httpClient = messageHandler != null ? new HttpClient(messageHandler) : new HttpClient(); _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl)); + _apiKey = apiKey; } public void Dispose() @@ -60,19 +57,11 @@ async Task EmitAsync(LogEvent logEvent) { if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); - string payload, payloadContentType; - if (_useCompactFormat) - { - payloadContentType = SeqApi.CompactLogEventFormatMimeType; - payload = FormatCompactPayload(logEvent); - } - else - { - payloadContentType = SeqApi.RawEventFormatMimeType; - payload = FormatRawPayload(logEvent); - } + var payload = new StringWriter(); + CompactJsonFormatter.FormatEvent(logEvent, payload, JsonValueFormatter); + payload.WriteLine(); - var content = new StringContent(payload, Encoding.UTF8, payloadContentType); + var content = new StringContent(payload.ToString(), Encoding.UTF8, SeqApi.CompactLogEventFormatMimeType); if (!string.IsNullOrWhiteSpace(_apiKey)) content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); @@ -80,22 +69,5 @@ async Task EmitAsync(LogEvent logEvent) if (!result.IsSuccessStatusCode) throw new LoggingFailedException($"Received failed result {result.StatusCode} when posting events to Seq"); } - - internal static string FormatCompactPayload(LogEvent logEvent) - { - var payload = new StringWriter(); - CompactJsonFormatter.FormatEvent(logEvent, payload, JsonValueFormatter); - payload.WriteLine(); - return payload.ToString(); - } - - internal static string FormatRawPayload(LogEvent logEvent) - { - var payload = new StringWriter(); - payload.Write("{\"Events\":["); - RawJsonFormatter.FormatContent(logEvent, payload); - payload.Write("]}"); - return payload.ToString(); - } } } diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs index d33742d..8dee47f 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/DurableSeqSink.cs @@ -19,6 +19,7 @@ using Serilog.Events; using System.Net.Http; using System.Text; +using Serilog.Formatting.Compact; namespace Serilog.Sinks.Seq.Durable { @@ -35,22 +36,23 @@ public DurableSeqSink( TimeSpan period, long? bufferSizeLimitBytes, long? eventBodyLimitBytes, - LoggingLevelSwitch levelControlSwitch, + ControlledLevelSwitch controlledSwitch, HttpMessageHandler messageHandler, long? retainedInvalidPayloadsLimitBytes) { if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); if (bufferBaseFilename == null) throw new ArgumentNullException(nameof(bufferBaseFilename)); + var fileSet = new FileSet(bufferBaseFilename); _shipper = new HttpLogShipper( + fileSet, serverUrl, - bufferBaseFilename, apiKey, batchPostingLimit, period, eventBodyLimitBytes, - levelControlSwitch, + controlledSwitch, messageHandler, retainedInvalidPayloadsLimitBytes, bufferSizeLimitBytes); @@ -58,8 +60,8 @@ public DurableSeqSink( const long individualFileSizeLimitBytes = 100L * 1024 * 1024; _sink = new LoggerConfiguration() .MinimumLevel.Verbose() - .WriteTo.File(new RawJsonFormatter(), - bufferBaseFilename + "-.json", + .WriteTo.File(new CompactJsonFormatter(), + fileSet.RollingFilePathFormat, rollingInterval: RollingInterval.Day, fileSizeLimitBytes: individualFileSizeLimitBytes, rollOnFileSizeLimit: true, diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSet.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSet.cs index 6ae74af..4322c20 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSet.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/FileSet.cs @@ -26,6 +26,8 @@ namespace Serilog.Sinks.Seq.Durable { class FileSet { + public const string RawFormatFileExtension = ".json"; + readonly string _bookmarkFilename; readonly string _candidateSearchPath; readonly string _logFolder; @@ -33,14 +35,21 @@ class FileSet const string InvalidPayloadFilePrefix = "invalid-"; + public string RollingFilePathFormat { get; } + public FileSet(string bufferBaseFilename) { if (bufferBaseFilename == null) throw new ArgumentNullException(nameof(bufferBaseFilename)); + RollingFilePathFormat = bufferBaseFilename + "-.clef"; + _bookmarkFilename = Path.GetFullPath(bufferBaseFilename + ".bookmark"); _logFolder = Path.GetDirectoryName(_bookmarkFilename); - _candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "-*.json"; - _filenameMatcher = new Regex("^" + Regex.Escape(Path.GetFileName(bufferBaseFilename)) + "-(?\\d{8})(?_[0-9]{3,}){0,1}\\.json$"); + + // The extension cannot be matched here because it may be either "json" (raw format) or "clef" (compact). + _candidateSearchPath = Path.GetFileName(bufferBaseFilename) + "-*.*"; + + _filenameMatcher = new Regex("^" + Regex.Escape(Path.GetFileName(bufferBaseFilename)) + "-(?\\d{8})(?_[0-9]{3,}){0,1}\\.(json|clef)$"); } public BookmarkFile OpenBookmarkFile() @@ -59,7 +68,7 @@ public string[] GetBufferFiles() .ToArray(); } - public void CleanUpBufferFiles(long bufferSizeLimitBytes, int alwaysRetainCount) + public void CleanUpBufferFiles(long bufferSizeLimitBytes) { try { @@ -84,9 +93,9 @@ public void CleanUpInvalidPayloadFiles(long maxNumberOfBytesToRetain) try { var candidateFiles = from file in Directory.EnumerateFiles(_logFolder, $"{InvalidPayloadFilePrefix}*.json") - let candiateFileInfo = new FileInfo(file) - orderby candiateFileInfo.LastWriteTimeUtc descending - select candiateFileInfo; + let candidateFileInfo = new FileInfo(file) + orderby candidateFileInfo.LastWriteTimeUtc descending + select candidateFileInfo; DeleteExceedingCumulativeSize(candidateFiles, maxNumberOfBytesToRetain, 0); } diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/HttpLogShipper.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/HttpLogShipper.cs index b707d49..e46571f 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/HttpLogShipper.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/HttpLogShipper.cs @@ -20,7 +20,6 @@ using System.Net; using System.Net.Http; using System.Text; -using Serilog.Core; using Serilog.Debugging; using Serilog.Events; using IOFile = System.IO.File; @@ -60,27 +59,27 @@ class HttpLogShipper : IDisposable volatile bool _unloading; public HttpLogShipper( + FileSet fileSet, string serverUrl, - string bufferBaseFilename, string apiKey, int batchPostingLimit, TimeSpan period, long? eventBodyLimitBytes, - LoggingLevelSwitch levelControlSwitch, + ControlledLevelSwitch controlledSwitch, HttpMessageHandler messageHandler, long? retainedInvalidPayloadsLimitBytes, long? bufferSizeLimitBytes) { + _fileSet = fileSet ?? throw new ArgumentNullException(nameof(fileSet)); _apiKey = apiKey; _batchPostingLimit = batchPostingLimit; _eventBodyLimitBytes = eventBodyLimitBytes; - _controlledSwitch = new ControlledLevelSwitch(levelControlSwitch); + _controlledSwitch = controlledSwitch; _connectionSchedule = new ExponentialBackoffConnectionSchedule(period); _retainedInvalidPayloadsLimitBytes = retainedInvalidPayloadsLimitBytes; _bufferSizeLimitBytes = bufferSizeLimitBytes; _httpClient = messageHandler != null ? new HttpClient(messageHandler) : new HttpClient(); _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl)); - _fileSet = new FileSet(bufferBaseFilename); _timer = new PortableTimer(c => OnTick()); SetTimer(); @@ -127,90 +126,88 @@ async Task OnTick() { count = 0; - using (var bookmarkFile = _fileSet.OpenBookmarkFile()) + using var bookmarkFile = _fileSet.OpenBookmarkFile(); + var position = bookmarkFile.TryReadBookmark(); + var files = _fileSet.GetBufferFiles(); + + if (position.File == null || !IOFile.Exists(position.File)) { - var position = bookmarkFile.TryReadBookmark(); - var files = _fileSet.GetBufferFiles(); + position = new FileSetPosition(0, files.FirstOrDefault()); + } - if (position.File == null || !IOFile.Exists(position.File)) - { - position = new FileSetPosition(0, files.FirstOrDefault()); - } + string payload, mimeType; + if (position.File == null) + { + payload = PayloadReader.MakeEmptyPayload(out mimeType); + count = 0; + } + else + { + payload = PayloadReader.ReadPayload(_batchPostingLimit, _eventBodyLimitBytes, ref position, ref count, out mimeType); + } - string payload; - if (position.File == null) + if (count > 0 || _controlledSwitch.IsActive && _nextRequiredLevelCheckUtc < DateTime.UtcNow) + { + _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval); + + var content = new StringContent(payload, Encoding.UTF8, mimeType); + if (!string.IsNullOrWhiteSpace(_apiKey)) + content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); + + var result = await _httpClient.PostAsync(SeqApi.BulkUploadResource, content).ConfigureAwait(false); + if (result.IsSuccessStatusCode) { - payload = PayloadReader.NoPayload; - count = 0; + _connectionSchedule.MarkSuccess(); + bookmarkFile.WriteBookmark(position); + var returned = await result.Content.ReadAsStringAsync().ConfigureAwait(false); + var minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned); + _controlledSwitch.Update(minimumAcceptedLevel); } - else + else if (result.StatusCode == HttpStatusCode.BadRequest || + result.StatusCode == HttpStatusCode.RequestEntityTooLarge) { - payload = PayloadReader.ReadPayload(_batchPostingLimit, _eventBodyLimitBytes, ref position, ref count); - } + // The connection attempt was successful - the payload we sent was the problem. + _connectionSchedule.MarkSuccess(); - if (count > 0 || _controlledSwitch.IsActive && _nextRequiredLevelCheckUtc < DateTime.UtcNow) - { - _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval); - - var content = new StringContent(payload, Encoding.UTF8, "application/json"); - if (!string.IsNullOrWhiteSpace(_apiKey)) - content.Headers.Add(SeqApi.ApiKeyHeaderName, _apiKey); - - var result = await _httpClient.PostAsync(SeqApi.BulkUploadResource, content).ConfigureAwait(false); - if (result.IsSuccessStatusCode) - { - _connectionSchedule.MarkSuccess(); - bookmarkFile.WriteBookmark(position); - var returned = await result.Content.ReadAsStringAsync().ConfigureAwait(false); - var minimumAcceptedLevel = SeqApi.ReadEventInputResult(returned); - _controlledSwitch.Update(minimumAcceptedLevel); - } - else if (result.StatusCode == HttpStatusCode.BadRequest || - result.StatusCode == HttpStatusCode.RequestEntityTooLarge) - { - // The connection attempt was successful - the payload we sent was the problem. - _connectionSchedule.MarkSuccess(); - - await DumpInvalidPayload(result, payload).ConfigureAwait(false); - - bookmarkFile.WriteBookmark(position); - } - else - { - _connectionSchedule.MarkFailure(); - SelfLog.WriteLine("Received failed HTTP shipping result {0}: {1}", result.StatusCode, - await result.Content.ReadAsStringAsync().ConfigureAwait(false)); - - if (_bufferSizeLimitBytes.HasValue) - _fileSet.CleanUpBufferFiles(_bufferSizeLimitBytes.Value, 2); - - break; - } + await DumpInvalidPayload(result, payload).ConfigureAwait(false); + + bookmarkFile.WriteBookmark(position); } - else if (position.File == null) + else { + _connectionSchedule.MarkFailure(); + SelfLog.WriteLine("Received failed HTTP shipping result {0}: {1}", result.StatusCode, + await result.Content.ReadAsStringAsync().ConfigureAwait(false)); + + if (_bufferSizeLimitBytes.HasValue) + _fileSet.CleanUpBufferFiles(_bufferSizeLimitBytes.Value); + break; } - else + } + else if (position.File == null) + { + break; + } + else + { + // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the + // regular interval, so mark the attempt as successful. + _connectionSchedule.MarkSuccess(); + + // Only advance the bookmark if no other process has the + // current file locked, and its length is as we found it. + if (files.Length == 2 && files.First() == position.File && + FileIsUnlockedAndUnextended(position)) { - // For whatever reason, there's nothing waiting to send. This means we should try connecting again at the - // regular interval, so mark the attempt as successful. - _connectionSchedule.MarkSuccess(); + bookmarkFile.WriteBookmark(new FileSetPosition(0, files[1])); + } - // Only advance the bookmark if no other process has the - // current file locked, and its length is as we found it. - if (files.Length == 2 && files.First() == position.File && - FileIsUnlockedAndUnextended(position)) - { - bookmarkFile.WriteBookmark(new FileSetPosition(0, files[1])); - } - - if (files.Length > 2) - { - // By this point, we expect writers to have relinquished locks - // on the oldest file. - IOFile.Delete(files[0]); - } + if (files.Length > 2) + { + // By this point, we expect writers to have relinquished locks + // on the oldest file. + IOFile.Delete(files[0]); } } } while (count == _batchPostingLimit); @@ -221,7 +218,7 @@ async Task OnTick() SelfLog.WriteLine("Exception while emitting periodic batch from {0}: {1}", this, ex); if (_bufferSizeLimitBytes.HasValue) - _fileSet.CleanUpBufferFiles(_bufferSizeLimitBytes.Value, 2); + _fileSet.CleanUpBufferFiles(_bufferSizeLimitBytes.Value); } finally { @@ -251,10 +248,8 @@ static bool FileIsUnlockedAndUnextended(FileSetPosition position) { try { - using (var fileStream = IOFile.Open(position.File, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read)) - { - return fileStream.Length <= position.NextLineStart; - } + using var fileStream = IOFile.Open(position.File, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.Read); + return fileStream.Length <= position.NextLineStart; } #if HRESULTS catch (IOException ex) diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/PayloadReader.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/PayloadReader.cs index e41e1db..49224bc 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/PayloadReader.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/Durable/PayloadReader.cs @@ -23,9 +23,56 @@ namespace Serilog.Sinks.Seq.Durable { static class PayloadReader { - public const string NoPayload = "{\"Events\":[]}"; + public static string ReadPayload( + int batchPostingLimit, + long? eventBodyLimitBytes, + ref FileSetPosition position, + ref int count, + out string mimeType) + { + if (position.File.EndsWith(".json")) + { + mimeType = SeqApi.RawEventFormatMimeType; + return ReadRawPayload(batchPostingLimit, eventBodyLimitBytes, ref position, ref count); + } + + mimeType = SeqApi.CompactLogEventFormatMimeType; + return ReadCompactPayload(batchPostingLimit, eventBodyLimitBytes, ref position, ref count); + } - public static string ReadPayload(int batchPostingLimit, long? eventBodyLimitBytes, ref FileSetPosition position, ref int count) + static string ReadCompactPayload(int batchPostingLimit, long? eventBodyLimitBytes, ref FileSetPosition position, ref int count) + { + var payload = new StringWriter(); + + using (var current = System.IO.File.Open(position.File, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)) + { + var nextLineStart = position.NextLineStart; + while (count < batchPostingLimit && TryReadLine(current, ref nextLineStart, out var nextLine)) + { + position = new FileSetPosition(nextLineStart, position.File); + + // Count is the indicator that work was done, so advances even in the (rare) case an + // oversized event is dropped. + ++count; + + if (eventBodyLimitBytes.HasValue && Encoding.UTF8.GetByteCount(nextLine) > eventBodyLimitBytes.Value) + { + SelfLog.WriteLine( + "Event JSON representation exceeds the byte size limit of {0} and will be dropped; data: {1}", + eventBodyLimitBytes, nextLine); + } + else + { + payload.WriteLine(nextLine); + } + } + } + + return payload.ToString(); + } + + + static string ReadRawPayload(int batchPostingLimit, long? eventBodyLimitBytes, ref FileSetPosition position, ref int count) { var payload = new StringWriter(); payload.Write("{\"Events\":["); @@ -87,6 +134,12 @@ static bool TryReadLine(Stream current, ref long nextStart, out string nextLine) return true; } + + public static string MakeEmptyPayload(out string mimeType) + { + mimeType = SeqApi.CompactLogEventFormatMimeType; + return SeqApi.NoPayload; + } } } diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/RawJsonFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/RawJsonFormatter.cs deleted file mode 100644 index 1c63f86..0000000 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/RawJsonFormatter.cs +++ /dev/null @@ -1,120 +0,0 @@ -// Serilog.Sinks.Seq Copyright 2016 Serilog Contributors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using Serilog.Events; -using Serilog.Formatting; -using Serilog.Formatting.Json; -using Serilog.Parsing; - -namespace Serilog.Sinks.Seq -{ - // Formatter for the JSON schema accepted by Seq's /raw endpoint. - class RawJsonFormatter : ITextFormatter - { - static readonly JsonValueFormatter ValueFormatter = new JsonValueFormatter(); - - public void Format(LogEvent logEvent, TextWriter output) - { - FormatContent(logEvent, output); - output.WriteLine(); - } - - public static void FormatContent(LogEvent logEvent, TextWriter output) - { - if (logEvent == null) throw new ArgumentNullException(nameof(logEvent)); - if (output == null) throw new ArgumentNullException(nameof(output)); - - output.Write("{\"Timestamp\":\""); - output.Write(logEvent.Timestamp.ToString("o")); - output.Write("\",\"Level\":\""); - output.Write(logEvent.Level); - output.Write("\",\"MessageTemplate\":"); - JsonValueFormatter.WriteQuotedJsonString(logEvent.MessageTemplate.Text, output); - if (logEvent.Exception != null) - { - output.Write(",\"Exception\":"); - JsonValueFormatter.WriteQuotedJsonString(logEvent.Exception.ToString(), output); - } - - if (logEvent.Properties.Count != 0) - WriteProperties(logEvent.Properties, output); - - var tokensWithFormat = logEvent.MessageTemplate.Tokens - .OfType() - .Where(pt => pt.Format != null); - - if (tokensWithFormat.Any()) - WriteRenderings(tokensWithFormat.GroupBy(pt => pt.PropertyName), logEvent.Properties, output); - - output.Write('}'); - } - - static void WriteProperties(IReadOnlyDictionary properties, TextWriter output) - { - output.Write(",\"Properties\":{"); - - var precedingDelimiter = ""; - foreach (var property in properties) - { - output.Write(precedingDelimiter); - precedingDelimiter = ","; - - JsonValueFormatter.WriteQuotedJsonString(property.Key, output); - output.Write(':'); - ValueFormatter.Format(property.Value, output); - } - - output.Write('}'); - } - - static void WriteRenderings(IEnumerable> tokensWithFormat, IReadOnlyDictionary properties, TextWriter output) - { - output.Write(",\"Renderings\":{"); - - var rdelim = ""; - foreach (var ptoken in tokensWithFormat) - { - output.Write(rdelim); - rdelim = ","; - - JsonValueFormatter.WriteQuotedJsonString(ptoken.Key, output); - output.Write(":["); - - var fdelim = ""; - foreach (var format in ptoken) - { - output.Write(fdelim); - fdelim = ","; - - output.Write("{\"Format\":"); - JsonValueFormatter.WriteQuotedJsonString(format.Format, output); - - output.Write(",\"Rendering\":"); - var sw = new StringWriter(); - format.Render(properties, sw); - JsonValueFormatter.WriteQuotedJsonString(sw.ToString(), output); - output.Write('}'); - } - - output.Write(']'); - } - - output.Write('}'); - } - } -} diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqApi.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqApi.cs index bc70a4d..a368b97 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqApi.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqApi.cs @@ -17,12 +17,13 @@ namespace Serilog.Sinks.Seq { - class SeqApi + static class SeqApi { public const string BulkUploadResource = "api/events/raw"; public const string ApiKeyHeaderName = "X-Seq-ApiKey"; public const string RawEventFormatMimeType = "application/json"; public const string CompactLogEventFormatMimeType = "application/vnd.serilog.clef"; + public const string NoPayload = ""; // Why not use a JSON parser here? For a very small case, it's not // worth taking on the extra payload/dependency management issues that @@ -49,8 +50,7 @@ class SeqApi return null; var value = eventInputResult.Substring(startValue, endValue - startValue); - LogEventLevel minimumLevel; - if (!Enum.TryParse(value, out minimumLevel)) + if (!Enum.TryParse(value, out LogEventLevel minimumLevel)) return null; return minimumLevel; diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqPayloadFormatter.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqPayloadFormatter.cs new file mode 100644 index 0000000..15248fe --- /dev/null +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqPayloadFormatter.cs @@ -0,0 +1,80 @@ +// Serilog.Sinks.Seq Copyright 2014-2019 Serilog Contributors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; +using Serilog.Debugging; +using Serilog.Events; +using Serilog.Formatting.Compact; +using Serilog.Formatting.Json; + +namespace Serilog.Sinks.Seq +{ + static class SeqPayloadFormatter + { + static readonly JsonValueFormatter JsonValueFormatter = new JsonValueFormatter("$type"); + + public static string FormatCompactPayload(IEnumerable events, long? eventBodyLimitBytes) + { + var payload = new StringWriter(); + + foreach (var logEvent in events) + { + var buffer = new StringWriter(); + + try + { + CompactJsonFormatter.FormatEvent(logEvent, buffer, JsonValueFormatter); + } + catch (Exception ex) + { + LogNonFormattableEvent(logEvent, ex); + continue; + } + + var json = buffer.ToString(); + if (CheckEventBodySize(json, eventBodyLimitBytes)) + { + payload.WriteLine(json); + } + } + + return payload.ToString(); + } + + static void LogNonFormattableEvent(LogEvent logEvent, Exception ex) + { + SelfLog.WriteLine( + "Event at {0} with message template {1} could not be formatted into JSON for Seq and will be dropped: {2}", + logEvent.Timestamp.ToString("o"), logEvent.MessageTemplate.Text, ex); + } + + static bool CheckEventBodySize(string json, long? eventBodyLimitBytes) + { + if (eventBodyLimitBytes.HasValue && + Encoding.UTF8.GetByteCount(json) > eventBodyLimitBytes.Value) + { + SelfLog.WriteLine( + "Event JSON representation exceeds the byte size limit of {0} set for this Seq sink and will be dropped; data: {1}", + eventBodyLimitBytes, json); + return false; + } + + return true; + } + + } +} \ No newline at end of file diff --git a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs index 5288b40..9ea9540 100644 --- a/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs +++ b/src/Serilog.Sinks.Seq/Sinks/Seq/SeqSink.cs @@ -1,4 +1,4 @@ -// Serilog.Sinks.Seq Copyright 2016 Serilog Contributors +// Serilog.Sinks.Seq Copyright 2014-2019 Serilog Contributors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -14,33 +14,27 @@ using System; using System.Collections.Generic; -using System.IO; using System.Linq; using System.Net.Http; using System.Text; using System.Threading.Tasks; -using Serilog.Core; using Serilog.Debugging; using Serilog.Events; -using Serilog.Formatting.Compact; -using Serilog.Formatting.Json; using Serilog.Sinks.PeriodicBatching; namespace Serilog.Sinks.Seq { - class SeqSink : PeriodicBatchingSink + class SeqSink : IBatchedLogEventSink, IDisposable { public const int DefaultBatchPostingLimit = 1000; public static readonly TimeSpan DefaultPeriod = TimeSpan.FromSeconds(2); public const int DefaultQueueSizeLimit = 100000; static readonly TimeSpan RequiredLevelCheckInterval = TimeSpan.FromMinutes(2); - static readonly JsonValueFormatter JsonValueFormatter = new JsonValueFormatter(); readonly string _apiKey; readonly long? _eventBodyLimitBytes; readonly HttpClient _httpClient; - readonly bool _useCompactFormat; DateTime _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval); readonly ControlledLevelSwitch _controlledSwitch; @@ -48,35 +42,26 @@ class SeqSink : PeriodicBatchingSink public SeqSink( string serverUrl, string apiKey, - int batchPostingLimit, - TimeSpan period, long? eventBodyLimitBytes, - LoggingLevelSwitch levelControlSwitch, - HttpMessageHandler messageHandler, - bool useCompactFormat, - int queueSizeLimit) - : base(batchPostingLimit, period, queueSizeLimit) + ControlledLevelSwitch controlledSwitch, + HttpMessageHandler messageHandler) { if (serverUrl == null) throw new ArgumentNullException(nameof(serverUrl)); + _controlledSwitch = controlledSwitch ?? throw new ArgumentNullException(nameof(controlledSwitch)); _apiKey = apiKey; _eventBodyLimitBytes = eventBodyLimitBytes; - _controlledSwitch = new ControlledLevelSwitch(levelControlSwitch); - _useCompactFormat = useCompactFormat; _httpClient = messageHandler != null ? new HttpClient(messageHandler) : new HttpClient(); _httpClient.BaseAddress = new Uri(SeqApi.NormalizeServerBaseAddress(serverUrl)); } - protected override void Dispose(bool disposing) + public void Dispose() { - base.Dispose(disposing); - - if (disposing) - _httpClient.Dispose(); + _httpClient.Dispose(); } // The sink must emit at least one event on startup, and the server be // configured to set a specific level, before background level checks will be performed. - protected override async Task OnEmptyBatchAsync() + public async Task OnEmptyBatchAsync() { if (_controlledSwitch.IsActive && _nextRequiredLevelCheckUtc < DateTime.UtcNow) @@ -85,21 +70,12 @@ protected override async Task OnEmptyBatchAsync() } } - protected override async Task EmitBatchAsync(IEnumerable events) + public async Task EmitBatchAsync(IEnumerable events) { _nextRequiredLevelCheckUtc = DateTime.UtcNow.Add(RequiredLevelCheckInterval); - string payload, payloadContentType; - if (_useCompactFormat) - { - payloadContentType = SeqApi.CompactLogEventFormatMimeType; - payload = FormatCompactPayload(events, _eventBodyLimitBytes); - } - else - { - payloadContentType = SeqApi.RawEventFormatMimeType; - payload = FormatRawPayload(events, _eventBodyLimitBytes); - } + var payloadContentType = SeqApi.CompactLogEventFormatMimeType; + var payload = SeqPayloadFormatter.FormatCompactPayload(events, _eventBodyLimitBytes); var content = new StringContent(payload, Encoding.UTF8, payloadContentType); if (!string.IsNullOrWhiteSpace(_apiKey)) @@ -112,92 +88,5 @@ protected override async Task EmitBatchAsync(IEnumerable events) var returned = await result.Content.ReadAsStringAsync(); _controlledSwitch.Update(SeqApi.ReadEventInputResult(returned)); } - - internal static string FormatCompactPayload(IEnumerable events, long? eventBodyLimitBytes) - { - var payload = new StringWriter(); - - foreach (var logEvent in events) - { - var buffer = new StringWriter(); - - try - { - CompactJsonFormatter.FormatEvent(logEvent, buffer, JsonValueFormatter); - } - catch (Exception ex) - { - LogNonFormattableEvent(logEvent, ex); - continue; - } - - var json = buffer.ToString(); - if (CheckEventBodySize(json, eventBodyLimitBytes)) - { - payload.WriteLine(json); - } - } - - return payload.ToString(); - } - - internal static string FormatRawPayload(IEnumerable events, long? eventBodyLimitBytes) - { - var payload = new StringWriter(); - payload.Write("{\"Events\":["); - - var delimStart = ""; - foreach (var logEvent in events) - { - var buffer = new StringWriter(); - - try - { - RawJsonFormatter.FormatContent(logEvent, buffer); - } - catch (Exception ex) - { - LogNonFormattableEvent(logEvent, ex); - continue; - } - - var json = buffer.ToString(); - if (CheckEventBodySize(json, eventBodyLimitBytes)) - { - payload.Write(delimStart); - payload.Write(json); - delimStart = ","; - } - } - - payload.Write("]}"); - return payload.ToString(); - } - - protected override bool CanInclude(LogEvent evt) - { - return _controlledSwitch.IsIncluded(evt); - } - - static bool CheckEventBodySize(string json, long? eventBodyLimitBytes) - { - if (eventBodyLimitBytes.HasValue && - Encoding.UTF8.GetByteCount(json) > eventBodyLimitBytes.Value) - { - SelfLog.WriteLine( - "Event JSON representation exceeds the byte size limit of {0} set for this Seq sink and will be dropped; data: {1}", - eventBodyLimitBytes, json); - return false; - } - - return true; - } - - static void LogNonFormattableEvent(LogEvent logEvent, Exception ex) - { - SelfLog.WriteLine( - "Event at {0} with message template {1} could not be formatted into JSON for Seq and will be dropped: {2}", - logEvent.Timestamp.ToString("o"), logEvent.MessageTemplate.Text, ex); - } } } diff --git a/test/Serilog.Sinks.Seq.Tests/Durable/PayloadReaderTests.cs b/test/Serilog.Sinks.Seq.Tests/Durable/PayloadReaderTests.cs index 4e7c5ba..91c4667 100644 --- a/test/Serilog.Sinks.Seq.Tests/Durable/PayloadReaderTests.cs +++ b/test/Serilog.Sinks.Seq.Tests/Durable/PayloadReaderTests.cs @@ -13,6 +13,33 @@ public class PayloadReaderTests { [Fact] public void ReadsEventsFromBufferFiles() + { + using (var tmp = new TempFolder()) + { + var fn = tmp.AllocateFilename("clef"); + var lines = IOFile.ReadAllText(Path.Combine("Resources", "ThreeBufferedEvents.clef.txt"), Encoding.UTF8).Split(new [] {'\r', '\n'}, StringSplitOptions.RemoveEmptyEntries); + using (var f = IOFile.Create(fn)) + using (var fw = new StreamWriter(f, Encoding.UTF8)) + { + foreach (var line in lines) + { + fw.WriteLine(line); + } + } + var position = new FileSetPosition(0, fn); + var count = 0; + PayloadReader.ReadPayload(1000, null, ref position, ref count, out var mimeType); + + Assert.Equal(SeqApi.CompactLogEventFormatMimeType, mimeType); + + Assert.Equal(3, count); + Assert.Equal(465 + 3 * (Environment.NewLine.Length - 1), position.NextLineStart); + Assert.Equal(fn, position.File); + } + } + + [Fact] + public void ReadsEventsFromRawBufferFiles() { using (var tmp = new TempFolder()) { @@ -28,7 +55,9 @@ public void ReadsEventsFromBufferFiles() } var position = new FileSetPosition(0, fn); var count = 0; - var payload = PayloadReader.ReadPayload(1000, null, ref position, ref count); + var payload = PayloadReader.ReadPayload(1000, null, ref position, ref count, out var mimeType); + + Assert.Equal(SeqApi.RawEventFormatMimeType, mimeType); Assert.Equal(3, count); Assert.Equal(576 + 3 * (Environment.NewLine.Length - 1), position.NextLineStart); diff --git a/test/Serilog.Sinks.Seq.Tests/RawJsonFormatterTests.cs b/test/Serilog.Sinks.Seq.Tests/RawJsonFormatterTests.cs deleted file mode 100644 index 33cba59..0000000 --- a/test/Serilog.Sinks.Seq.Tests/RawJsonFormatterTests.cs +++ /dev/null @@ -1,69 +0,0 @@ -using System; -using System.IO; -using Newtonsoft.Json.Linq; -using Xunit; -using Serilog.Sinks.Seq.Tests.Support; - -namespace Serilog.Sinks.Seq.Tests -{ - public class RawJsonFormatterTests - { - void AssertValidJson(Action act) - { - var output = new StringWriter(); - var formatter = new RawJsonFormatter(); - var log = new LoggerConfiguration() - .WriteTo.Sink(new TextWriterSink(output, formatter)) - .CreateLogger(); - - act(log); - - var json = output.ToString(); - - // Unfortunately this will not detect all JSON formatting issues; better than nothing however. - JObject.Parse(json); - } - - [Fact] - public void AnEmptyEventIsValidJson() - { - AssertValidJson(log => log.Information("No properties")); - } - - [Fact] - public void AMinimalEventIsValidJson() - { - AssertValidJson(log => log.Information("One {Property}", 42)); - } - - [Fact] - public void MultiplePropertiesAreDelimited() - { - AssertValidJson(log => log.Information("Property {First} and {Second}", "One", "Two")); - } - - [Fact] - public void ExceptionsAreFormattedToValidJson() - { - AssertValidJson(log => log.Information(new DivideByZeroException(), "With exception")); - } - - [Fact] - public void ExceptionAndPropertiesAreValidJson() - { - AssertValidJson(log => log.Information(new DivideByZeroException(), "With exception and {Property}", 42)); - } - - [Fact] - public void RenderingsAreValidJson() - { - AssertValidJson(log => log.Information("One {Rendering:x8}", 42)); - } - - [Fact] - public void MultipleRenderingsAreDelimited() - { - AssertValidJson(log => log.Information("Rendering {First:x8} and {Second:x8}", 1, 2)); - } - } -} diff --git a/test/Serilog.Sinks.Seq.Tests/Resources/ThreeBufferedEvents.clef.txt b/test/Serilog.Sinks.Seq.Tests/Resources/ThreeBufferedEvents.clef.txt new file mode 100644 index 0000000..d728951 --- /dev/null +++ b/test/Serilog.Sinks.Seq.Tests/Resources/ThreeBufferedEvents.clef.txt @@ -0,0 +1,3 @@ +{"@t":"2017-10-19T12:04:39.9775056+10:00","@l":"Information","@mt":"Running loop {Counter}, switch is at {Level}","Counter":700019,"Level":"Information"} +{"@t":"2017-10-19T12:04:39.9792156+10:00","@l":"Information","@mt":"Running loop {Counter}, switch is at {Level}","Counter":700020,"Level":"Information"} +{"@t":"2017-10-19T12:04:39.9792575+10:00","@l":"Information","@mt":"Running loop {Counter}, switch is at {Level}","Counter":700021,"Level":"Information"} diff --git a/test/Serilog.Sinks.Seq.Tests/SeqPayloadFormatterTests.cs b/test/Serilog.Sinks.Seq.Tests/SeqPayloadFormatterTests.cs new file mode 100644 index 0000000..69fde7d --- /dev/null +++ b/test/Serilog.Sinks.Seq.Tests/SeqPayloadFormatterTests.cs @@ -0,0 +1,24 @@ +using Serilog.Sinks.Seq.Tests.Support; +using Xunit; + +namespace Serilog.Sinks.Seq.Tests +{ + public class SeqPayloadFormatterTests + { + [Fact] + public void EventsAreFormattedIntoCompactJsonPayloads() + { + var evt = Some.LogEvent("Hello, {Name}!", "Alice"); + var json = SeqPayloadFormatter.FormatCompactPayload(new[] { evt }, null); + Assert.Contains("Name\":\"Alice", json); + } + + [Fact] + public void EventsAreDroppedWhenCompactJsonRenderingFails() + { + var evt = Some.LogEvent(new NastyException(), "Hello, {Name}!", "Alice"); + var json = SeqPayloadFormatter.FormatCompactPayload(new[] { evt }, null); + Assert.Empty(json); + } + } +} diff --git a/test/Serilog.Sinks.Seq.Tests/SeqSinkTests.cs b/test/Serilog.Sinks.Seq.Tests/SeqSinkTests.cs deleted file mode 100644 index e663a89..0000000 --- a/test/Serilog.Sinks.Seq.Tests/SeqSinkTests.cs +++ /dev/null @@ -1,40 +0,0 @@ -using Serilog.Sinks.Seq.Tests.Support; -using Xunit; - -namespace Serilog.Sinks.Seq.Tests -{ - public class SeqSinkTests - { - [Fact] - public void EventsAreFormattedIntoJsonPayloads() - { - var evt = Some.LogEvent("Hello, {Name}!", "Alice"); - var json = SeqSink.FormatRawPayload(new[] {evt}, null); - Assert.Contains("Name\":\"Alice", json); - } - - [Fact] - public void EventsAreDroppedWhenJsonRenderingFails() - { - var evt = Some.LogEvent(new NastyException(), "Hello, {Name}!", "Alice"); - var json = SeqSink.FormatRawPayload(new[] { evt }, null); - Assert.Contains("[]", json); - } - - [Fact] - public void EventsAreFormattedIntoCompactJsonPayloads() - { - var evt = Some.LogEvent("Hello, {Name}!", "Alice"); - var json = SeqSink.FormatCompactPayload(new[] { evt }, null); - Assert.Contains("Name\":\"Alice", json); - } - - [Fact] - public void EventsAreDroppedWhenCompactJsonRenderingFails() - { - var evt = Some.LogEvent(new NastyException(), "Hello, {Name}!", "Alice"); - var json = SeqSink.FormatCompactPayload(new[] { evt }, null); - Assert.Empty(json); - } - } -} diff --git a/test/Serilog.Sinks.Seq.Tests/Serilog.Sinks.Seq.Tests.csproj b/test/Serilog.Sinks.Seq.Tests/Serilog.Sinks.Seq.Tests.csproj index 7ea48eb..987fc8a 100644 --- a/test/Serilog.Sinks.Seq.Tests/Serilog.Sinks.Seq.Tests.csproj +++ b/test/Serilog.Sinks.Seq.Tests/Serilog.Sinks.Seq.Tests.csproj @@ -1,6 +1,6 @@  - netcoreapp1.0;net452;net46 + net48;netcoreapp3.1;net5.0 Serilog.Sinks.Seq.Tests ../../assets/Serilog.snk true @@ -8,9 +8,9 @@ true - - - + + PreserveNewest + PreserveNewest @@ -21,17 +21,13 @@ - - - - - - - - + + + + - +