From 94ca9b40a21da8a3f6bbd2f9875f3e8a335c1006 Mon Sep 17 00:00:00 2001 From: Jonathan Preddy Date: Mon, 4 Aug 2014 14:07:16 -0400 Subject: [PATCH] Adding stdout output --- .../TimberWinR.ServiceHost.csproj | 3 + TimberWinR.ServiceHost/loopback.json | 29 +++++++ TimberWinR/Configuration.cs | 9 +- TimberWinR/Manager.cs | 8 ++ TimberWinR/Outputs/Elasticsearch.cs | 1 + TimberWinR/Outputs/Stdout.cs | 85 +++++++++++++++++++ TimberWinR/Parser.cs | 21 ++++- TimberWinR/TimberWinR.csproj | 1 + 8 files changed, 154 insertions(+), 3 deletions(-) create mode 100644 TimberWinR.ServiceHost/loopback.json create mode 100644 TimberWinR/Outputs/Stdout.cs diff --git a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj index b3c5b3c..33bd0d5 100644 --- a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj +++ b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj @@ -59,6 +59,9 @@ PreserveNewest + + PreserveNewest + Designer diff --git a/TimberWinR.ServiceHost/loopback.json b/TimberWinR.ServiceHost/loopback.json new file mode 100644 index 0000000..1497307 --- /dev/null +++ b/TimberWinR.ServiceHost/loopback.json @@ -0,0 +1,29 @@ +{ + "TimberWinR": { + "Inputs": { + "Stdin": [ + { + } + ] + }, + "Outputs": { + "Stdout": [ + { + } + ] + }, + "Filters": [ + { + "grok": { + "match": [ + "message", + "" + ], + "add_field": [ + "ComputerName", "test" + ] + } + } + ] + } +} diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 81c209f..62c0512 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -45,6 +45,11 @@ namespace TimberWinR get { return _elasticsearchOutputs; } } + private List _stdoutOutputs = new List(); + public IEnumerable StdoutOutputs + { + get { return _stdoutOutputs; } + } private List _tcps = new List(); public IEnumerable Tcps @@ -136,9 +141,10 @@ namespace TimberWinR { if (x.TimberWinR.Outputs.Redis != null) c._redisOutputs = x.TimberWinR.Outputs.Redis.ToList(); - if (x.TimberWinR.Outputs.Elasticsearch != null) c._elasticsearchOutputs = x.TimberWinR.Outputs.Elasticsearch.ToList(); + if (x.TimberWinR.Outputs.Stdout != null) + c._stdoutOutputs = x.TimberWinR.Outputs.Stdout.ToList(); } if (x.TimberWinR.Filters != null) @@ -175,6 +181,7 @@ namespace TimberWinR _logs = new List(); _redisOutputs = new List(); _elasticsearchOutputs = new List(); + _stdoutOutputs = new List(); _tcps = new List(); } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index ba37118..eda4ae1 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -95,6 +95,14 @@ namespace TimberWinR Outputs.Add(els); } } + if (Config.StdoutOutputs != null) + { + foreach (var ro in Config.StdoutOutputs) + { + var stdout = new StdoutOutput(this, ro, cancelToken); + Outputs.Add(stdout); + } + } foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C) { diff --git a/TimberWinR/Outputs/Elasticsearch.cs b/TimberWinR/Outputs/Elasticsearch.cs index 1d91635..cee2694 100644 --- a/TimberWinR/Outputs/Elasticsearch.cs +++ b/TimberWinR/Outputs/Elasticsearch.cs @@ -31,6 +31,7 @@ namespace TimberWinR.Outputs _timeout = eo.Timeout; _manager = manager; _port = eo.Port; + _interval = eo.Interval; _host = eo.Host; _index = eo.Index; _hostIndex = 0; diff --git a/TimberWinR/Outputs/Stdout.cs b/TimberWinR/Outputs/Stdout.cs new file mode 100644 index 0000000..91da0f0 --- /dev/null +++ b/TimberWinR/Outputs/Stdout.cs @@ -0,0 +1,85 @@ +using Newtonsoft.Json.Linq; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace TimberWinR.Outputs +{ + public class StdoutOutput : OutputSender + { + private TimberWinR.Manager _manager; + private readonly int _interval; + private readonly object _locker = new object(); + private readonly List _jsonQueue; + + public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken) + : base(cancelToken) + { + _manager = manager; + _interval = eo.Interval; + _jsonQueue = new List(); + + var elsThread = new Task(StdoutSender, cancelToken); + elsThread.Start(); + } + + // + // Pull off messages from the Queue, batch them up and send them all across + // + private void StdoutSender() + { + while (!CancelToken.IsCancellationRequested) + { + JObject[] messages; + lock (_locker) + { + messages = _jsonQueue.Take(1).ToArray(); + _jsonQueue.RemoveRange(0, messages.Length); + } + + if (messages.Length > 0) + { + try + { + foreach (JObject obj in messages) + { + Console.WriteLine(obj.ToString()); + } + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + } + } + System.Threading.Thread.Sleep(_interval); + } + } + + protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage) + { + if (_manager.Config.Filters != null) + ApplyFilters(jsonMessage); + + var message = jsonMessage.ToString(); + LogManager.GetCurrentClassLogger().Debug(message); + + lock (_locker) + { + _jsonQueue.Add(jsonMessage); + } + } + + private void ApplyFilters(JObject json) + { + foreach (var filter in _manager.Config.Filters) + { + filter.Apply(json); + } + } + + } +} + diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 5b5b263..53e368f 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -406,6 +406,8 @@ namespace TimberWinR.Parser public int NumThreads { get; set; } [JsonProperty(PropertyName = "protocol")] public string Protocol { get; set; } + [JsonProperty(PropertyName = "interval")] + public int Interval { get; set; } public ElasticsearchOutput() { @@ -414,7 +416,8 @@ namespace TimberWinR.Parser Index = ""; Host = new string[] { "localhost" }; Timeout = 10000; - NumThreads = 1; + NumThreads = 1; + Interval = 1000; } } @@ -446,7 +449,18 @@ namespace TimberWinR.Parser Interval = 5000; } } - + + public class StdoutOutput + { + [JsonProperty(PropertyName = "interval")] + public int Interval { get; set; } + + public StdoutOutput() + { + Interval = 1000; + } + } + public class OutputTargets { [JsonProperty("Redis")] @@ -454,6 +468,9 @@ namespace TimberWinR.Parser [JsonProperty("Elasticsearch")] public ElasticsearchOutput[] Elasticsearch { get; set; } + + [JsonProperty("Stdout")] + public StdoutOutput[] Stdout { get; set; } } public class InputSources diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index e4d546b..1120911 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -84,6 +84,7 @@ +