diff --git a/README.md b/README.md index fdf50c7..68f8f54 100644 --- a/README.md +++ b/README.md @@ -107,6 +107,7 @@ represented as a JSON Property or Array. 1. [Redis](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md) 2. [Elasticsearch](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md) 3. [Stdout](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md) +4. [File](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/FileOutput.md) ## Sample Configuration TimberWinR reads a JSON configuration file, an example file is shown here: diff --git a/TimberWinR.TestGenerator/UdpTestGenerator.cs b/TimberWinR.TestGenerator/UdpTestGenerator.cs index a8426b3..6e93cbf 100644 --- a/TimberWinR.TestGenerator/UdpTestGenerator.cs +++ b/TimberWinR.TestGenerator/UdpTestGenerator.cs @@ -1,4 +1,6 @@ -using System.Threading; +using System.Security.Cryptography; +using System.Threading; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using NLog.Config; @@ -52,13 +54,27 @@ namespace TimberWinR.TestGenerator {"Executable", "VP.Common.SvcFrm.Services.Host, Version=29.7.0.0, Culture=neutral, PublicKeyToken=null"}, {"RenderedMessage", "Responding to RequestSchedule message from 10.1.230.36 with Ack because: PRJ byte array is null."}, {"Team", "Manufacturing Software"}, + {"RecordNumber", i}, {"Host", hostName}, {"UtcTimestamp", DateTime.UtcNow.ToString("o")}, {"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"}, {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")}, {"Index", "logstash"} }; - byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString()); + + string hashedString = ""; + foreach(var key in o) + { + hashedString += key.ToString(); + } + + var source = ASCIIEncoding.ASCII.GetBytes(hashedString); + var md5 = new MD5CryptoServiceProvider().ComputeHash(source); + var hash = string.Concat(md5.Select(x => x.ToString("X2"))); + + o["md5"] = hash; + + byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None)); IPEndPoint ep = new IPEndPoint(broadcast, parms.Port); s.SendTo(sendbuf, ep); diff --git a/TimberWinR.TestGenerator/results5.json b/TimberWinR.TestGenerator/results5.json index 754b3a8..112222e 100644 --- a/TimberWinR.TestGenerator/results5.json +++ b/TimberWinR.TestGenerator/results5.json @@ -3,7 +3,7 @@ "Inputs": [ { "udp": { - "test1: message sent count": "[messages] == 10000", + "test1: message sent count": "[messages] == 80000", "test2: average cpu": "[avgCpuUsage] <= 30", "test3: maximum memory": "[maxMemUsage] <= 30" } diff --git a/TimberWinR.TestGenerator/test5-twconfig.json b/TimberWinR.TestGenerator/test5-twconfig.json index cd8030a..7f0cefa 100644 --- a/TimberWinR.TestGenerator/test5-twconfig.json +++ b/TimberWinR.TestGenerator/test5-twconfig.json @@ -9,7 +9,7 @@ "Environment", "PLANT_TST_TIMBERWINR" ], - "rename": [ + "rename": [ "Type", "type" ] @@ -25,16 +25,10 @@ ] }, "Outputs": { - "Redis": [ + "File": [ { - "_comment": "Change the host to your Redis instance", - "port": 6379, - "batch_count": 500, - "interval": 1000, - "threads": 4, - "host": [ - "tstlexiceapp006.mycompany.svc" - ] + "format": "indented", + "file_name": "test5_output.txt" } ] } diff --git a/TimberWinR.TestGenerator/test5.json b/TimberWinR.TestGenerator/test5.json index a431fe2..303f7f0 100644 --- a/TimberWinR.TestGenerator/test5.json +++ b/TimberWinR.TestGenerator/test5.json @@ -5,11 +5,11 @@ "--testFile": "test5.json", "--testDir": "test5", "--timberWinRConfig": "test5-twconfig.json", - "--numMessages": 20000, + "--numMessages": 80000, "--logLevel": "debug", "--udp-host": "localhost", "--udp": "5140", - "--udp-rate": 5, + "--udp-rate": 1, "--resultsFile": "results5.json" } } diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index cb37125..09a6ab4 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -58,6 +58,12 @@ namespace TimberWinR get { return _stdoutOutputs; } } + private List _fileOutputs = new List(); + public IEnumerable FileOutputs + { + get { return _fileOutputs; } + } + private List _tcps = new List(); public IEnumerable Tcps { @@ -265,6 +271,8 @@ namespace TimberWinR c._elasticsearchOutputs.AddRange(x.TimberWinR.Outputs.Elasticsearch.ToList()); if (x.TimberWinR.Outputs.Stdout != null) c._stdoutOutputs.AddRange(x.TimberWinR.Outputs.Stdout.ToList()); + if (x.TimberWinR.Outputs.File != null) + c._fileOutputs.AddRange(x.TimberWinR.Outputs.File.ToList()); } if (x.TimberWinR.Filters != null) @@ -302,6 +310,7 @@ namespace TimberWinR _redisOutputs = new List(); _elasticsearchOutputs = new List(); _stdoutOutputs = new List(); + _fileOutputs = new List(); _tcps = new List(); _udps = new List(); } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index a678de7..258c21f 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -190,6 +190,15 @@ namespace TimberWinR } } + if (config.FileOutputs != null) + { + foreach (var ro in config.FileOutputs) + { + var output = new FileOutput(this, ro, cancelToken); + Outputs.Add(output); + } + } + foreach (Parser.IISW3CLogParameters iisw3cConfig in config.IISW3C) { var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken); diff --git a/TimberWinR/Outputs/File.cs b/TimberWinR/Outputs/File.cs new file mode 100644 index 0000000..f7d6cf4 --- /dev/null +++ b/TimberWinR/Outputs/File.cs @@ -0,0 +1,140 @@ +using System.IO; +using System.Text; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NLog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading; +using System.Threading.Tasks; + +namespace TimberWinR.Outputs +{ + public class FileOutput : OutputSender + { + private TimberWinR.Manager _manager; + private readonly int _interval; + private readonly object _locker = new object(); + private readonly List _jsonQueue; + private long _sentMessages; + private Parser.FileOutputParameters _arguments; + public bool Stop { get; set; } + + public FileOutput(TimberWinR.Manager manager, Parser.FileOutputParameters arguments, CancellationToken cancelToken) + : base(cancelToken, "File") + { + _arguments = arguments; + _sentMessages = 0; + _manager = manager; + _interval = arguments.Interval; + _jsonQueue = new List(); + + var elsThread = new Task(FileSender, cancelToken); + elsThread.Start(); + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("file-output", + new JObject( + new JProperty("queuedMessageCount", _jsonQueue.Count), + new JProperty("sentMessageCount", _sentMessages)))); + + return json; + } + + // + // Pull off messages from the Queue, batch them up and send them all across + // + private void FileSender() + { + + using (var syncHandle = new ManualResetEventSlim()) + { + var fi = new FileInfo(_arguments.FileName); + if (File.Exists(_arguments.FileName)) + File.Delete(_arguments.FileName); + + LogManager.GetCurrentClassLogger().Info("File Output Sending To: {0}", fi.FullName); + + using (StreamWriter sw = File.AppendText(_arguments.FileName)) + { + // Execute the query + while (!Stop) + { + if (!CancelToken.IsCancellationRequested) + { + try + { + JObject[] messages; + lock (_locker) + { + messages = _jsonQueue.Take(_jsonQueue.Count).ToArray(); + _jsonQueue.RemoveRange(0, messages.Length); + } + + if (messages.Length > 0) + { + try + { + foreach (JObject obj in messages) + { + sw.WriteLine(obj.ToString(_arguments.ToFormat())); + _sentMessages++; + } + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + } + } + if (!Stop) + syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception) + { + } + } + } + } + } + } + + protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage) + { + if (_manager.Config.Filters != null) + { + if (ApplyFilters(jsonMessage)) + return; + } + + var message = jsonMessage.ToString(); + + lock (_locker) + { + _jsonQueue.Add(jsonMessage); + } + } + + private bool ApplyFilters(JObject json) + { + bool drop = false; + + foreach (var filter in _manager.Config.Filters) + { + if (!filter.Apply(json)) + drop = true; + } + + return drop; + } + + } +} + diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 47cd82f..5bdc6e5 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -43,7 +43,7 @@ namespace TimberWinR.Parser { JToken token = json[oldName]; if (token != null) - { + { json.Add(newName, token.DeepClone()); json.Remove(oldName); } @@ -679,6 +679,43 @@ namespace TimberWinR.Parser } } + public class FileOutputParameters + { + public enum FormatKind + { + none, indented + }; + + [JsonProperty(PropertyName = "interval")] + public int Interval { get; set; } + + [JsonProperty(PropertyName = "file_name")] + public string FileName { get; set; } + + [JsonProperty(PropertyName = "format")] + public FormatKind Format { get; set; } + + public FileOutputParameters() + { + Format = FormatKind.none; + Interval = 1000; + FileName = "timberwinr.out"; + } + + public Newtonsoft.Json.Formatting ToFormat() + { + switch (Format) + { + case FormatKind.indented: + return Newtonsoft.Json.Formatting.Indented; + + case FormatKind.none: + default: + return Newtonsoft.Json.Formatting.None; + } + } + } + public class OutputTargets { [JsonProperty("Redis")] @@ -689,6 +726,9 @@ namespace TimberWinR.Parser [JsonProperty("Stdout")] public StdoutOutputParameters[] Stdout { get; set; } + + [JsonProperty("File")] + public FileOutputParameters[] File { get; set; } } public class InputSources diff --git a/TimberWinR/ReleaseNotes.md b/TimberWinR/ReleaseNotes.md index 12a3b06..05d71b6 100644 --- a/TimberWinR/ReleaseNotes.md +++ b/TimberWinR/ReleaseNotes.md @@ -8,6 +8,7 @@ Version / Date 2. Fixed potential non-thread safe when renaming properties 3. Added add_field, rename support to Udp/Tcp Input Listeners 4. Fixed issue with multiple renames (was previously only renaming the first one) +5. Added File outputter for testing. ### 1.3.24.0 - 2015-04-29 1. Fixed potential bug in TailFiles when tailing log files which are partially flushed diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index 0ad74d3..8803ca9 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -115,6 +115,7 @@ + @@ -141,6 +142,7 @@ + diff --git a/TimberWinR/mdocs/FileOutput.md b/TimberWinR/mdocs/FileOutput.md new file mode 100644 index 0000000..753377f --- /dev/null +++ b/TimberWinR/mdocs/FileOutput.md @@ -0,0 +1,27 @@ +# Output: File + +The File output passes on data into a text file. + +## Parameters +The following parameters are allowed when configuring the File output. + +| Parameter | Type | Description | Details | Default | +| :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- | +| *interval* | integer | Interval in milliseconds to sleep before appending data | Interval | 1000 | +| *file_name* | string | Name of the file to be created | | timberwinr.out | + +Example Input: +```json +{ + "TimberWinR": { + "Outputs": { + "File": [ + { + "file_name": "foo.out", + "interval": 1000 + } + ] + } + } +} +``` \ No newline at end of file