diff --git a/README.md b/README.md index cb79577..465a96c 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,68 @@ TimberWinR ========== +A Native Windows to Redis Logstash Agent which runs as a service. -A Native Windows to Redis Logstash Agent +## Why have TimberWinR? +TimberWinR is a native .NET implementation utilizing Microsoft's [LogParser](http://technet.microsoft.com/en-us/scriptcenter/dd919274.aspx). This means +no JVM/JRuby is required, and LogParser does all the heavy lifting. TimberWinR collects +the data from LogParser and ships it to Logstash via Redis. + +## Configuration +TimberWinR reads a JSON configuration file, an example file is shown here: + + { + "TimberWinR": { + "Inputs": { + "WindowsEvents": [ + { + "source": "System,Application", + "binaryFormat": "PRINT", + "resolveSIDS": true + } + ] + }, + "Outputs": { + "Redis": [ + { + "host": [ + "server1.host.com" + ] + } + ] + } + } +This configuration collects Events from the Windows Event Logs (System, Application) and forwards them +to Redis. + +## what is Markdown? +see [Wikipedia](http://en.wikipedia.org/wiki/Markdown) + +> Markdown is a lightweight markup language, originally created by John Gruber and Aaron Swartz allowing people "to write using an easy-to-read, easy-to-write plain text format, then convert it to structurally valid XHTML (or HTML)". + +---- +## usage +1. Write markdown text in this textarea. +2. Click 'HTML Preview' button. + +---- +## markdown quick reference +# headers + +*emphasis* + +**strong** + +* list + +>block quote + + code (4 spaces indent) +[links](http://wikipedia.org) + +---- +## changelog +* 17-Feb-2013 re-design + +---- +## thanks +* [markdown-js](https://github.com/evilstreak/markdown-js) \ No newline at end of file diff --git a/TimberWinR.ServiceHost/config.json b/TimberWinR.ServiceHost/config.json index c3aa13d..23da196 100644 --- a/TimberWinR.ServiceHost/config.json +++ b/TimberWinR.ServiceHost/config.json @@ -28,9 +28,12 @@ }, "Outputs": { "Redis": [ - { + { + "threads": 1, + "interval": 5000, + "batch_count": 500, "host": [ - "logaggregator.vistaprint.svc" + "tstlexiceapp006.vistaprint.svc" ] } ] @@ -42,7 +45,7 @@ "match": [ "Message", "" - ], + ], "remove_field": [ "ComputerName" ] @@ -78,13 +81,15 @@ }, { "date": { - "condition": "[type] == \"Win32-IISLog\"", + "condition": "[type] == \"Win32-FileLog\"", "match": [ "timestamp", "MMM d HH:mm:sss", "MMM dd HH:mm:ss" ], - "target": "UtcTimestamp", + "add_field": [ + "UtcTimestamp" + ], "convertToUTC": true } }, @@ -99,8 +104,7 @@ "SID", "Username" ] } - }, - + } ] } } diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs index b72ca22..3ee2665 100644 --- a/TimberWinR/Filters/DateFilter.cs +++ b/TimberWinR/Filters/DateFilter.cs @@ -22,6 +22,7 @@ namespace TimberWinR.Parser if (Matches(json)) { ApplyFilter(json); + AddFields(json); } return true; @@ -48,9 +49,28 @@ namespace TimberWinR.Parser } } + // copy_field "field1" -> "field2" + private void AddFields(Newtonsoft.Json.Linq.JObject json) + { + string srcField = Match[0]; + + if (AddField != null && AddField.Length > 0) + { + for (int i = 0; i < AddField.Length; i++) + { + string dstField = ExpandField(AddField[i], json); + if (json[srcField] != null) + AddOrModify(json, dstField, json[srcField]); + } + } + } + + private bool Matches(Newtonsoft.Json.Linq.JObject json) { - string field = Match[0]; + string field = Match[0]; + + CultureInfo ci = new CultureInfo(Locale); JToken token = null; if (json.TryGetValue(field, out token)) @@ -66,7 +86,7 @@ namespace TimberWinR.Parser var pattern = resolver.ResolveToRegex(exprArray[i]); exprArray[i] = pattern; } - if (DateTime.TryParseExact(text, exprArray, CultureInfo.InvariantCulture,DateTimeStyles.None, out ts)) + if (DateTime.TryParseExact(text, exprArray, ci,DateTimeStyles.None, out ts)) AddOrModify(json, ts); } return true; // Empty field is no match diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index d286ec6..df13588 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -40,7 +40,7 @@ namespace TimberWinR.Parser if (Matches(json)) { AddFields(json); - AddTags(json); + AddTags(json); RemoveFields(json); RemoveTags(json); return true; @@ -91,6 +91,8 @@ namespace TimberWinR.Parser } } + + private void RemoveFields(Newtonsoft.Json.Linq.JObject json) { if (RemoveField != null && RemoveField.Length > 0) diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index 4c38191..767c170 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -36,7 +36,7 @@ namespace TimberWinR.Inputs public override void Shutdown() { base.Shutdown(); - } + } private void IISW3CWatcher() { diff --git a/TimberWinR/Inputs/InputListener.cs b/TimberWinR/Inputs/InputListener.cs index 4c3a402..5f4a4e9 100644 --- a/TimberWinR/Inputs/InputListener.cs +++ b/TimberWinR/Inputs/InputListener.cs @@ -32,6 +32,22 @@ namespace TimberWinR.Inputs .ToString(); } + protected string ToPrintable(string inputString) + { + string asAscii = Encoding.ASCII.GetString( + Encoding.Convert( + Encoding.UTF8, + Encoding.GetEncoding( + Encoding.ASCII.EncodingName, + new EncoderReplacementFallback(string.Empty), + new DecoderExceptionFallback() + ), + Encoding.UTF8.GetBytes(inputString) + ) + ); + return asAscii; + } + public void Finished() { FinishedEvent.Set(); @@ -57,6 +73,9 @@ namespace TimberWinR.Inputs if (json["host"] == null) json.Add(new JProperty("host", _computerName)); + if (json["@version"] == null) + json.Add(new JProperty("@version", 1)); + if (json["@timestamp"] == null) json.Add(new JProperty("@timestamp", DateTime.UtcNow)); } diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index cf6a1af..29f5e2f 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -81,6 +81,8 @@ namespace TimberWinR.Inputs foreach (var field in _arguments.Fields) { object v = record.getValue(field.Name); + if (field.Name == "Data") + v = ToPrintable(v.ToString()); json.Add(new JProperty(field.Name, v)); } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index 032802e..567f101 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -58,7 +58,7 @@ namespace TimberWinR { foreach (var ro in Config.RedisOutputs) { - var redis = new RedisOutput(this, ro.Host, cancelToken, ro.Index, ro.Port, ro.Timeout); + var redis = new RedisOutput(this, ro, cancelToken); Outputs.Add(redis); } } @@ -108,7 +108,7 @@ namespace TimberWinR return new FileTarget { ArchiveEvery = FileArchivePeriod.None, - ArchiveAboveSize = 10 * 1024 * 1024, + ArchiveAboveSize = 5 * 1024 * 1024, MaxArchiveFiles = 5, BufferSize = 10, FileName = Path.Combine(logPath, "TimberWinR", "TimberWinR.txt"), diff --git a/TimberWinR/Outputs/Redis.cs b/TimberWinR/Outputs/Redis.cs index dd09c46..afae312 100644 --- a/TimberWinR/Outputs/Redis.cs +++ b/TimberWinR/Outputs/Redis.cs @@ -23,10 +23,12 @@ namespace TimberWinR.Outputs private readonly int _timeout; private readonly object _locker = new object(); private readonly List _jsonQueue; - readonly Task _consumerTask; + // readonly Task _consumerTask; private readonly string[] _redisHosts; private int _redisHostIndex; private TimberWinR.Manager _manager; + private readonly int _batchCount; + private readonly int _interval; /// /// Get the next client @@ -50,9 +52,8 @@ namespace TimberWinR.Outputs return client; } - catch (Exception ex) + catch (Exception ) { - } numTries++; } @@ -60,18 +61,24 @@ namespace TimberWinR.Outputs return null; } - public RedisOutput(TimberWinR.Manager manager, string[] redisHosts, CancellationToken cancelToken, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000) + public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken) //string[] redisHosts, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000, int batch_count = 10) : base(cancelToken) { + _batchCount = ro.BatchCount; _manager = manager; _redisHostIndex = 0; - _redisHosts = redisHosts; + _redisHosts = ro.Host; _jsonQueue = new List(); - _port = port; - _timeout = timeout; - _logstashIndexName = logstashIndexName; - _consumerTask = new Task(RedisSender, cancelToken); - _consumerTask.Start(); + _port = ro.Port; + _timeout = ro.Timeout; + _logstashIndexName = ro.Index; + _interval = ro.Interval; + + for (int i = 0; i < ro.NumThreads; i++) + { + var redisThread = new Task(RedisSender, cancelToken); + redisThread.Start(); + } } @@ -82,10 +89,10 @@ namespace TimberWinR.Outputs protected override void MessageReceivedHandler(JObject jsonMessage) { if (_manager.Config.Filters != null) - ProcessGroks(jsonMessage); + ApplyFilters(jsonMessage); var message = jsonMessage.ToString(); - LogManager.GetCurrentClassLogger().Info(message); + LogManager.GetCurrentClassLogger().Trace(message); lock (_locker) { @@ -93,11 +100,11 @@ namespace TimberWinR.Outputs } } - private void ProcessGroks(JObject json) + private void ApplyFilters(JObject json) { - foreach (var grok in _manager.Config.Filters) + foreach (var filter in _manager.Config.Filters) { - grok.Apply(json); + filter.Apply(json); } } @@ -111,8 +118,8 @@ namespace TimberWinR.Outputs string[] messages; lock (_locker) { - messages = _jsonQueue.ToArray(); - _jsonQueue.Clear(); + messages = _jsonQueue.Take(_batchCount).ToArray(); + _jsonQueue.RemoveRange(0, messages.Length); } if (messages.Length > 0) @@ -128,15 +135,18 @@ namespace TimberWinR.Outputs if (client != null) { client.StartPipe(); + LogManager.GetCurrentClassLogger() + .Info("Sending {0} Messages to {1}", messages.Length, client.Host); foreach (string jsonMessage in messages) { try - { + { client.RPush(_logstashIndexName, jsonMessage); } - catch (SocketException) + catch (SocketException ex) { + LogManager.GetCurrentClassLogger().Warn(ex); } } client.EndPipe(); @@ -156,7 +166,7 @@ namespace TimberWinR.Outputs } } } - System.Threading.Thread.Sleep(1000); + System.Threading.Thread.Sleep(_interval); } } } diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 6802e1a..6cfcaf6 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -121,6 +121,15 @@ namespace TimberWinR.Parser json[fieldName] = fieldValue; } + protected void AddOrModify(JObject json, string fieldName, JToken token) + { + if (json[fieldName] == null) + json.Add(fieldName, token); + else + json[fieldName] = token; + } + + protected string ExpandField(string fieldName, JObject json) { foreach (var token in json.Children()) @@ -228,6 +237,8 @@ namespace TimberWinR.Parser StringsSep = "|"; FormatMsg = true; FullText = true; + BinaryFormat = FormatKinds.ASC; + Fields = new List(); Fields.Add(new Field("EventLog", "string")); Fields.Add(new Field("RecordNumber", "int")); @@ -374,6 +385,12 @@ namespace TimberWinR.Parser public int Port { get; set; } [JsonProperty(PropertyName = "timeout")] public int Timeout { get; set; } + [JsonProperty(PropertyName = "batch_count")] + public int BatchCount { get; set; } + [JsonProperty(PropertyName = "threads")] + public int NumThreads { get; set; } + [JsonProperty(PropertyName = "interval")] + public int Interval { get; set; } public RedisOutput() { @@ -381,6 +398,9 @@ namespace TimberWinR.Parser Index = "logstash"; Host = new string[] {"localhost"}; Timeout = 10000; + BatchCount = 10; + NumThreads = 1; + Interval = 5000; } } @@ -435,7 +455,7 @@ namespace TimberWinR.Parser public string[] AddTag { get; set; } [JsonProperty("add_field")] - public string[] AddField { get; set; } + public string[] AddField { get; set; } [JsonProperty("remove_field")] public string[] RemoveField { get; set; } @@ -474,6 +494,9 @@ namespace TimberWinR.Parser [JsonProperty("condition")] public string Condition { get; set; } + [JsonProperty("locale")] + public string Locale { get; set; } + [JsonProperty("match")] public string[] Match { get; set; } @@ -486,6 +509,9 @@ namespace TimberWinR.Parser [JsonProperty("pattern")] public string[] Patterns { get; set; } + [JsonProperty("add_field")] + public string[] AddField { get; set; } + public override void Validate() { if (Match == null || Match.Length < 2) @@ -494,6 +520,12 @@ namespace TimberWinR.Parser if (string.IsNullOrEmpty(Target)) throw new DateFilterTargetException(); } + + public DateFilter() + { + Target = "timestamp"; + Locale = "en-US"; + } } public partial class Mutate : LogstashFilter