diff --git a/README.md b/README.md index efc5fe9..8d48dd3 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,7 @@ TimberWinR reads a JSON configuration file, an example file is shown here: "Filters": [ { "grok": { - "condition": "[type] == \"Win32-Eventlog\"", + "condition": "\"[type]\" == \"Win32-Eventlog\"", "match": [ "Message", "" diff --git a/TimberWinR.ServiceHost/config.json b/TimberWinR.ServiceHost/config.json index 187b0fa..41aec34 100644 --- a/TimberWinR.ServiceHost/config.json +++ b/TimberWinR.ServiceHost/config.json @@ -23,7 +23,7 @@ "Filters": [ { "grok": { - "condition": "[type] == \"Win32-Eventlog\"", + "condition": "\"[type]\" == \"Win32-Eventlog\"", "match": [ "Message", "" @@ -63,7 +63,7 @@ }, { "date": { - "condition": "[type] == \"Win32-FileLog\"", + "condition": "\"[type]\" == \"Win32-FileLog\"", "match": [ "timestamp", "MMM d HH:mm:sss", diff --git a/TimberWinR.ServiceHost/default.json b/TimberWinR.ServiceHost/default.json index c726aa9..fd463ee 100644 --- a/TimberWinR.ServiceHost/default.json +++ b/TimberWinR.ServiceHost/default.json @@ -3,7 +3,7 @@ "Inputs": { "WindowsEvents": [ { - "source": "Application", + "source": "Application,System", "binaryFormat": "PRINT", "resolveSIDS": true } @@ -15,13 +15,25 @@ } ] }, + "Filters":[ + { + "grok":{ + "condition": "[EventTypeName] == \"Information Event\"", + "match":[ + "Text", + "" + ], + "drop": "true" + } + }] + }, "Outputs": { "Redis": [ { "_comment": "Change the host to your Redis instance", "port": 6379, "host": [ - "tstlexiceapp006.vistaprint.svc" + "logaggregator.vistaprint.svc" ] } ] diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs index 8c25a51..ae91d23 100644 --- a/TimberWinR.UnitTests/GrokFilterTests.cs +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -30,7 +30,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -79,7 +79,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -129,7 +129,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -147,7 +147,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type].Contains(\""Win32-FileLog\"")"", + ""condition"": ""\""[type]\"".Contains(\""Win32-FileLog\"")"", ""match"":[ ""Text"", """" @@ -166,7 +166,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type].Contains(\""Win32-Filelog\"")"", + ""condition"": ""\""[type]\"".Contains(\""Win32-Filelog\"")"", ""match"":[ ""Text"", """" @@ -217,7 +217,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -248,7 +248,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -274,7 +274,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -300,7 +300,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" @@ -330,7 +330,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""[type] == \""Win32-FileLog\"""", + ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", ""match"":[ ""Text"", """" diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index df13588..a9d17cc 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -39,15 +39,18 @@ namespace TimberWinR.Parser if (Matches(json)) { + if (DropIfMatch) + return false; + AddFields(json); AddTags(json); RemoveFields(json); - RemoveTags(json); + RemoveTags(json); return true; } return false; } - + private bool Matches(Newtonsoft.Json.Linq.JObject json) { string field = Match[0]; diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index deda962..90bfcfa 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -68,6 +68,7 @@ namespace TimberWinR.Inputs LogQuery oLogQuery = new LogQuery(); LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready"); + // Instantiate the Event Log Input Format object var iFmt = new EventLogInputFormat() @@ -81,50 +82,71 @@ namespace TimberWinR.Inputs stringsSep = _arguments.StringsSep, resolveSIDs = _arguments.ResolveSIDS }; - - var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", location); - var rcount = oLogQuery.Execute(qcount, iFmt); - var qr = rcount.getRecord(); - var lastRecordNumber = qr.getValueEx("MaxRecordNumber"); - + oLogQuery = null; - + + Dictionary logFileMaxRecords = new Dictionary(); + + // Execute the query while (!CancelToken.IsCancellationRequested) { try { + oLogQuery = new LogQuery(); + Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; - oLogQuery = new LogQuery(); - var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber); - - var rs = oLogQuery.Execute(query, iFmt); - // Browse the recordset - for (; !rs.atEnd(); rs.moveNext()) - { - - var record = rs.getRecord(); - var json = new JObject(); - foreach (var field in _arguments.Fields) + var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location); + var rsfiles = oLogQuery.Execute(qfiles, iFmt); + for (; !rsfiles.atEnd(); rsfiles.moveNext()) + { + var record = rsfiles.getRecord(); + string logName = record.getValue("EventLog") as string; + if (!logFileMaxRecords.ContainsKey(logName)) { - object v = record.getValue(field.Name); - if (field.Name == "Data") - v = ToPrintable(v.ToString()); - json.Add(new JProperty(field.Name, v)); + var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName); + var rcount = oLogQuery.Execute(qcount, iFmt); + var qr = rcount.getRecord(); + var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); + logFileMaxRecords[logName] = lrn; } - - lastRecordNumber = record.getValueEx("RecordNumber"); - - record = null; - ProcessJson(json); - _receivedMessages++; - json = null; - } - // Close the recordset - rs.close(); - rs = null; + + + foreach (string fileName in logFileMaxRecords.Keys.ToList()) + { + var lastRecordNumber = logFileMaxRecords[fileName]; + var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber); + + var rs = oLogQuery.Execute(query, iFmt); + // Browse the recordset + for (; !rs.atEnd(); rs.moveNext()) + { + + var record = rs.getRecord(); + var json = new JObject(); + foreach (var field in _arguments.Fields) + { + object v = record.getValue(field.Name); + if (field.Name == "Data") + v = ToPrintable(v.ToString()); + json.Add(new JProperty(field.Name, v)); + } + + var lrn = (Int64)record.getValueEx("RecordNumber"); + logFileMaxRecords[fileName] = lrn; + + record = null; + ProcessJson(json); + _receivedMessages++; + json = null; + + } + // Close the recordset + rs.close(); + rs = null; + } } catch (Exception ex) { diff --git a/TimberWinR/Outputs/Redis.cs b/TimberWinR/Outputs/Redis.cs index 415a1e9..a166a9f 100644 --- a/TimberWinR/Outputs/Redis.cs +++ b/TimberWinR/Outputs/Redis.cs @@ -124,7 +124,10 @@ namespace TimberWinR.Outputs protected override void MessageReceivedHandler(JObject jsonMessage) { if (_manager.Config.Filters != null) - ApplyFilters(jsonMessage); + { + if (ApplyFilters(jsonMessage)) + return; + } var message = jsonMessage.ToString(); LogManager.GetCurrentClassLogger().Debug(message); @@ -135,12 +138,18 @@ namespace TimberWinR.Outputs } } - private void ApplyFilters(JObject json) + private bool ApplyFilters(JObject json) { + bool drop = false; foreach (var filter in _manager.Config.Filters) { - filter.Apply(json); + if (!filter.Apply(json)) + { + LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString()); + drop = true; + } } + return drop; } // diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 53e368f..044ace2 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -41,7 +41,7 @@ namespace TimberWinR.Parser IList keys = json.Properties().Select(p => p.Name).ToList(); foreach (string key in keys) - cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString())); + cond = cond.Replace(string.Format("[{0}]", key), string.Format("{0}", json[key].ToString())); var compiler = new CSharpCodeProvider(); @@ -511,7 +511,7 @@ namespace TimberWinR.Parser [JsonProperty("condition")] public string Condition { get; set; } - [JsonProperty("drop_if_match")] + [JsonProperty("drop")] public bool DropIfMatch { get; set; } [JsonProperty("match")] diff --git a/TimberWinR/mdocs/DateFilter.md b/TimberWinR/mdocs/DateFilter.md index 3678acc..b6381c4 100644 --- a/TimberWinR/mdocs/DateFilter.md +++ b/TimberWinR/mdocs/DateFilter.md @@ -34,7 +34,7 @@ Given this configuration "Filters": [ { "date": { - "condition": "[type] == \"Win32-FileLog\"", + "condition": "\"[type]\" == \"Win32-FileLog\"", "match": [ "timestamp", "MMM d HH:mm:sss", @@ -56,7 +56,7 @@ then the operation(s) will be executed in order. "Filters": [ { "grok": { - "condition": "[type] == \"Win32-EventLog\"" + "condition": "\"[type]\" == \"Win32-EventLog\"" "add_field": [ "ComputerName", "%{Host}" ] @@ -72,7 +72,7 @@ The fields must be in pairs with fieldName first and value second. "Filters": [ { "date": { - "condition": "[type] == \"Win32-FileLog\"", + "condition": "\"[type]\" == \"Win32-FileLog\"", "match": [ "timestamp", "MMM d HH:mm:sss", @@ -92,7 +92,7 @@ If true and the filter matches, the time parsed will be converted to UTC "Filters": [ { "date": { - "condition": "[type] == \"Win32-FileLog\"", + "condition": "\"[type]\" == \"Win32-FileLog\"", "match": [ "timestamp", "MMM d HH:mm:sss", diff --git a/TimberWinR/mdocs/Filters.md b/TimberWinR/mdocs/Filters.md index d56b31a..92a5846 100644 --- a/TimberWinR/mdocs/Filters.md +++ b/TimberWinR/mdocs/Filters.md @@ -12,7 +12,7 @@ Example Input: "Filters": [ { "grok": { - "condition": "[type] == \"Win32-Eventlog\"", + "condition": "\"[type]\" == \"Win32-Eventlog\"", "match": [ "Message", "" @@ -35,7 +35,7 @@ Example Input: }, { "date": { - "condition": "[type] == \"Win32-FileLog\"", + "condition": "\"[type]\" == \"Win32-FileLog\"", "match": [ "timestamp", "MMM d HH:mm:sss", diff --git a/TimberWinR/mdocs/GrokFilter.md b/TimberWinR/mdocs/GrokFilter.md index f61a5d0..9b65e00 100644 --- a/TimberWinR/mdocs/GrokFilter.md +++ b/TimberWinR/mdocs/GrokFilter.md @@ -83,7 +83,7 @@ then the operation(s) will be executed in order. "Filters": [ { "grok": { - "condition": "[type] == \"Win32-EventLog\"" + "condition": "\"[type]\" == \"Win32-EventLog\"" "add_field": [ "ComputerName", "%{Host}" ] diff --git a/TimberWinR/mdocs/MutateFilter.md b/TimberWinR/mdocs/MutateFilter.md index 788276a..3a70b45 100644 --- a/TimberWinR/mdocs/MutateFilter.md +++ b/TimberWinR/mdocs/MutateFilter.md @@ -20,7 +20,7 @@ then the operation(s) will be executed in order. "Filters": [ { "mutate": { - "condition": "[type] == \"Win32-EventLog\"" + "condition": "\"[type]\" == \"Win32-EventLog\"" "rename": [ "ComputerName", "Host" ]