From 19f8a496f8624c8187c9b552a9f2da4822df9567 Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Mon, 8 Sep 2014 09:07:18 -0400 Subject: [PATCH] Added type to GrokFilter --- TimberWinR.UnitTests/GrokFilterTests.cs | 2 +- TimberWinR/Filters/GrokFilter.cs | 11 +++- TimberWinR/Inputs/LogsListener.cs | 77 ++++++++++++++++--------- TimberWinR/Parser.cs | 4 ++ TimberWinR/mdocs/GrokFilter.md | 5 +- 5 files changed, 67 insertions(+), 32 deletions(-) diff --git a/TimberWinR.UnitTests/GrokFilterTests.cs b/TimberWinR.UnitTests/GrokFilterTests.cs index ae91d23..436b34c 100644 --- a/TimberWinR.UnitTests/GrokFilterTests.cs +++ b/TimberWinR.UnitTests/GrokFilterTests.cs @@ -30,7 +30,7 @@ namespace TimberWinR.UnitTests ""Filters"":[ { ""grok"":{ - ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""", + ""type"": ""Win32-FileLog"", ""match"":[ ""Text"", """" diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index a9d17cc..8d61da2 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -34,6 +34,13 @@ namespace TimberWinR.Parser { public override bool Apply(JObject json) { + if (!string.IsNullOrEmpty(Type)) + { + JToken json_type = json["type"]; + if (json_type != null && json_type.ToString() != Type) + return true; // Filter does not apply. + } + if (Condition != null && !EvaluateCondition(json, Condition)) return false; @@ -92,9 +99,7 @@ namespace TimberWinR.Parser AddOrModify(json, fieldName, fieldValue); } } - } - - + } private void RemoveFields(Newtonsoft.Json.Linq.JObject json) { diff --git a/TimberWinR/Inputs/LogsListener.cs b/TimberWinR/Inputs/LogsListener.cs index 1c4220d..648184f 100644 --- a/TimberWinR/Inputs/LogsListener.cs +++ b/TimberWinR/Inputs/LogsListener.cs @@ -32,8 +32,12 @@ namespace TimberWinR.Inputs _receivedMessages = 0; _arguments = arguments; _pollingIntervalInSeconds = pollingIntervalInSeconds; - var task = new Task(FileWatcher, cancelToken); - task.Start(); + + foreach (string srcFile in _arguments.Location.Split(',')) + { + string file = srcFile.Trim(); + Task.Factory.StartNew(() => FileWatcher(file)); + } } public override void Shutdown() @@ -55,40 +59,56 @@ namespace TimberWinR.Inputs return json; } - private void FileWatcher() + private void FileWatcher(string fileToWatch) { var iFmt = new TextLineInputFormat() { iCodepage = _arguments.CodePage, - splitLongLines = _arguments.SplitLongLines, - iCheckpoint = CheckpointFileName, + splitLongLines = _arguments.SplitLongLines, recurse = _arguments.Recurse }; - - // Create the query - var query = string.Format("SELECT * FROM {0}", _arguments.Location); - - var firstQuery = true; + + Dictionary logFileMaxRecords = new Dictionary(); + // Execute the query while (!CancelToken.IsCancellationRequested) { var oLogQuery = new LogQuery(); try - { - var rs = oLogQuery.Execute(query, iFmt); - Dictionary colMap = new Dictionary(); - for (int col=0; col {1}", fileName, lastRecordNumber); + + var rs = oLogQuery.Execute(query, iFmt); + Dictionary colMap = new Dictionary(); + for (int col = 0; col < rs.getColumnCount(); col++) + { + string colName = rs.getColumnName(col); + colMap[colName] = col; + } + + // Browse the recordset + for (; !rs.atEnd(); rs.moveNext()) + { var record = rs.getRecord(); var json = new JObject(); foreach (var field in _arguments.Fields) @@ -111,11 +131,15 @@ namespace TimberWinR.Inputs ProcessJson(json); _receivedMessages++; } + + var lrn = (Int64)record.getValueEx("Index"); + logFileMaxRecords[fileName] = lrn; } + + // Close the recordset + rs.close(); + rs = null; } - // Close the recordset - rs.close(); - rs = null; } catch (Exception ex) { @@ -125,7 +149,8 @@ namespace TimberWinR.Inputs { oLogQuery = null; } - firstQuery = false; + + Thread.CurrentThread.Priority = ThreadPriority.Normal; System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 044ace2..fd0a8ed 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -508,6 +508,10 @@ namespace TimberWinR.Parser { } } + + [JsonProperty("type")] + public string Type { get; set; } + [JsonProperty("condition")] public string Condition { get; set; } diff --git a/TimberWinR/mdocs/GrokFilter.md b/TimberWinR/mdocs/GrokFilter.md index 9b65e00..e215e23 100644 --- a/TimberWinR/mdocs/GrokFilter.md +++ b/TimberWinR/mdocs/GrokFilter.md @@ -26,6 +26,7 @@ The following operations are allowed when mutating a field. | Operation | Type | Description | :---------------|:----------------|:-----------------------------------------------------------------------| +| *type* | property:string |Type to which this filter applyes, if empty, applies to all types. | *condition* | property:string |C# expression | *match* | property:string |Required field must match before any subsequent grok operations are executed. | *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs. @@ -82,8 +83,8 @@ then the operation(s) will be executed in order. ```json "Filters": [ { - "grok": { - "condition": "\"[type]\" == \"Win32-EventLog\"" + "grok": { + "type": "Win32-EventLog", "add_field": [ "ComputerName", "%{Host}" ]