TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added type to GrokFilter

Browse Source
This commit is contained in:
Eric Fontana
2014-09-08 09:07:18 -04:00
parent e1f87678d0
commit 19f8a496f8
5 changed files with 67 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
TimberWinR.UnitTests/GrokFilterTests.cs
View File

@@ -30,7 +30,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""type"": ""Win32-FileLog"",
""match"":[
""Text"",
""""

11
TimberWinR/Filters/GrokFilter.cs
View File

@@ -34,6 +34,13 @@ namespace TimberWinR.Parser
{
public override bool Apply(JObject json)
{
if (!string.IsNullOrEmpty(Type))
{
JToken json_type = json["type"];
if (json_type != null && json_type.ToString() != Type)
return true; // Filter does not apply.
}
if (Condition != null && !EvaluateCondition(json, Condition))
return false;
@@ -92,9 +99,7 @@ namespace TimberWinR.Parser
AddOrModify(json, fieldName, fieldValue);
}
}
}
}
private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
{

77
TimberWinR/Inputs/LogsListener.cs
View File

@@ -32,8 +32,12 @@ namespace TimberWinR.Inputs
_receivedMessages = 0;
_arguments = arguments;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
var task = new Task(FileWatcher, cancelToken);
task.Start();
foreach (string srcFile in _arguments.Location.Split(','))
{
string file = srcFile.Trim();
Task.Factory.StartNew(() => FileWatcher(file));
}
}
public override void Shutdown()
@@ -55,40 +59,56 @@ namespace TimberWinR.Inputs
return json;
}
private void FileWatcher()
private void FileWatcher(string fileToWatch)
{
var iFmt = new TextLineInputFormat()
{
iCodepage = _arguments.CodePage,
splitLongLines = _arguments.SplitLongLines,
iCheckpoint = CheckpointFileName,
splitLongLines = _arguments.SplitLongLines,
recurse = _arguments.Recurse
};
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Location);
var firstQuery = true;
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
// Execute the query
while (!CancelToken.IsCancellationRequested)
{
var oLogQuery = new LogQuery();
try
{
var rs = oLogQuery.Execute(query, iFmt);
Dictionary<string, int> colMap = new Dictionary<string, int>();
for (int col=0; col<rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
colMap[colName] = col;
}
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", fileToWatch);
var rsfiles = oLogQuery.Execute(qfiles, iFmt);
for (; !rsfiles.atEnd(); rsfiles.moveNext())
{
// We want to "tail" the log, so skip the first query results.
if (!firstQuery)
var record = rsfiles.getRecord();
string logName = record.getValue("LogFilename") as string;
if (!logFileMaxRecords.ContainsKey(logName))
{
var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
logFileMaxRecords[logName] = lrn;
}
}
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName, lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
Dictionary<string, int> colMap = new Dictionary<string, int>();
for (int col = 0; col < rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
colMap[colName] = col;
}
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
@@ -111,11 +131,15 @@ namespace TimberWinR.Inputs
ProcessJson(json);
_receivedMessages++;
}
var lrn = (Int64)record.getValueEx("Index");
logFileMaxRecords[fileName] = lrn;
}
// Close the recordset
rs.close();
rs = null;
}
// Close the recordset
rs.close();
rs = null;
}
catch (Exception ex)
{
@@ -125,7 +149,8 @@ namespace TimberWinR.Inputs
{
oLogQuery = null;
}
firstQuery = false;
Thread.CurrentThread.Priority = ThreadPriority.Normal;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}

4
TimberWinR/Parser.cs
View File

@@ -508,6 +508,10 @@ namespace TimberWinR.Parser
{
}
}
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("condition")]
public string Condition { get; set; }

5
TimberWinR/mdocs/GrokFilter.md
View File

@@ -26,6 +26,7 @@ The following operations are allowed when mutating a field.
| Operation | Type | Description
| :---------------|:----------------|:-----------------------------------------------------------------------|
| *type* | property:string |Type to which this filter applyes, if empty, applies to all types.
| *condition* | property:string |C# expression
| *match* | property:string |Required field must match before any subsequent grok operations are executed.
| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
@@ -82,8 +83,8 @@ then the operation(s) will be executed in order.
```json
"Filters": [
{
"grok": {
"condition": "\"[type]\" == \"Win32-EventLog\""
"grok": {
"type": "Win32-EventLog",
"add_field": [
"ComputerName", "%{Host}"
]