TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed problem with WindowsEvent Files and wildcards.

Browse Source
This commit is contained in:
Eric Fontana
2014-08-29 16:41:28 -04:00
parent ab96ad3ea8
commit f5c6001865
12 changed files with 109 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -54,7 +54,7 @@ TimberWinR reads a JSON configuration file, an example file is shown here:
"Filters": [ "Filters": [
{ {
"grok": { "grok": {
"condition": "[type] == \"Win32-Eventlog\"", "condition": "\"[type]\" == \"Win32-Eventlog\"",
"match": [ "match": [
"Message", "Message",
"" ""

4
TimberWinR.ServiceHost/config.json
View File

@@ -23,7 +23,7 @@
"Filters": [ "Filters": [
{ {
"grok": { "grok": {
"condition": "[type] == \"Win32-Eventlog\"", "condition": "\"[type]\" == \"Win32-Eventlog\"",
"match": [ "match": [
"Message", "Message",
"" ""
@@ -63,7 +63,7 @@
}, },
{ {
"date": { "date": {
"condition": "[type] == \"Win32-FileLog\"", "condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [ "match": [
"timestamp", "timestamp",
"MMM d HH:mm:sss", "MMM d HH:mm:sss",

16
TimberWinR.ServiceHost/default.json
View File

@@ -3,7 +3,7 @@
"Inputs": { "Inputs": {
"WindowsEvents": [ "WindowsEvents": [
{ {
"source": "Application", "source": "Application,System",
"binaryFormat": "PRINT", "binaryFormat": "PRINT",
"resolveSIDS": true "resolveSIDS": true
} }
@@ -15,13 +15,25 @@
} }
] ]
}, },
"Filters":[
{
"grok":{
"condition": "[EventTypeName] == \"Information Event\"",
"match":[
"Text",
""
],
"drop": "true"
}
}]
},
"Outputs": { "Outputs": {
"Redis": [ "Redis": [
{ {
"_comment": "Change the host to your Redis instance", "_comment": "Change the host to your Redis instance",
"port": 6379, "port": 6379,
"host": [ "host": [
"tstlexiceapp006.vistaprint.svc" "logaggregator.vistaprint.svc"
] ]
} }
] ]

20
TimberWinR.UnitTests/GrokFilterTests.cs
View File

@@ -30,7 +30,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -79,7 +79,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -129,7 +129,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -147,7 +147,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type].Contains(\""Win32-FileLog\"")"", ""condition"": ""\""[type]\"".Contains(\""Win32-FileLog\"")"",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -166,7 +166,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type].Contains(\""Win32-Filelog\"")"", ""condition"": ""\""[type]\"".Contains(\""Win32-Filelog\"")"",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -217,7 +217,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -248,7 +248,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -274,7 +274,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -300,7 +300,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"
@@ -330,7 +330,7 @@ namespace TimberWinR.UnitTests
""Filters"":[ ""Filters"":[
{ {
""grok"":{ ""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""", ""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[ ""match"":[
""Text"", ""Text"",
"""" """"

7
TimberWinR/Filters/GrokFilter.cs
View File

@@ -39,15 +39,18 @@ namespace TimberWinR.Parser
if (Matches(json)) if (Matches(json))
{ {
if (DropIfMatch)
return false;
AddFields(json); AddFields(json);
AddTags(json); AddTags(json);
RemoveFields(json); RemoveFields(json);
RemoveTags(json); RemoveTags(json);
return true; return true;
} }
return false; return false;
} }
private bool Matches(Newtonsoft.Json.Linq.JObject json) private bool Matches(Newtonsoft.Json.Linq.JObject json)
{ {
string field = Match[0]; string field = Match[0];

88
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -68,6 +68,7 @@ namespace TimberWinR.Inputs
LogQuery oLogQuery = new LogQuery(); LogQuery oLogQuery = new LogQuery();
LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready"); LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready");
// Instantiate the Event Log Input Format object // Instantiate the Event Log Input Format object
var iFmt = new EventLogInputFormat() var iFmt = new EventLogInputFormat()
@@ -81,50 +82,71 @@ namespace TimberWinR.Inputs
stringsSep = _arguments.StringsSep, stringsSep = _arguments.StringsSep,
resolveSIDs = _arguments.ResolveSIDS resolveSIDs = _arguments.ResolveSIDS
}; };
var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", location);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lastRecordNumber = qr.getValueEx("MaxRecordNumber");
oLogQuery = null; oLogQuery = null;
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
// Execute the query // Execute the query
while (!CancelToken.IsCancellationRequested) while (!CancelToken.IsCancellationRequested)
{ {
try try
{ {
oLogQuery = new LogQuery();
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
oLogQuery = new LogQuery(); var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber); var rsfiles = oLogQuery.Execute(qfiles, iFmt);
for (; !rsfiles.atEnd(); rsfiles.moveNext())
var rs = oLogQuery.Execute(query, iFmt); {
// Browse the recordset var record = rsfiles.getRecord();
for (; !rs.atEnd(); rs.moveNext()) string logName = record.getValue("EventLog") as string;
{ if (!logFileMaxRecords.ContainsKey(logName))
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
{ {
object v = record.getValue(field.Name); var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName);
if (field.Name == "Data") var rcount = oLogQuery.Execute(qcount, iFmt);
v = ToPrintable(v.ToString()); var qr = rcount.getRecord();
json.Add(new JProperty(field.Name, v)); var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
logFileMaxRecords[logName] = lrn;
} }
lastRecordNumber = record.getValueEx("RecordNumber");
record = null;
ProcessJson(json);
_receivedMessages++;
json = null;
} }
// Close the recordset
rs.close();
rs = null; foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
{
object v = record.getValue(field.Name);
if (field.Name == "Data")
v = ToPrintable(v.ToString());
json.Add(new JProperty(field.Name, v));
}
var lrn = (Int64)record.getValueEx("RecordNumber");
logFileMaxRecords[fileName] = lrn;
record = null;
ProcessJson(json);
_receivedMessages++;
json = null;
}
// Close the recordset
rs.close();
rs = null;
}
} }
catch (Exception ex) catch (Exception ex)
{ {

15
TimberWinR/Outputs/Redis.cs
View File

@@ -124,7 +124,10 @@ namespace TimberWinR.Outputs
protected override void MessageReceivedHandler(JObject jsonMessage) protected override void MessageReceivedHandler(JObject jsonMessage)
{ {
if (_manager.Config.Filters != null) if (_manager.Config.Filters != null)
ApplyFilters(jsonMessage); {
if (ApplyFilters(jsonMessage))
return;
}
var message = jsonMessage.ToString(); var message = jsonMessage.ToString();
LogManager.GetCurrentClassLogger().Debug(message); LogManager.GetCurrentClassLogger().Debug(message);
@@ -135,12 +138,18 @@ namespace TimberWinR.Outputs
} }
} }
private void ApplyFilters(JObject json) private bool ApplyFilters(JObject json)
{ {
bool drop = false;
foreach (var filter in _manager.Config.Filters) foreach (var filter in _manager.Config.Filters)
{ {
filter.Apply(json); if (!filter.Apply(json))
{
LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
drop = true;
}
} }
return drop;
} }
// //

4
TimberWinR/Parser.cs
View File

@@ -41,7 +41,7 @@ namespace TimberWinR.Parser
IList<string> keys = json.Properties().Select(p => p.Name).ToList(); IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys) foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString())); cond = cond.Replace(string.Format("[{0}]", key), string.Format("{0}", json[key].ToString()));
var compiler = new CSharpCodeProvider(); var compiler = new CSharpCodeProvider();
@@ -511,7 +511,7 @@ namespace TimberWinR.Parser
[JsonProperty("condition")] [JsonProperty("condition")]
public string Condition { get; set; } public string Condition { get; set; }
[JsonProperty("drop_if_match")] [JsonProperty("drop")]
public bool DropIfMatch { get; set; } public bool DropIfMatch { get; set; }
[JsonProperty("match")] [JsonProperty("match")]

8
TimberWinR/mdocs/DateFilter.md
View File

@@ -34,7 +34,7 @@ Given this configuration
"Filters": [ "Filters": [
{ {
"date": { "date": {
"condition": "[type] == \"Win32-FileLog\"", "condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [ "match": [
"timestamp", "timestamp",
"MMM d HH:mm:sss", "MMM d HH:mm:sss",
@@ -56,7 +56,7 @@ then the operation(s) will be executed in order.
"Filters": [ "Filters": [
{ {
"grok": { "grok": {
"condition": "[type] == \"Win32-EventLog\"" "condition": "\"[type]\" == \"Win32-EventLog\""
"add_field": [ "add_field": [
"ComputerName", "%{Host}" "ComputerName", "%{Host}"
] ]
@@ -72,7 +72,7 @@ The fields must be in pairs with fieldName first and value second.
"Filters": [ "Filters": [
{ {
"date": { "date": {
"condition": "[type] == \"Win32-FileLog\"", "condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [ "match": [
"timestamp", "timestamp",
"MMM d HH:mm:sss", "MMM d HH:mm:sss",
@@ -92,7 +92,7 @@ If true and the filter matches, the time parsed will be converted to UTC
"Filters": [ "Filters": [
{ {
"date": { "date": {
"condition": "[type] == \"Win32-FileLog\"", "condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [ "match": [
"timestamp", "timestamp",
"MMM d HH:mm:sss", "MMM d HH:mm:sss",

4
TimberWinR/mdocs/Filters.md
View File

@@ -12,7 +12,7 @@ Example Input:
"Filters": [ "Filters": [
{ {
"grok": { "grok": {
"condition": "[type] == \"Win32-Eventlog\"", "condition": "\"[type]\" == \"Win32-Eventlog\"",
"match": [ "match": [
"Message", "Message",
"" ""
@@ -35,7 +35,7 @@ Example Input:
}, },
{ {
"date": { "date": {
"condition": "[type] == \"Win32-FileLog\"", "condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [ "match": [
"timestamp", "timestamp",
"MMM d HH:mm:sss", "MMM d HH:mm:sss",

2
TimberWinR/mdocs/GrokFilter.md
View File

@@ -83,7 +83,7 @@ then the operation(s) will be executed in order.
"Filters": [ "Filters": [
{ {
"grok": { "grok": {
"condition": "[type] == \"Win32-EventLog\"" "condition": "\"[type]\" == \"Win32-EventLog\""
"add_field": [ "add_field": [
"ComputerName", "%{Host}" "ComputerName", "%{Host}"
] ]

2
TimberWinR/mdocs/MutateFilter.md
View File

@@ -20,7 +20,7 @@ then the operation(s) will be executed in order.
"Filters": [ "Filters": [
{ {
"mutate": { "mutate": {
"condition": "[type] == \"Win32-EventLog\"" "condition": "\"[type]\" == \"Win32-EventLog\""
"rename": [ "rename": [
"ComputerName", "Host" "ComputerName", "Host"
] ]