TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed problem with WindowsEvent Files and wildcards.

Browse Source
This commit is contained in:
Eric Fontana
2014-08-29 16:41:28 -04:00
parent ab96ad3ea8
commit f5c6001865
12 changed files with 109 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@@ -54,7 +54,7 @@ TimberWinR reads a JSON configuration file, an example file is shown here:
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-Eventlog\"",
"condition": "\"[type]\" == \"Win32-Eventlog\"",
"match": [
"Message",
""

4
TimberWinR.ServiceHost/config.json
View File

@@ -23,7 +23,7 @@
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-Eventlog\"",
"condition": "\"[type]\" == \"Win32-Eventlog\"",
"match": [
"Message",
""
@@ -63,7 +63,7 @@
},
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",

16
TimberWinR.ServiceHost/default.json
View File

@@ -3,7 +3,7 @@
"Inputs": {
"WindowsEvents": [
{
"source": "Application",
"source": "Application,System",
"binaryFormat": "PRINT",
"resolveSIDS": true
}
@@ -15,13 +15,25 @@
}
]
},
"Filters":[
{
"grok":{
"condition": "[EventTypeName] == \"Information Event\"",
"match":[
"Text",
""
],
"drop": "true"
}
}]
},
"Outputs": {
"Redis": [
{
"_comment": "Change the host to your Redis instance",
"port": 6379,
"host": [
"tstlexiceapp006.vistaprint.svc"
"logaggregator.vistaprint.svc"
]
}
]

20
TimberWinR.UnitTests/GrokFilterTests.cs
View File

@@ -30,7 +30,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -79,7 +79,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -129,7 +129,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -147,7 +147,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type].Contains(\""Win32-FileLog\"")"",
""condition"": ""\""[type]\"".Contains(\""Win32-FileLog\"")"",
""match"":[
""Text"",
""""
@@ -166,7 +166,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type].Contains(\""Win32-Filelog\"")"",
""condition"": ""\""[type]\"".Contains(\""Win32-Filelog\"")"",
""match"":[
""Text"",
""""
@@ -217,7 +217,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -248,7 +248,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -274,7 +274,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -300,7 +300,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""
@@ -330,7 +330,7 @@ namespace TimberWinR.UnitTests
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
""match"":[
""Text"",
""""

7
TimberWinR/Filters/GrokFilter.cs
View File

@@ -39,15 +39,18 @@ namespace TimberWinR.Parser
if (Matches(json))
{
if (DropIfMatch)
return false;
AddFields(json);
AddTags(json);
RemoveFields(json);
RemoveTags(json);
RemoveTags(json);
return true;
}
return false;
}
private bool Matches(Newtonsoft.Json.Linq.JObject json)
{
string field = Match[0];

88
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -68,6 +68,7 @@ namespace TimberWinR.Inputs
LogQuery oLogQuery = new LogQuery();
LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready");
// Instantiate the Event Log Input Format object
var iFmt = new EventLogInputFormat()
@@ -81,50 +82,71 @@ namespace TimberWinR.Inputs
stringsSep = _arguments.StringsSep,
resolveSIDs = _arguments.ResolveSIDS
};
var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", location);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lastRecordNumber = qr.getValueEx("MaxRecordNumber");
oLogQuery = null;
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
// Execute the query
while (!CancelToken.IsCancellationRequested)
{
try
{
oLogQuery = new LogQuery();
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
oLogQuery = new LogQuery();
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
var rsfiles = oLogQuery.Execute(qfiles, iFmt);
for (; !rsfiles.atEnd(); rsfiles.moveNext())
{
var record = rsfiles.getRecord();
string logName = record.getValue("EventLog") as string;
if (!logFileMaxRecords.ContainsKey(logName))
{
object v = record.getValue(field.Name);
if (field.Name == "Data")
v = ToPrintable(v.ToString());
json.Add(new JProperty(field.Name, v));
var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
logFileMaxRecords[logName] = lrn;
}
lastRecordNumber = record.getValueEx("RecordNumber");
record = null;
ProcessJson(json);
_receivedMessages++;
json = null;
}
// Close the recordset
rs.close();
rs = null;
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
{
object v = record.getValue(field.Name);
if (field.Name == "Data")
v = ToPrintable(v.ToString());
json.Add(new JProperty(field.Name, v));
}
var lrn = (Int64)record.getValueEx("RecordNumber");
logFileMaxRecords[fileName] = lrn;
record = null;
ProcessJson(json);
_receivedMessages++;
json = null;
}
// Close the recordset
rs.close();
rs = null;
}
}
catch (Exception ex)
{

15
TimberWinR/Outputs/Redis.cs
View File

@@ -124,7 +124,10 @@ namespace TimberWinR.Outputs
protected override void MessageReceivedHandler(JObject jsonMessage)
{
if (_manager.Config.Filters != null)
ApplyFilters(jsonMessage);
{
if (ApplyFilters(jsonMessage))
return;
}
var message = jsonMessage.ToString();
LogManager.GetCurrentClassLogger().Debug(message);
@@ -135,12 +138,18 @@ namespace TimberWinR.Outputs
}
}
private void ApplyFilters(JObject json)
private bool ApplyFilters(JObject json)
{
bool drop = false;
foreach (var filter in _manager.Config.Filters)
{
filter.Apply(json);
if (!filter.Apply(json))
{
LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
drop = true;
}
}
return drop;
}
//

4
TimberWinR/Parser.cs
View File

@@ -41,7 +41,7 @@ namespace TimberWinR.Parser
IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString()));
cond = cond.Replace(string.Format("[{0}]", key), string.Format("{0}", json[key].ToString()));
var compiler = new CSharpCodeProvider();
@@ -511,7 +511,7 @@ namespace TimberWinR.Parser
[JsonProperty("condition")]
public string Condition { get; set; }
[JsonProperty("drop_if_match")]
[JsonProperty("drop")]
public bool DropIfMatch { get; set; }
[JsonProperty("match")]

8
TimberWinR/mdocs/DateFilter.md
View File

@@ -34,7 +34,7 @@ Given this configuration
"Filters": [
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
@@ -56,7 +56,7 @@ then the operation(s) will be executed in order.
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-EventLog\""
"condition": "\"[type]\" == \"Win32-EventLog\""
"add_field": [
"ComputerName", "%{Host}"
]
@@ -72,7 +72,7 @@ The fields must be in pairs with fieldName first and value second.
"Filters": [
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
@@ -92,7 +92,7 @@ If true and the filter matches, the time parsed will be converted to UTC
"Filters": [
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",

4
TimberWinR/mdocs/Filters.md
View File

@@ -12,7 +12,7 @@ Example Input:
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-Eventlog\"",
"condition": "\"[type]\" == \"Win32-Eventlog\"",
"match": [
"Message",
""
@@ -35,7 +35,7 @@ Example Input:
},
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"condition": "\"[type]\" == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",

2
TimberWinR/mdocs/GrokFilter.md
View File

@@ -83,7 +83,7 @@ then the operation(s) will be executed in order.
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-EventLog\""
"condition": "\"[type]\" == \"Win32-EventLog\""
"add_field": [
"ComputerName", "%{Host}"
]

2
TimberWinR/mdocs/MutateFilter.md
View File

@@ -20,7 +20,7 @@ then the operation(s) will be executed in order.
"Filters": [
{
"mutate": {
"condition": "[type] == \"Win32-EventLog\""
"condition": "\"[type]\" == \"Win32-EventLog\""
"rename": [
"ComputerName", "Host"
]