TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Validate method

Browse Source 
Added tests to validate the configuration.
This commit is contained in:
Eric Fontana
2014-07-28 10:55:17 -04:00
parent 3202c19b7e
commit 65de7cbc93
17 changed files with 623 additions and 235 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
TimberWinR.ServiceHost/Program.cs
View File

@@ -4,6 +4,7 @@ using System.Globalization;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using TimberWinR.Outputs;
@@ -22,7 +23,11 @@ namespace TimberWinR.ServiceHost
{
Arguments arguments = new Arguments();
var text = "Nov 21 17:27:53";
var pattern = "MMM dd HH:mm:ss";
var match = Regex.Match(text, pattern);
Type x = Type.GetType("string");
Type x1 = Type.GetType("System.string");
@@ -95,8 +100,9 @@ namespace TimberWinR.ServiceHost
/// </summary>
private void RunService()
{
TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile);
TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken);
#if false
var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken);
_nlogListener = new TcpInputListener(_cancellationToken, 5140);
@@ -119,6 +125,7 @@ namespace TimberWinR.ServiceHost
var elistner = new TailFileInputListener(logConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
#endif
}
}
}

186
TimberWinR.ServiceHost/config.json
View File

@@ -1,76 +1,112 @@
{
"TimberWinR":{
"Inputs":{
"WindowsEvents":[
{
"source":"System,Application",
"binaryFormat":"PRINT",
"resolveSIDS":true
{
"TimberWinR": {
"Inputs": {
"WindowsEvents": [
{
"source": "System,Application",
"binaryFormat": "PRINT",
"resolveSIDS": true
}
],
"Tcp": [
{
"port": "5140"
}
],
"Logs": [
{
"name": "Syslogs1",
"location": "C:\\Logs1\\*.log"
}
],
"IISW3CLogs": [
{
"name": "Default site",
"location": "c:\\inetpub\\logs\\LogFiles\\W3SVC1\\*"
}
]
},
"Outputs": {
"Redis": [
{
"host": [
"logaggregator.vistaprint.svc"
]
}
]
},
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"Text",
""
],
"add_field": [
"host",
"%{ComputerName}"
]
}
},
{
"grok": {
"condition": "[type] == \"Win32-Eventlog\"",
"match": [
"Message",
""
],
"remove_field": [
"ComputerName"
]
}
},
{
"grok": {
"match": [
"message",
"%{SYSLOGLINE}"
],
"add_tag": [
"rn_%{Index}",
"bar"
],
"add_field": [
"foo_%{logsource}",
"Hello dude from %{ComputerName}"
]
}
},
{
"grok": {
"match": [
"Text",
"%{SYSLOGLINE}"
],
"add_tag": [
"rn_%{RecordNumber}",
"bar"
]
}
},
{
"mutate": {
"rename": [
"host", "Host",
"message","Message",
"SID", "Username"
]
}
},
{
"date": {
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
]
}
}
],
"Logs":[
{
"name":"Syslogs1",
"location":"C:\\Logs1\\*.log"
}
],
"IISW3CLogs":[
{
"name":"Default site",
"location":"c:\\inetpub\\logs\\LogFiles\\W3SVC1\\*"
}
]
},
"Filters":[
{
"grok":{
"condition": "[type] == \"Win32-FileLog\"",
"match":[
"Text",
""
],
"add_field":[
"host",
"%{ComputerName}"
]
}
},
{
"grok":{
"match":[
"message",
"%{SYSLOGLINE}"
],
"add_tag":[
"rn_%{Index}",
"bar"
],
"add_field":[
"foo_%{logsource}",
"Hello dude from %{ComputerName}"
]
}
},
{
"grok":{
"match":[
"Text",
"%{SYSLOGLINE}"
],
"add_tag":[
"rn_%{RecordNumber}",
"bar"
]
}
},
{
"mutate":{
"rename":[
"message",
"Message"
]
}
}
]
}
}
]
}
}

60
TimberWinR.UnitTests/Configuration.cs
View File

@@ -7,13 +7,69 @@ using System.Threading.Tasks;
using TimberWinR;
using TimberWinR.Inputs;
using TimberWinR.Filters;
using Newtonsoft.Json.Linq;
namespace TimberWinR.UnitTests
{
[TestFixture]
public class ConfigurationTest
{
[Test]
public void TestInvalidMatchConfig()
{
string grokJson = @"{
""TimberWinR"":{
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""match"":[
""Text""
]
}
}]
}
}";
try
{
Configuration c = Configuration.FromString(grokJson);
Assert.IsTrue(false, "Should have thrown an exception");
}
catch (TimberWinR.Parser.Grok.GrokFilterException ex)
{
}
}
[Test]
public void TestInvalidAddTagConfig()
{
string grokJson = @"{
""TimberWinR"":{
""Filters"":[
{
""grok"":{
""condition"": ""[type] == \""Win32-FileLog\"""",
""match"":[
""Text"", """"
],
""add_tag"": [
""rn_%{Index}"",
],
}
}]
}
}";
try
{
Configuration c = Configuration.FromString(grokJson);
Assert.IsTrue(false, "Should have thrown an exception");
}
catch (TimberWinR.Parser.Grok.GrokAddTagException ex)
{
}
}
}
}

1
TimberWinR.UnitTests/GrokFilterTests.cs
View File

@@ -192,7 +192,6 @@ namespace TimberWinR.UnitTests
c = Configuration.FromString(grokJson3);
grok = c.Filters.First() as Grok;
Assert.IsFalse(grok.Apply(json));
}
[Test]

47
TimberWinR/Configuration.cs
View File

@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Data.Odbc;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Text;
@@ -17,6 +18,7 @@ using TimberWinR.Filters;
using NLog;
using TimberWinR.Parser;
using Topshelf.Configurators;
using IISW3CLog = TimberWinR.Parser.IISW3CLog;
using WindowsEvent = TimberWinR.Parser.WindowsEvent;
@@ -25,14 +27,25 @@ namespace TimberWinR
public class Configuration
{
private List<WindowsEvent> _events = new List<WindowsEvent>();
public IEnumerable<WindowsEvent> Events
{
get { return _events; }
}
private List<Log> _logs = new List<Log>();
private List<RedisOutput> _redisOutputs = new List<RedisOutput>();
public IEnumerable<RedisOutput> RedisOutputs
{
get { return _redisOutputs; }
}
private List<Tcp> _tcps = new List<Tcp>();
public IEnumerable<Tcp> Tcps
{
get { return _tcps; }
}
private List<Log> _logs = new List<Log>();
public IEnumerable<Log> Logs
{
get { return _logs; }
@@ -78,23 +91,47 @@ namespace TimberWinR
if (x.TimberWinR.Inputs != null)
{
c._events = x.TimberWinR.Inputs.WindowsEvents.ToList();
c._events = x.TimberWinR.Inputs.WindowsEvents.ToList();
c._iisw3clogs = x.TimberWinR.Inputs.IISW3CLogs.ToList();
c._logs = x.TimberWinR.Inputs.Logs.ToList();
c._logs = x.TimberWinR.Inputs.Logs.ToList();
c._redisOutputs = x.TimberWinR.Outputs.Redis.ToList();
c._tcps = x.TimberWinR.Inputs.Tcps.ToList();
}
if (x.TimberWinR.Filters != null)
c._filters = x.TimberWinR.AllFilters.ToList();
c.Validate(c);
// Validate
return c;
}
void Validate(Configuration c)
{
try
{
foreach (var e in c.Events)
e.Validate();
foreach (var f in c.Filters)
f.Validate();
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
throw ex;
}
}
public Configuration()
{
_filters = new List<LogstashFilter>();
_events = new List<WindowsEvent>();
_iisw3clogs = new List<IISW3CLog>();
_logs = new List<Log>();
_redisOutputs = new List<RedisOutput>();
_tcps = new List<Tcp>();
}
public static Object GetPropValue(String name, Object obj)

96
TimberWinR/Filters/DateFilter.cs
View File

@@ -6,69 +6,63 @@ using System.Text;
using System.Xml;
using Newtonsoft.Json.Linq;
using System.Xml.Linq;
using TimberWinR.Parser;
using RapidRegex.Core;
using System.Text.RegularExpressions;
namespace TimberWinR.Filters
namespace TimberWinR.Parser
{
public class DateFilter : FilterBase
{
public new const string TagName = "Date";
public string Field { get; private set; }
public string Target { get; private set; }
public bool ConvertToUTC { get; private set; }
public List<string> Patterns { get; private set; }
public static void Parse(List<FilterBase> filters, XElement dateElement)
public partial class DateFilter : LogstashFilter
{
public override bool Apply(JObject json)
{
filters.Add(parseDate(dateElement));
}
static DateFilter parseDate(XElement e)
{
return new DateFilter(e);
}
DateFilter(XElement parent)
{
Patterns = new List<string>();
Field = ParseStringAttribute(parent, "field");
Target = ParseStringAttribute(parent, "target", Field);
ConvertToUTC = ParseBoolAttribute(parent, "convertToUTC", false);
ParsePatterns(parent);
}
private void ParsePatterns(XElement parent)
{
foreach (var e in parent.Elements("Pattern"))
if (Matches(json))
{
string pattern = e.Value;
Patterns.Add(pattern);
ApplyFilter(json);
}
return true;
}
private void ApplyFilter(JObject json)
{
string text = json.ToString();
if (!string.IsNullOrEmpty(text))
{
DateTime ts;
if (Patterns == null || Patterns.Length == 0)
{
if (DateTime.TryParse(text, out ts))
AddOrModify(json, ts);
}
else
{
if (DateTime.TryParseExact(text, Patterns.ToArray(), CultureInfo.InvariantCulture,
DateTimeStyles.None, out ts))
AddOrModify(json, ts);
}
}
}
public override void Apply(JObject json)
private bool Matches(Newtonsoft.Json.Linq.JObject json)
{
string field = Match[0];
JToken token = null;
if (json.TryGetValue(Field, StringComparison.OrdinalIgnoreCase, out token))
if (json.TryGetValue(field, out token))
{
string text = token.ToString();
if (!string.IsNullOrEmpty(text))
{
DateTime ts;
if (Patterns == null || Patterns.Count == 0)
{
if (DateTime.TryParse(text, out ts))
AddOrModify(json, ts);
}
else
{
if (DateTime.TryParseExact(text, Patterns.ToArray(), CultureInfo.InvariantCulture, DateTimeStyles.None, out ts))
AddOrModify(json, ts);
}
var exprArray = Match.Skip(1).ToArray();
if (DateTime.TryParseExact(text, exprArray, CultureInfo.InvariantCulture,DateTimeStyles.None, out ts))
AddOrModify(json, ts);
}
return true; // Empty field is no match
}
return false; // Not specified is failure
}
@@ -77,10 +71,10 @@ namespace TimberWinR.Filters
if (ConvertToUTC)
ts = ts.ToUniversalTime();
if (json[Target] == null)
json.Add(Target, ts);
else
json[Target] = ts;
//if (json[Target] == null)
// json.Add(Target, ts);
//else
// json[Target] = ts;
}
}
}

65
TimberWinR/Filters/EvaluatingFilter.cs Normal file
View File

@@ -0,0 +1,65 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Microsoft.CSharp;
using Newtonsoft.Json.Linq;
using NLog;
namespace TimberWinR.Filters
{
class EvaluatingFilter
{
protected bool EvaluateCondition(JObject json, string condition)
{
// Create a new instance of the C# compiler
var cond = condition;
IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString()));
var compiler = new CSharpCodeProvider();
// Create some parameters for the compiler
var parms = new System.CodeDom.Compiler.CompilerParameters
{
GenerateExecutable = false,
GenerateInMemory = true
};
parms.ReferencedAssemblies.Add("System.dll");
var code = string.Format(@" using System;
class EvaluatorClass
{{
public bool Evaluate()
{{
return {0};
}}
}}", cond);
// Try to compile the string into an assembly
var results = compiler.CompileAssemblyFromSource(parms, new string[] { code });
// If there weren't any errors get an instance of "MyClass" and invoke
// the "Message" method on it
if (results.Errors.Count == 0)
{
var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass");
var result = evClass.GetType().
GetMethod("Evaluate").
Invoke(evClass, null);
return bool.Parse(result.ToString());
}
else
{
foreach (var e in results.Errors)
{
LogManager.GetCurrentClassLogger().Error(e);
LogManager.GetCurrentClassLogger().Error("Bad Code: {0}", code);
}
}
return false;
}
}
}

51
TimberWinR/Filters/GrokFilter.cs
View File

@@ -34,7 +34,7 @@ namespace TimberWinR.Parser
{
public override bool Apply(JObject json)
{
if (Condition != null && !EvaluateCondition(json))
if (Condition != null && !EvaluateCondition(json, Condition))
return false;
if (Matches(json))
@@ -48,55 +48,6 @@ namespace TimberWinR.Parser
return false;
}
private bool EvaluateCondition(JObject json)
{
// Create a new instance of the C# compiler
var cond = Condition;
IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString()));
var compiler = new CSharpCodeProvider();
// Create some parameters for the compiler
var parms = new System.CodeDom.Compiler.CompilerParameters
{
GenerateExecutable = false,
GenerateInMemory = true
};
parms.ReferencedAssemblies.Add("System.dll");
var code = string.Format(@" using System;
class EvaluatorClass
{{
public bool Evaluate()
{{
return {0};
}}
}}", cond);
// Try to compile the string into an assembly
var results = compiler.CompileAssemblyFromSource(parms, new string[] { code });
// If there weren't any errors get an instance of "MyClass" and invoke
// the "Message" method on it
if (results.Errors.Count == 0)
{
var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass");
var result = evClass.GetType().
GetMethod("Evaluate").
Invoke(evClass, null);
return bool.Parse(result.ToString());
}
else
{
foreach (var e in results.Errors)
LogManager.GetCurrentClassLogger().Error(e);
}
return false;
}
private bool Matches(Newtonsoft.Json.Linq.JObject json)
{
string field = Match[0];

3
TimberWinR/Filters/MutateFilter.cs
View File

@@ -13,6 +13,9 @@ namespace TimberWinR.Parser
{
public override bool Apply(JObject json)
{
if (Condition != null && !EvaluateCondition(json, Condition))
return false;
ApplySplits(json);
ApplyRenames(json);
ApplyReplace(json);

5
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -25,7 +25,7 @@ namespace TimberWinR.Inputs
public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
: base(cancelToken, "Win32-IISLog")
{
_arguments = arguments;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
@@ -92,8 +92,7 @@ namespace TimberWinR.Inputs
}
else
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-IISLog"));
}
ProcessJson(json);
}
}

31
TimberWinR/Inputs/InputListener.cs
View File

@@ -1,4 +1,5 @@
using Newtonsoft.Json.Linq;
using System.Runtime.InteropServices;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
@@ -11,18 +12,36 @@ namespace TimberWinR.Inputs
{
public CancellationToken CancelToken { get; set; }
public event Action<JObject> OnMessageRecieved;
private string _computerName;
private string _typeName;
public InputListener(CancellationToken token)
public InputListener(CancellationToken token, string typeName)
{
this.CancelToken = token;
this.CancelToken = token;
this._typeName = typeName;
this._computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
}
private void AddDefaultFileds(JObject json)
{
if (json["type"] == null)
json.Add(new JProperty("type", _typeName));
if (json["host"] == null)
json.Add(new JProperty("host", _computerName));
}
protected void ProcessJson(JObject json)
{
{
if (OnMessageRecieved != null)
OnMessageRecieved(json);
{
AddDefaultFileds(json);
OnMessageRecieved(json);
}
}
}
}

13
TimberWinR/Inputs/TailFileInputListener.cs
View File

@@ -26,7 +26,7 @@ namespace TimberWinR.Inputs
private TimberWinR.Parser.Log _arguments;
public TailFileInputListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
: base(cancelToken, "Win32-FileLog")
{
_arguments = arguments;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
@@ -49,12 +49,7 @@ namespace TimberWinR.Inputs
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Location);
string computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
var firstQuery = true;
// Execute the query
@@ -92,9 +87,7 @@ namespace TimberWinR.Inputs
}
else
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-FileLog"));
json.Add(new JProperty("ComputerName", computerName));
}
ProcessJson(json);
}
}

18
TimberWinR/Inputs/TcpInputListener.cs
View File

@@ -15,9 +15,12 @@ namespace TimberWinR.Inputs
private readonly System.Net.Sockets.TcpListener _tcpListener;
private Thread _listenThread;
const int bufferSize = 16535;
private int _port;
public TcpInputListener(CancellationToken cancelToken, int port = 5140) : base(cancelToken)
{
public TcpInputListener(CancellationToken cancelToken, int port = 5140)
: base(cancelToken, "Win32-Tcp")
{
_port = port;
_tcpListener = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
_listenThread = new Thread(new ThreadStart(ListenForClients));
_listenThread.Start();
@@ -32,6 +35,8 @@ namespace TimberWinR.Inputs
{
this._tcpListener.Start();
LogManager.GetCurrentClassLogger().Info("Tcp Input on Port {0} Ready", _port);
while (!CancelToken.IsCancellationRequested)
{
try
@@ -58,6 +63,12 @@ namespace TimberWinR.Inputs
var tcpClient = (TcpClient)client;
NetworkStream clientStream = tcpClient.GetStream();
string computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
var message = new byte[bufferSize];
while (!CancelToken.IsCancellationRequested)
{
@@ -83,7 +94,8 @@ namespace TimberWinR.Inputs
var encoder = new ASCIIEncoding();
var encodedMessage = encoder.GetString(message, 0, bytesRead);
ProcessJson(JObject.Parse(encodedMessage));
JObject json = JObject.Parse(encodedMessage);
ProcessJson(json);
}
tcpClient.Close();
}

15
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -26,7 +26,7 @@ namespace TimberWinR.Inputs
private TimberWinR.Parser.WindowsEvent _arguments;
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
: base(cancelToken, "Win32-Eventlog")
{
_arguments = arguments;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
@@ -55,6 +55,13 @@ namespace TimberWinR.Inputs
iCheckpoint = checkpointFileName,
};
string computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Source);
@@ -76,13 +83,9 @@ namespace TimberWinR.Inputs
foreach (var field in _arguments.Fields)
{
object v = record.getValue(field.Name);
//if (field.FieldType == typeof(DateTime))
// v = field.ToDateTime(v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-Eventlog"));
ProcessJson(json);
}
}

51
TimberWinR/Manager.cs
View File

@@ -6,6 +6,9 @@ using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using TimberWinR.Inputs;
using TimberWinR.Outputs;
using System.Threading;
namespace TimberWinR
{
@@ -15,9 +18,12 @@ namespace TimberWinR
public class Manager
{
public Configuration Config { get; set; }
public Manager(string xmlConfigFile, string jsonConfigFile)
{
public List<OutputSender> Outputs { get; set; }
public Manager(string xmlConfigFile, string jsonConfigFile, CancellationToken cancelToken)
{
Outputs = new List<OutputSender>();
var loggingConfiguration = new LoggingConfiguration();
// Create our default targets
@@ -35,11 +41,46 @@ namespace TimberWinR
LogManager.EnableLogging();
LogManager.GetCurrentClassLogger().Info("Initialized");
// Read the Configuration file
Config = Configuration.FromFile(jsonConfigFile);
if (Config.RedisOutputs != null)
{
foreach (var ro in Config.RedisOutputs)
{
var redis = new RedisOutput(this, ro.Host, cancelToken, ro.Index, ro.Port, ro.Timeout);
Outputs.Add(redis);
}
}
foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
foreach(var output in Outputs)
output.Connect(elistner);
}
foreach (Parser.WindowsEvent eventConfig in Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);
foreach (var output in Outputs)
output.Connect(elistner);
}
foreach (var logConfig in Config.Logs)
{
var elistner = new TailFileInputListener(logConfig, cancelToken);
foreach (var output in Outputs)
output.Connect(elistner);
}
foreach (var tcp in Config.Tcps)
{
var elistner = new TcpInputListener(cancelToken, tcp.Port);
foreach (var output in Outputs)
output.Connect(elistner);
}
}
/// <summary>

203
TimberWinR/Parser.cs
View File

@@ -2,18 +2,28 @@
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Reflection;
using System.Runtime.Remoting.Channels;
using System.Text;
using Microsoft.CSharp;
using Microsoft.SqlServer.Server;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using TimberWinR.Outputs;
namespace TimberWinR.Parser
{
public abstract class LogstashFilter
interface IValidateSchema
{
void Validate();
}
public abstract class LogstashFilter : IValidateSchema
{
public abstract bool Apply(JObject json);
protected void RenameProperty(JObject json, string oldName, string newName)
{
JToken token = json[oldName];
@@ -24,6 +34,57 @@ namespace TimberWinR.Parser
}
}
protected bool EvaluateCondition(JObject json, string condition)
{
// Create a new instance of the C# compiler
var cond = condition;
IList<string> keys = json.Properties().Select(p => p.Name).ToList();
foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("\"{0}\"", json[key].ToString()));
var compiler = new CSharpCodeProvider();
// Create some parameters for the compiler
var parms = new System.CodeDom.Compiler.CompilerParameters
{
GenerateExecutable = false,
GenerateInMemory = true
};
parms.ReferencedAssemblies.Add("System.dll");
var code = string.Format(@" using System;
class EvaluatorClass
{{
public bool Evaluate()
{{
return {0};
}}
}}", cond);
// Try to compile the string into an assembly
var results = compiler.CompileAssemblyFromSource(parms, new string[] { code });
// If there weren't any errors get an instance of "MyClass" and invoke
// the "Message" method on it
if (results.Errors.Count == 0)
{
var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass");
var result = evClass.GetType().
GetMethod("Evaluate").
Invoke(evClass, null);
return bool.Parse(result.ToString());
}
else
{
foreach (var e in results.Errors)
{
LogManager.GetCurrentClassLogger().Error(e);
LogManager.GetCurrentClassLogger().Error("Bad Code: {0}", code);
}
}
return false;
}
protected void RemoveProperties(JToken token, string[] fields)
{
JContainer container = token as JContainer;
@@ -46,14 +107,12 @@ namespace TimberWinR.Parser
}
}
protected void ReplaceProperty(JObject json, string propertyName, string propertyValue)
{
if (json[propertyName] != null)
json[propertyName] = propertyValue;
}
protected void AddOrModify(JObject json, string fieldName, string fieldValue)
{
if (json[fieldName] == null)
@@ -71,6 +130,9 @@ namespace TimberWinR.Parser
}
return fieldName;
}
public abstract void Validate();
}
[JsonObject(MemberSerialization.OptIn)]
@@ -120,7 +182,7 @@ namespace TimberWinR.Parser
}
}
public class WindowsEvent
public class WindowsEvent : IValidateSchema
{
public enum FormatKinds
{
@@ -183,9 +245,14 @@ namespace TimberWinR.Parser
Fields.Add(new Field("Message", "string"));
Fields.Add(new Field("Data", "string"));
}
public void Validate()
{
}
}
public class Log
public class Log : IValidateSchema
{
[JsonProperty(PropertyName = "location")]
public string Location { get; set; }
@@ -206,9 +273,30 @@ namespace TimberWinR.Parser
Fields.Add(new Field("Index", "integer"));
Fields.Add(new Field("Text", "string"));
}
public void Validate()
{
}
}
public class Tcp : IValidateSchema
{
[JsonProperty(PropertyName = "port")]
public int Port { get; set; }
public Tcp()
{
Port = 5140;
}
public void Validate()
{
}
}
public class IISW3CLog
public class IISW3CLog : IValidateSchema
{
[JsonProperty(PropertyName = "name")]
public string Name { get; set; }
@@ -269,6 +357,37 @@ namespace TimberWinR.Parser
Fields.Add(new Field("s-active-procs", "integer" ));
Fields.Add(new Field("s-stopped-procs", "integer"));
}
public void Validate()
{
}
}
public partial class RedisOutput
{
[JsonProperty(PropertyName = "host")]
public string[] Host { get; set; }
[JsonProperty(PropertyName = "index")]
public string Index { get; set; }
[JsonProperty(PropertyName = "port")]
public int Port { get; set; }
[JsonProperty(PropertyName = "timeout")]
public int Timeout { get; set; }
public RedisOutput()
{
Port = 6379;
Index = "logstash";
Host = new string[] {"localhost"};
Timeout = 10000;
}
}
public class OutputTargets
{
[JsonProperty("Redis")]
public RedisOutput[] Redis { get; set; }
}
public class InputSources
@@ -279,12 +398,30 @@ namespace TimberWinR.Parser
[JsonProperty("Logs")]
public Log[] Logs { get; set; }
[JsonProperty("Tcp")]
public Tcp[] Tcps { get; set; }
[JsonProperty("IISW3CLogs")]
public IISW3CLog[] IISW3CLogs { get; set; }
}
public partial class Grok : LogstashFilter
public partial class Grok : LogstashFilter, IValidateSchema
{
public class GrokFilterException : Exception
{
public GrokFilterException()
: base("Grok filter missing required match, must be 2 array entries.")
{
}
}
public class GrokAddTagException : Exception
{
public GrokAddTagException()
: base("Grok filter add_tag requires tuples")
{
}
}
[JsonProperty("condition")]
public string Condition { get; set; }
@@ -304,24 +441,44 @@ namespace TimberWinR.Parser
public string[] RemoveField { get; set; }
[JsonProperty("remove_tag")]
public string[] RemoveTag { get; set; }
public string[] RemoveTag { get; set; }
public override void Validate()
{
if (Match == null || Match.Length != 2)
throw new GrokFilterException();
if (AddTag != null && AddTag.Length%2 != 0)
throw new GrokAddTagException();
}
}
public class Date : LogstashFilter
public partial class DateFilter : LogstashFilter
{
public string field { get; set; }
public string target { get; set; }
public bool convertToUTC { get; set; }
public List<string> Pattern { get; set; }
[JsonProperty("match")]
public string[] Match { get; set; }
public override bool Apply(JObject json)
[JsonProperty("target")]
public string[] Target { get; set; }
[JsonProperty("convertToUTC")]
public bool ConvertToUTC { get; set; }
[JsonProperty("pattern")]
public string[] Patterns { get; set; }
public override void Validate()
{
return false;
}
}
public partial class Mutate : LogstashFilter
{
[JsonProperty("condition")]
public string Condition { get; set; }
[JsonProperty("rename")]
public string[] Rename { get; set; }
@@ -329,7 +486,12 @@ namespace TimberWinR.Parser
public string[] Replace { get; set; }
[JsonProperty("split")]
public string[] Split { get; set; }
public string[] Split { get; set; }
public override void Validate()
{
}
}
public class Filter
@@ -339,13 +501,20 @@ namespace TimberWinR.Parser
[JsonProperty("mutate")]
public Mutate Mutate { get; set; }
[JsonProperty("date")]
public DateFilter Date { get; set; }
}
public class TimberWinR
{
[JsonProperty("Inputs")]
public InputSources Inputs { get; set; }
[JsonProperty("Filters")]
public List<Filter> Filters { get; set; }
[JsonProperty("Outputs")]
public OutputTargets Outputs { get; set; }
public LogstashFilter[] AllFilters
{
get

4
TimberWinR/TimberWinR.csproj
View File

@@ -66,6 +66,7 @@
<Compile Include="Configuration.cs" />
<Compile Include="ConfigurationErrors.cs" />
<Compile Include="Filters\DateFilter.cs" />
<Compile Include="Filters\EvaluatingFilter.cs" />
<Compile Include="Filters\FilterBase.cs" />
<Compile Include="Filters\GrokFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
@@ -105,6 +106,9 @@
<LastGenOutput>Resources.Designer.cs</LastGenOutput>
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
<Folder Include="Parsing\" />
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.