TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Changed input format to Json

Browse Source 
Converted input format to JSON, new unit tests
This commit is contained in:
Eric Fontana
2014-07-25 14:31:16 -04:00
parent 0aabd4ce0b
commit 171907e76f
15 changed files with 524 additions and 669 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
TimberWinR.ServiceHost/Program.cs
View File

@@ -20,7 +20,11 @@ namespace TimberWinR.ServiceHost
{
private static void Main(string[] args)
{
Arguments arguments = new Arguments();
Arguments arguments = new Arguments();
Type x = Type.GetType("string");
Type x1 = Type.GetType("System.string");
HostFactory.Run(hostConfigurator =>
{
@@ -34,6 +38,7 @@ namespace TimberWinR.ServiceHost
});
hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c);
hostConfigurator.AddCommandLineDefinition("jsonFile", c => arguments.JsonFile = c);
hostConfigurator.ApplyCommandLine();
hostConfigurator.RunAsLocalSystem();
@@ -49,6 +54,7 @@ namespace TimberWinR.ServiceHost
internal class Arguments
{
public string ConfigFile { get; set; }
public string JsonFile { get; set; }
public Arguments()
{
@@ -89,20 +95,20 @@ namespace TimberWinR.ServiceHost
/// </summary>
private void RunService()
{
TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile);
TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile);
var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken);
_nlogListener = new TcpInputListener(_cancellationToken, 5140);
outputRedis.Connect(_nlogListener);
foreach (Inputs.IISW3CLog iisw3cConfig in manager.Config.IISW3C)
foreach (Parser.IISW3CLog iisw3cConfig in manager.Config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
foreach (Inputs.WindowsEvent eventConfig in manager.Config.Events)
foreach (Parser.WindowsEvent eventConfig in manager.Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken);
outputRedis.Connect(elistner);

3
TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
View File

@@ -52,6 +52,9 @@
</ItemGroup>
<ItemGroup>
<None Include="App.config" />
<Content Include="config.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<None Include="packages.config" />
</ItemGroup>
<ItemGroup>

76
TimberWinR.ServiceHost/config.json Normal file
View File

@@ -0,0 +1,76 @@
{
"TimberWinR":{
"Inputs":{
"WindowsEvents":[
{
"source":"System,Application",
"binaryFormat":"PRINT",
"resolveSIDS":true
}
],
"Logs":[
{
"name":"Syslogs1",
"location":"C:\\Logs1\\*.log"
}
],
"IISW3CLogs":[
{
"name":"Default site",
"location":"c:\\inetpub\\logs\\LogFiles\\W3SVC1\\*"
}
]
},
"Filters":[
{
"grok":{
"condition": "[type] == \"Win32-FileLog\"",
"match":[
"Text",
""
],
"add_field":[
"host",
"%{ComputerName}"
]
}
},
{
"grok":{
"match":[
"message",
"%{SYSLOGLINE}"
],
"add_tag":[
"rn_%{Index}",
"bar"
],
"add_field":[
"foo_%{logsource}",
"Hello dude from %{ComputerName}"
]
}
},
{
"grok":{
"match":[
"Text",
"%{SYSLOGLINE}"
],
"add_tag":[
"rn_%{RecordNumber}",
"bar"
]
}
},
{
"mutate":{
"rename":[
"message",
"Message"
]
}
}
]
}
}

3
TimberWinR.ServiceHost/sampleconf.xml
View File

@@ -19,6 +19,7 @@
<AddTag>rn_%{RecordNumber}</AddTag>
<AddTag>bar</AddTag>
</Grok>
<Mutate>
<Rename oldName="TimeGenerated" newName="timestamp"/>
</Mutate>
@@ -26,6 +27,6 @@
<Pattern>MMM d HH:mm:ss</Pattern>
<Pattern>MMM dd HH:mm:ss</Pattern>
<Pattern>ISO8601</Pattern>
</Date>
</Date>
</Filters>
</TimberWinR>

295
TimberWinR.UnitTests/Configuration.cs
View File

@@ -13,298 +13,7 @@ namespace TimberWinR.UnitTests
[TestFixture]
public class ConfigurationTest
{
Configuration c = new Configuration("testconf.xml");
public void OutputEvents()
{
foreach (var evt in c.Events)
Console.WriteLine(evt);
}
public void OutputLogs()
{
foreach (var log in c.Logs)
Console.WriteLine(log);
}
public void OutputIIS()
{
foreach (var iis in c.IIS)
Console.WriteLine(iis);
}
public void OutputIISW3C()
{
foreach (var iisw3c in c.IISW3C)
Console.WriteLine(iisw3c);
}
public void OutputFilters()
{
foreach (var filter in c.Filters)
Console.WriteLine(filter);
}
[Test]
public void Output()
{
OutputEvents();
OutputLogs();
OutputIIS();
OutputIISW3C();
OutputFilters();
}
[Test]
public void NumOfEvents()
{
Assert.AreEqual(1, c.Events.ToArray().Length);
}
[Test]
public void NumOfLogs()
{
Assert.AreEqual(3, c.Logs.ToArray().Length);
}
[Test]
public void NumOfIIS()
{
Assert.AreEqual(0, c.IIS.ToArray().Length);
}
[Test]
public void NumOfIISW3C()
{
Assert.AreEqual(1, c.IISW3C.ToArray().Length);
}
[Test]
public void FieldsOfEvents()
{
Dictionary<string, Type> fields = new Dictionary<string, Type>()
{
{ "EventLog", typeof(string) },
{ "RecordNumber", typeof(int) },
{ "TimeGenerated", typeof(DateTime) },
{ "TimeWritten", typeof(DateTime) },
{ "EventID", typeof(int) },
{ "EventType", typeof(int) },
{ "EventTypeName", typeof(string) },
{ "EventCategory", typeof(int) },
{ "EventCategoryName", typeof(string) },
{ "SourceName", typeof(string) },
{ "Strings", typeof(string) },
{ "ComputerName", typeof(string) },
{ "SID", typeof(string) },
{ "Message", typeof(string) },
{ "Data", typeof(string) }
};
foreach (FieldDefinition field in c.Events.ToArray()[0].Fields)
{
Assert.Contains(field.Name, fields.Keys);
}
}
[Test]
public void FieldsOfLogs()
{
Dictionary<string, Type> fields = new Dictionary<string, Type>()
{
{ "LogFilename", typeof(string) },
{ "Index", typeof(int) },
{ "Text", typeof(string) }
};
foreach (FieldDefinition field in c.Logs.ToArray()[0].Fields)
{
Assert.Contains(field.Name, fields.Keys);
}
foreach (FieldDefinition field in c.Logs.ToArray()[1].Fields)
{
Assert.Contains(field.Name, fields.Keys);
}
foreach (FieldDefinition field in c.Logs.ToArray()[2].Fields)
{
Assert.Contains(field.Name, fields.Keys);
}
}
[Test]
public void FieldsOfIIS()
{
Dictionary<string, Type> fields = new Dictionary<string, Type>()
{
{ "LogFilename", typeof(string) },
{ "LogRow", typeof(int) },
{ "UserIP", typeof(string) },
{ "UserName", typeof(string) },
{ "Date", typeof(DateTime) },
{ "Time", typeof(DateTime) },
{ "ServiceInstance", typeof(string) },
{ "HostName", typeof(string) },
{ "ServerIP", typeof(string) },
{ "TimeTaken", typeof(int) },
{ "BytesSent", typeof(int) },
{ "BytesReceived", typeof(int) },
{ "StatusCode", typeof(int) },
{ "Win32StatusCode", typeof(int) },
{ "RequestType", typeof(string) },
{ "Target", typeof(string) },
{ "Parameters", typeof(string) }
};
foreach (var iis in c.IIS.ToArray())
{
foreach (FieldDefinition field in iis.Fields)
{
Assert.Contains(field.Name, fields.Keys);
}
}
}
[Test]
public void FieldsOfIISW3C()
{
Dictionary<string, Type> fields = new Dictionary<string, Type>()
{
{ "LogFilename", typeof(string) },
{ "LogRow", typeof(int) },
{ "date", typeof(DateTime) },
{ "time", typeof(DateTime) },
{ "c-ip", typeof(string) },
{ "cs-username", typeof(string) },
{ "s-sitename", typeof(string) },
{ "s-computername", typeof(int) },
{ "s-ip", typeof(string) },
{ "s-port", typeof(int) },
{ "cs-method", typeof(string) },
{ "cs-uri-stem", typeof(string) },
{ "cs-uri-query", typeof(string) },
{ "sc-status", typeof(int) },
{ "sc-substatus", typeof(int) },
{ "sc-win32-status", typeof(int) },
{ "sc-bytes", typeof(int) },
{ "cs-bytes", typeof(int) },
{ "time-taken", typeof(int) },
{ "cs-version", typeof(string) },
{ "cs-host", typeof(string) },
{ "cs(User-Agent)", typeof(string) },
{ "cs(Cookie)", typeof(string) },
{ "cs(Referer)", typeof(string) },
{ "s-event", typeof(string) },
{ "s-process-type", typeof(string) },
{ "s-user-time", typeof(double) },
{ "s-kernel-time", typeof(double) },
{ "s-page-faults", typeof(int) },
{ "s-total-procs", typeof(int) },
{ "s-active-procs", typeof(int) },
{ "s-stopped-procs", typeof(int) }
};
foreach (FieldDefinition field in c.IISW3C.ToArray()[0].Fields)
{
Assert.Contains(field.Name, fields.Keys);
}
}
[Test]
public void ParametersOfEvents()
{
string source = "System,Application";
bool fullText = true;
bool resolveSIDS = true;
bool formatMsg = true;
string msgErrorMode = "MSG";
bool fullEventCode = false;
string direction = "FW";
string stringsSep = "|";
string binaryFormat = "PRINT";
TimberWinR.Inputs.WindowsEvent evt = c.Events.ToArray()[0];
Assert.AreEqual(source, evt.Source);
Assert.AreEqual(fullText, evt.FullText);
Assert.AreEqual(resolveSIDS, evt.ResolveSIDS);
Assert.AreEqual(formatMsg, evt.FormatMsg);
Assert.AreEqual(msgErrorMode, evt.MsgErrorMode);
Assert.AreEqual(fullEventCode, evt.FullEventCode);
Assert.AreEqual(direction, evt.Direction);
Assert.AreEqual(stringsSep, evt.StringsSep);
Assert.IsNull(evt.ICheckpoint);
Assert.AreEqual(binaryFormat, evt.BinaryFormat);
}
[Test]
public void ParametersOfLogs()
{
string name = "First Set";
string location = @"C:\Logs1\*.log";
int iCodepage = 0;
int recurse = 0;
bool splitLongLines = false;
TimberWinR.Inputs.TailFileInput log = c.Logs.ToArray()[0];
Assert.AreEqual(name, log.Name);
Assert.AreEqual(location, log.Location);
Assert.AreEqual(iCodepage, log.ICodepage);
Assert.AreEqual(recurse, log.Recurse);
Assert.AreEqual(splitLongLines, log.SplitLongLines);
name = "Second Set";
location = @"C:\Logs2\*.log";
iCodepage = 0;
recurse = 0;
splitLongLines = false;
log = c.Logs.ToArray()[1];
Assert.AreEqual(name, log.Name);
Assert.AreEqual(location, log.Location);
Assert.AreEqual(iCodepage, log.ICodepage);
Assert.AreEqual(recurse, log.Recurse);
Assert.AreEqual(splitLongLines, log.SplitLongLines);
name = "Third Set";
location = @"C:\Logs2\1.log,C:\Logs2\2.log";
iCodepage = 0;
recurse = 0;
splitLongLines = false;
log = c.Logs.ToArray()[2];
Assert.AreEqual(name, log.Name);
Assert.AreEqual(location, log.Location);
Assert.AreEqual(iCodepage, log.ICodepage);
Assert.AreEqual(recurse, log.Recurse);
Assert.AreEqual(splitLongLines, log.SplitLongLines);
}
[Test]
public void ParametersOfIISW3C()
{
string name = "Default site";
string location = @"c:\inetpub\logs\LogFiles\W3SVC1\*";
int iCodepage = -2;
int recurse = 0;
bool dQuotes = false;
bool dirTime = false;
bool consolidateLogs = false;
TimberWinR.Inputs.IISW3CLog iisw3c = c.IISW3C.ToArray()[0];
Assert.AreEqual(name, iisw3c.Name);
Assert.AreEqual(location, iisw3c.Location);
Assert.AreEqual(iCodepage, iisw3c.ICodepage);
Assert.AreEqual(recurse, iisw3c.Recurse);
Assert.IsNull(iisw3c.MinDateMod);
Assert.AreEqual(dQuotes, iisw3c.DQuotes);
Assert.AreEqual(dirTime, iisw3c.DirTime);
Assert.AreEqual(consolidateLogs, iisw3c.ConsolidateLogs);
Assert.IsEmpty(iisw3c.ICheckpoint);
}
}
}

168
TimberWinR/Configuration.cs
View File

@@ -2,162 +2,114 @@
using System.Collections.Generic;
using System.Data.Odbc;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Xml;
using System.Xml.Linq;
using System.IO;
using System.Globalization;
using System.Xml.Schema;
using Newtonsoft.Json;
using Newtonsoft.Json.Bson;
using Newtonsoft.Json.Linq;
using TimberWinR.Inputs;
using TimberWinR.Filters;
using NLog;
using TimberWinR.Parser;
using IISW3CLog = TimberWinR.Parser.IISW3CLog;
using WindowsEvent = TimberWinR.Parser.WindowsEvent;
namespace TimberWinR
{
public class Configuration
{
private static List<WindowsEvent> _events = new List<WindowsEvent>();
private List<WindowsEvent> _events = new List<WindowsEvent>();
public IEnumerable<WindowsEvent> Events
{
get { return _events; }
}
private static List<TailFileInput> _logs = new List<TailFileInput>();
private List<Log> _logs = new List<Log>();
public IEnumerable<TailFileInput> Logs
public IEnumerable<Log> Logs
{
get { return _logs; }
}
}
private static List<IISLog> _iislogs = new List<IISLog>();
public IEnumerable<IISLog> IIS
{
get { return _iislogs; }
}
private static List<IISW3CLog> _iisw3clogs = new List<IISW3CLog>();
private List<IISW3CLog> _iisw3clogs = new List<IISW3CLog>();
public IEnumerable<IISW3CLog> IISW3C
{
get { return _iisw3clogs; }
}
private static List<FilterBase> _filters = new List<FilterBase>();
private List<LogstashFilter> _filters = new List<LogstashFilter>();
public IEnumerable<FilterBase> Filters
public IEnumerable<LogstashFilter> Filters
{
get { return _filters; }
}
public Configuration(string xmlConfFile)
public static Configuration FromFile(string jsonConfFile)
{
validateWithSchema(xmlConfFile, Properties.Resources.configSchema);
Configuration c = new Configuration();
try
if (!string.IsNullOrEmpty(jsonConfFile))
{
parseConfInput(xmlConfFile);
parseConfFilter(xmlConfFile);
string json = File.ReadAllText(jsonConfFile);
return FromString(json);
}
catch(Exception ex)
return null;
}
public static Configuration FromString(string json)
{
Configuration c = new Configuration();
JsonSerializer serializer = new JsonSerializer();
TextReader re = new StringReader(json);
JsonTextReader reader = new JsonTextReader(re);
var x = serializer.Deserialize<TimberWinR.Parser.RootObject>(reader);
if (x.TimberWinR.Inputs != null)
{
LogManager.GetCurrentClassLogger().Error(ex);
c._events = x.TimberWinR.Inputs.WindowsEvents.ToList();
c._iisw3clogs = x.TimberWinR.Inputs.IISW3CLogs.ToList();
c._logs = x.TimberWinR.Inputs.Logs.ToList();
}
if (x.TimberWinR.Filters != null)
c._filters = x.TimberWinR.AllFilters.ToList();
return c;
}
public Configuration()
{
_filters = new List<LogstashFilter>();
_events = new List<WindowsEvent>();
_iisw3clogs = new List<IISW3CLog>();
_logs = new List<Log>();
}
private static void validateWithSchema(string xmlConfFile, string xsdSchema)
public static Object GetPropValue(String name, Object obj)
{
XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri);
// Ensure that the xml configuration file provided obeys the xsd schema.
XmlSchemaSet schemas = new XmlSchemaSet();
schemas.Add("", XmlReader.Create(new StringReader(xsdSchema)));
#if true
bool errorsFound = false;
config.Validate(schemas, (o, e) =>
foreach (String part in name.Split('.'))
{
errorsFound = true;
LogManager.GetCurrentClassLogger().Error(e.Message);
}, true);
if (obj == null) { return null; }
if (errorsFound)
DumpInvalidNodes(config.Root);
#endif
}
Type type = obj.GetType();
PropertyInfo info = type.GetProperty(part);
if (info == null) { return null; }
static void DumpInvalidNodes(XElement el)
{
if (el.GetSchemaInfo().Validity != XmlSchemaValidity.Valid)
LogManager.GetCurrentClassLogger().Error("Invalid Element {0}",
el.AncestorsAndSelf()
.InDocumentOrder()
.Aggregate("", (s, i) => s + "/" + i.Name.ToString()));
foreach (XAttribute att in el.Attributes())
if (att.GetSchemaInfo().Validity != XmlSchemaValidity.Valid)
LogManager.GetCurrentClassLogger().Error("Invalid Attribute {0}",
att
.Parent
.AncestorsAndSelf()
.InDocumentOrder()
.Aggregate("",
(s, i) => s + "/" + i.Name.ToString()) + "/@" + att.Name.ToString()
);
foreach (XElement child in el.Elements())
DumpInvalidNodes(child);
}
static void parseConfInput(string xmlConfFile)
{
XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri);
XElement allInputs = config.Root.Element(InputBase.TagName);
if (allInputs == null)
throw new TimberWinR.ConfigurationErrors.MissingRequiredTagException(InputBase.TagName);
createInputs(allInputs, WindowsEvent.ParentTagName, WindowsEvent.TagName, _events, WindowsEvent.Parse);
createInputs(allInputs, TailFileInput.ParentTagName, TailFileInput.TagName, _logs, TailFileInput.Parse);
createInputs(allInputs, IISLog.ParentTagName, IISLog.TagName, _iislogs, IISLog.Parse);
createInputs(allInputs, IISW3CLog.ParentTagName, IISW3CLog.TagName, _iisw3clogs, IISW3CLog.Parse);
}
static void createInputs<T>(XElement allInputs, string parentTagName, string tagName, List<T> inputList, Action<List<T>, XElement> parse)
{
IEnumerable<XElement> inputs =
from el in allInputs.Elements(parentTagName).Elements(tagName)
select el;
foreach (XElement input in inputs)
parse(inputList, input);
}
static void parseConfFilter(string xmlConfFile)
{
XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri);
IEnumerable<XElement> filters =
from el in config.Root.Elements(FilterBase.TagName)
select el;
foreach (XElement e in filters.Elements())
{
switch (e.Name.ToString())
{
case DateFilter.TagName:
DateFilter.Parse(_filters, e);
break;
case GrokFilter.TagName:
GrokFilter.Parse(_filters, e);
break;
case MutateFilter.TagName:
MutateFilter.Parse(_filters, e);
break;
default:
throw new Exception(string.Format("Unknown tag: {0}", e.Name.ToString()));
}
obj = info.GetValue(obj, null);
}
}
return obj;
}
}
}

72
TimberWinR/Inputs/IISLog.cs
View File

@@ -1,72 +0,0 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Text;
using System.Xml.Linq;
namespace TimberWinR.Inputs
{
public class IISLog : InputBase
{
public const string ParentTagName = "IISLogs";
public new const string TagName = "IISLog";
public string Name { get; private set; }
public string Location { get; private set; }
public int ICodepage { get; private set; }
public int Recurse { get; private set; }
public string MinDateMod { get; private set; }
public string Locale { get; private set; }
public string ICheckpoint { get; private set; }
public List<FieldDefinition> Fields { get; private set; }
public static void Parse(List<IISLog> iislogs, XElement iislogElement)
{
iislogs.Add(parseIISLog(iislogElement));
}
static IISLog parseIISLog(XElement e)
{
return new IISLog(e);
}
public IISLog(XElement parent)
{
Name = ParseRequiredStringAttribute(parent, "name");
Location = ParseRequiredStringAttribute(parent, "location");
ICodepage = ParseIntAttribute(parent, "iCodepage", -2);
Recurse = ParseIntAttribute(parent, "recurse", 0);
MinDateMod = ParseDateAttribute(parent, "minDateMod");
Locale = ParseStringAttribute(parent, "locale", "DEF");
ICheckpoint = ParseStringAttribute(parent, "iCheckpoint");
ParseFields(parent);
}
private void ParseFields(XElement parent)
{
Dictionary<string, Type> allPossibleFields = new Dictionary<string, Type>()
{
{ "LogFilename", typeof(string) },
{ "LogRow", typeof(int) },
{ "UserIP", typeof(string) },
{ "UserName", typeof(string) },
{ "Date", typeof(DateTime) },
{ "Time", typeof(DateTime) },
{ "ServiceInstance", typeof(string) },
{ "HostName", typeof(string) },
{ "ServerIP", typeof(string) },
{ "TimeTaken", typeof(int) },
{ "BytesSent", typeof(int) },
{ "BytesReceived", typeof(int) },
{ "StatusCode", typeof(int) },
{ "Win32StatusCode", typeof(int) },
{ "RequestType", typeof(string) },
{ "Target", typeof(string) },
{ "Parameters", typeof(string) }
};
Fields = base.parseFields(parent, allPossibleFields);
}
}
}

28
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -21,10 +21,10 @@ namespace TimberWinR.Inputs
public class IISW3CInputListener : InputListener
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Inputs.IISW3CLog _arguments;
private TimberWinR.Parser.IISW3CLog _arguments;
public IISW3CInputListener(TimberWinR.Inputs.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
{
_arguments = arguments;
@@ -42,17 +42,17 @@ namespace TimberWinR.Inputs
var iFmt = new IISW3CLogInputFormat()
{
codepage = _arguments.ICodepage,
codepage = _arguments.CodePage,
consolidateLogs = _arguments.ConsolidateLogs,
dirTime = _arguments.DirTime,
dQuotes = _arguments.DQuotes,
dQuotes = _arguments.DoubleQuotes,
iCheckpoint = checkpointFileName,
recurse = _arguments.Recurse,
useDoubleQuotes = _arguments.DQuotes
useDoubleQuotes = _arguments.DoubleQuotes
};
if (!string.IsNullOrEmpty(_arguments.MinDateMod))
iFmt.minDateMod = _arguments.MinDateMod;
if (_arguments.MinDateMod.HasValue)
iFmt.minDateMod = _arguments.MinDateMod.Value.ToString("yyyy-MM-dd hh:mm:ss");
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Location);
@@ -84,12 +84,14 @@ namespace TimberWinR.Inputs
if (!colMap.ContainsKey(field.Name))
continue;
object v = record.getValue(field.Name);
if (field.FieldType == typeof(DateTime))
v = field.ToDateTime(v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
object v = record.getValue(field.Name);
if (field.DataType == typeof(DateTime))
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field.Name, dt));
}
else
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-IISLog"));
ProcessJson(json);

89
TimberWinR/Inputs/IISW3CLog.cs
View File

@@ -1,89 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Xml.Linq;
namespace TimberWinR.Inputs
{
public class IISW3CLog : InputBase
{
public const string ParentTagName = "IISW3CLogs";
public new const string TagName = "IISW3CLog";
public string Name { get; private set; }
public string Location { get; private set; }
public int ICodepage { get; private set; }
public int Recurse { get; private set; }
public string MinDateMod { get; private set; }
public bool DQuotes { get; private set; }
public bool DirTime { get; private set; }
public bool ConsolidateLogs { get; private set; }
public string ICheckpoint { get; private set; }
public List<FieldDefinition> Fields { get; private set; }
public static void Parse(List<IISW3CLog> iisw3clogs, XElement iisw3clogElement)
{
iisw3clogs.Add(parseIISW3CLog(iisw3clogElement));
}
static IISW3CLog parseIISW3CLog(XElement e)
{
return new IISW3CLog(e);
}
public IISW3CLog(XElement parent)
{
Name = ParseRequiredStringAttribute(parent, "name");
Location = ParseRequiredStringAttribute(parent, "location");
ICodepage = ParseIntAttribute(parent, "iCodepage", -2);
Recurse = ParseIntAttribute(parent, "recurse", 0);
DQuotes = ParseBoolAttribute(parent, "dQuotes", false);
DirTime = ParseBoolAttribute(parent, "dirTime", false);
ConsolidateLogs = ParseBoolAttribute(parent, "consolidateLogs", false);
ICheckpoint = ParseStringAttribute(parent, "iCheckpoint");
ParseFields(parent);
}
private void ParseFields(XElement parent)
{
Dictionary<string, Type> allPossibleFields = new Dictionary<string, Type>()
{
{ "LogFilename", typeof(string) },
{ "LogRow", typeof(int) },
{ "date", typeof(DateTime) },
{ "time", typeof(DateTime) },
{ "c-ip", typeof(string) },
{ "cs-username", typeof(string) },
{ "s-sitename", typeof(string) },
{ "s-computername", typeof(int) },
{ "s-ip", typeof(string) },
{ "s-port", typeof(int) },
{ "cs-method", typeof(string) },
{ "cs-uri-stem", typeof(string) },
{ "cs-uri-query", typeof(string) },
{ "sc-status", typeof(int) },
{ "sc-substatus", typeof(int) },
{ "sc-win32-status", typeof(int) },
{ "sc-bytes", typeof(int) },
{ "cs-bytes", typeof(int) },
{ "time-taken", typeof(int) },
{ "cs-version", typeof(string) },
{ "cs-host", typeof(string) },
{ "cs(User-Agent)", typeof(string) },
{ "cs(Cookie)", typeof(string) },
{ "cs(Referer)", typeof(string) },
{ "s-event", typeof(string) },
{ "s-process-type", typeof(string) },
{ "s-user-time", typeof(double) },
{ "s-kernel-time", typeof(double) },
{ "s-page-faults", typeof(int) },
{ "s-total-procs", typeof(int) },
{ "s-active-procs", typeof(int) },
{ "s-stopped-procs", typeof(int) }
};
Fields = base.parseFields(parent, allPossibleFields);
}
}
}

75
TimberWinR/Inputs/WindowsEvent.cs
View File

@@ -1,75 +0,0 @@
using System;
using System.Collections.Generic;
using System.Text;
using System.Xml.Linq;
using Microsoft.SqlServer.Server;
namespace TimberWinR.Inputs
{
public class WindowsEvent : InputBase
{
public const string ParentTagName = "WindowsEvents";
public new const string TagName = "Event";
public string Source { get; private set; }
public bool FullText { get; private set; }
public bool ResolveSIDS { get; private set; }
public bool FormatMsg { get; private set; }
public string MsgErrorMode { get; private set; }
public bool FullEventCode { get; private set; }
public string Direction { get; private set; }
public string StringsSep { get; private set; }
public string ICheckpoint { get; private set; }
public string BinaryFormat { get; private set; }
public List<FieldDefinition> Fields { get; private set; }
public static void Parse(List<WindowsEvent> events, XElement eventElement)
{
events.Add(parseEvent(eventElement));
}
static WindowsEvent parseEvent(XElement e)
{
return new WindowsEvent(e);
}
WindowsEvent(XElement parent)
{
Source = ParseRequiredStringAttribute(parent, "source");
FullText = ParseBoolAttribute(parent, "fullText", true);
ResolveSIDS = ParseBoolAttribute(parent, "resolveSIDS", true);
FormatMsg = ParseBoolAttribute(parent, "formatMsg", true);
MsgErrorMode = ParseEnumAttribute(parent, "msgErrorMode", new string[] {"NULL", "ERROR", "MSG"}, "MSG");
FullEventCode = ParseBoolAttribute(parent, "fullEventCode", false); ;
Direction = ParseEnumAttribute(parent, "direction", new string[] { "FW", "BW" }, "FW");
StringsSep = ParseStringAttribute(parent, "stringsSep", "|");
BinaryFormat = ParseEnumAttribute(parent, "binaryFormat", new string[] { "ASC", "PRINT", "HEX" }, "PRINT");
ParseFields(parent);
}
private void ParseFields(XElement parent)
{
Dictionary<string, Type> allPossibleFields = new Dictionary<string, Type>()
{
{ "EventLog", typeof(string) },
{ "RecordNumber", typeof(int) },
{ "TimeGenerated", typeof(DateTime) },
{ "TimeWritten", typeof(DateTime) },
{ "EventID", typeof(int) },
{ "EventType", typeof(int) },
{ "EventTypeName", typeof(string) },
{ "EventCategory", typeof(int) },
{ "EventCategoryName", typeof(string) },
{ "SourceName", typeof(string) },
{ "Strings", typeof(string) },
{ "ComputerName", typeof(string) },
{ "SID", typeof(string) },
{ "Message", typeof(string) },
{ "Data", typeof(string) }
};
Fields = base.parseFields(parent, allPossibleFields);
}
}
}

14
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -23,9 +23,9 @@ namespace TimberWinR.Inputs
public class WindowsEvtInputListener : InputListener
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Inputs.WindowsEvent _arguments;
private TimberWinR.Parser.WindowsEvent _arguments;
public WindowsEvtInputListener(TimberWinR.Inputs.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
{
_arguments = arguments;
@@ -44,12 +44,12 @@ namespace TimberWinR.Inputs
// Instantiate the Event Log Input Format object
var iFmt = new EventLogInputFormat()
{
binaryFormat = _arguments.BinaryFormat,
direction = _arguments.Direction,
binaryFormat = _arguments.BinaryFormat.ToString(),
direction = _arguments.Direction.ToString(),
formatMsg = _arguments.FormatMsg,
fullEventCode = _arguments.FullEventCode,
fullText = _arguments.FullText,
msgErrorMode = _arguments.MsgErrorMode,
msgErrorMode = _arguments.MsgErrorMode.ToString(),
stringsSep = _arguments.StringsSep,
resolveSIDs = _arguments.ResolveSIDS,
iCheckpoint = checkpointFileName,
@@ -77,8 +77,8 @@ namespace TimberWinR.Inputs
{
object v = record.getValue(field.Name);
if (field.FieldType == typeof(DateTime))
v = field.ToDateTime(v).ToUniversalTime();
//if (field.FieldType == typeof(DateTime))
// v = field.ToDateTime(v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
}

6
TimberWinR/Manager.cs
View File

@@ -16,7 +16,7 @@ namespace TimberWinR
{
public Configuration Config { get; set; }
public Manager(string configurationFile=null)
public Manager(string xmlConfigFile, string jsonConfigFile)
{
var loggingConfiguration = new LoggingConfiguration();
@@ -36,8 +36,10 @@ namespace TimberWinR
LogManager.GetCurrentClassLogger().Info("Initialized");
// Read the Configuration file
Config = new Configuration(configurationFile);
Config = Configuration.FromFile(jsonConfigFile);
}
/// <summary>

3
TimberWinR/Outputs/Redis.cs
View File

@@ -6,6 +6,7 @@ using System.Net.Sockets;
using System.Text;
using System.Threading;
using ctstone.Redis;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using System.Threading.Tasks;
@@ -97,7 +98,7 @@ namespace TimberWinR.Outputs
foreach (var grok in _manager.Config.Filters)
{
grok.Apply(json);
}
}
}
//

342
TimberWinR/Parser.cs Normal file
View File

@@ -0,0 +1,342 @@
using System;
using System.Collections.Generic;
using System.Configuration;
using System.Linq;
using System.Text;
using Microsoft.SqlServer.Server;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
namespace TimberWinR.Parser
{
public abstract class LogstashFilter
{
public abstract bool Apply(JObject json);
protected void RenameProperty(JObject json, string oldName, string newName)
{
JToken token = json[oldName];
if (token != null)
{
json.Remove(oldName);
json.Add(newName, token);
}
}
protected void ReplaceProperty(JObject json, string propertyName, string propertyValue)
{
if (json[propertyName] != null)
json[propertyName] = propertyValue;
}
protected void AddOrModify(JObject json, string fieldName, string fieldValue)
{
if (json[fieldName] == null)
json.Add(fieldName, fieldValue);
else
json[fieldName] = fieldValue;
}
protected string ExpandField(string fieldName, JObject json)
{
foreach (var token in json.Children())
{
string replaceString = "%{" + token.Path + "}";
fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
}
return fieldName;
}
}
[JsonObject(MemberSerialization.OptIn)]
public class Field
{
[JsonProperty(PropertyName = "name")]
public string Name { get; set; }
[JsonProperty(PropertyName = "to")]
public string To { get; set; }
[JsonProperty(PropertyName = "type")]
public string FieldType { get; set; }
public Type DataType
{
get { return Type.GetType(FieldType); }
}
public Field()
{
FieldType = "string";
}
public Field(string name)
{
Name = name;
To = Name;
FieldType = "string";
}
public Field(string name, string type)
{
Name = name;
if (type.ToLower() == "string")
type = "System.String";
else if (type.ToLower() == "datetime")
type = "System.DateTime";
else if (type.ToLower() == "int" || type.ToLower() == "integer")
type = "System.Int32";
FieldType = type;
To = Name;
}
public Field(string name, string type, string to)
{
Name = name;
FieldType = type;
To = to;
}
}
public class WindowsEvent
{
public enum FormatKinds
{
PRINT, ASC, HEX
};
public enum MessageErrorModes
{
MSG,
ERROR,
NULL
};
public enum DirectionKinds
{
FW,
BW
};
[JsonProperty(PropertyName = "source")]
public string Source { get; set; }
[JsonProperty(PropertyName = "binaryFormat")]
public FormatKinds BinaryFormat { get; set; }
[JsonProperty(PropertyName = "msgErrorMode")]
public MessageErrorModes MsgErrorMode { get; set; }
[JsonProperty(PropertyName = "direction")]
public DirectionKinds Direction { get; set; }
[JsonProperty(PropertyName = "stringsSep")]
public string StringsSep { get; set; }
[JsonProperty(PropertyName = "fullEventCode")]
public bool FullEventCode { get; set; }
[JsonProperty(PropertyName = "fullText")]
public bool FullText { get; set; }
[JsonProperty(PropertyName = "resolveSIDS")]
public bool ResolveSIDS { get; set; }
[JsonProperty(PropertyName = "fields")]
public List<Field> Fields { get; set; }
[JsonProperty(PropertyName = "formatMsg")]
public bool FormatMsg { get; set; }
public WindowsEvent()
{
StringsSep = "|";
FormatMsg = true;
FullText = true;
Fields = new List<Field>();
Fields.Add(new Field("EventLog", "string"));
Fields.Add(new Field("RecordNumber", "int"));
Fields.Add(new Field("TimeGenerated", "DateTime"));
Fields.Add(new Field("TimeWritten", "DateTime"));
Fields.Add(new Field("EventID", "int"));
Fields.Add(new Field("EventType", "int"));
Fields.Add(new Field("EventTypeName", "string"));
Fields.Add(new Field("EventCategory", "int"));
Fields.Add(new Field("EventCategoryName", "string"));
Fields.Add(new Field("SourceName", "string"));
Fields.Add(new Field("Strings", "string"));
Fields.Add(new Field("ComputerName", "string"));
Fields.Add(new Field("SID", "string"));
Fields.Add(new Field("Message", "string"));
Fields.Add(new Field("Data", "string"));
}
}
public class Log
{
[JsonProperty(PropertyName = "location")]
public string Location { get; set; }
[JsonProperty(PropertyName = "iCodepage")]
public int CodePage { get; set; }
[JsonProperty(PropertyName = "recurse")]
public int Recurse { get; set; }
[JsonProperty(PropertyName = "splitLongLines")]
public bool SplitLongLines { get; set; }
[JsonProperty(PropertyName = "fields")]
public List<Field> Fields { get; set; }
public Log()
{
Fields = new List<Field>();
Fields.Add(new Field("LogFilename", "string"));
Fields.Add(new Field("Index", "integer"));
Fields.Add(new Field("Text", "string"));
}
}
public class IISW3CLog
{
[JsonProperty(PropertyName = "name")]
public string Name { get; set; }
[JsonProperty(PropertyName = "location")]
public string Location { get; set; }
[JsonProperty(PropertyName = "iCodepage")]
public int CodePage { get; set; }
[JsonProperty(PropertyName = "recurse")]
public int Recurse { get; set; }
[JsonProperty(PropertyName = "dQuotes")]
public bool DoubleQuotes { get; set; }
[JsonProperty(PropertyName = "dirTime")]
public bool DirTime { get; private set; }
[JsonProperty(PropertyName = "consolidateLogs")]
public bool ConsolidateLogs { get; private set; }
[JsonProperty(PropertyName = "minDateMod")]
public DateTime? MinDateMod { get; private set; }
[JsonProperty(PropertyName = "fields")]
public List<Field> Fields { get; set; }
public IISW3CLog()
{
CodePage = -2;
Recurse = 0;
Fields = new List<Field>();
Fields.Add(new Field("LogFilename", "string"));
Fields.Add(new Field("LogRow", "integer" ));
Fields.Add(new Field("date", "DateTime" ));
Fields.Add(new Field("time", "DateTime" ));
Fields.Add(new Field("c-ip", "string" ));
Fields.Add(new Field("cs-username", "string" ));
Fields.Add(new Field("s-sitename", "string" ));
Fields.Add(new Field("s-computername", "integer" ));
Fields.Add(new Field("s-ip", "string" ));
Fields.Add(new Field("s-port", "integer" ));
Fields.Add(new Field("cs-method", "string" ));
Fields.Add(new Field("cs-uri-stem", "string" ));
Fields.Add(new Field("cs-uri-query", "string" ));
Fields.Add(new Field("sc-status", "integer" ));
Fields.Add(new Field("sc-substatus", "integer" ));
Fields.Add(new Field("sc-win32-status", "integer" ));
Fields.Add(new Field("sc-bytes", "integer" ));
Fields.Add(new Field("cs-bytes", "integer" ));
Fields.Add(new Field("time-taken", "integer" ));
Fields.Add(new Field("cs-version", "string" ));
Fields.Add(new Field("cs-host", "string" ));
Fields.Add(new Field("cs(User-Agent)", "string" ));
Fields.Add(new Field("cs(Cookie)", "string" ));
Fields.Add(new Field("cs(Referer)", "string" ));
Fields.Add(new Field("s-event", "string" ));
Fields.Add(new Field("s-process-type", "string" ));
Fields.Add(new Field("s-user-time", "double" ));
Fields.Add(new Field("s-kernel-time", "double" ));
Fields.Add(new Field("s-page-faults", "integer" ));
Fields.Add(new Field("s-total-procs", "integer" ));
Fields.Add(new Field("s-active-procs", "integer" ));
Fields.Add(new Field("s-stopped-procs", "integer"));
}
}
public class InputSources
{
[JsonProperty("WindowsEvents")]
public WindowsEvent[] WindowsEvents { get; set; }
[JsonProperty("Logs")]
public Log[] Logs { get; set; }
[JsonProperty("IISW3CLogs")]
public IISW3CLog[] IISW3CLogs { get; set; }
}
public partial class Grok : LogstashFilter
{
[JsonProperty("condition")]
public string Condition { get; set; }
[JsonProperty("match")]
public string[] Match { get; set; }
[JsonProperty("add_tag")]
public string[] AddTag { get; set; }
[JsonProperty("add_field")]
public string[] AddField { get; set; }
}
public class Date : LogstashFilter
{
public string field { get; set; }
public string target { get; set; }
public bool convertToUTC { get; set; }
public List<string> Pattern { get; set; }
public override bool Apply(JObject json)
{
return false;
}
}
public partial class Mutate : LogstashFilter
{
[JsonProperty("rename")]
public string[] Rename { get; set; }
[JsonProperty("replace")]
public string[] Replace { get; set; }
[JsonProperty("split")]
public string[] Split { get; set; }
}
public class Filter
{
[JsonProperty("grok")]
public Grok Grok { get; set; }
[JsonProperty("mutate")]
public Mutate Mutate { get; set; }
}
public class TimberWinR
{
[JsonProperty("Inputs")]
public InputSources Inputs { get; set; }
public List<Filter> Filters { get; set; }
public LogstashFilter[] AllFilters
{
get
{
var list = new List<LogstashFilter>();
foreach (var filter in Filters)
{
foreach (var prop in filter.GetType().GetProperties())
{
object typedFilter = filter.GetType().GetProperty(prop.Name).GetValue(filter, null);
if (typedFilter != null && typedFilter is LogstashFilter)
{
list.Add(typedFilter as LogstashFilter);
}
}
}
return list.ToArray();
}
}
}
public class RootObject
{
public TimberWinR TimberWinR { get; set; }
}
}

5
TimberWinR/TimberWinR.csproj
View File

@@ -70,20 +70,17 @@
<Compile Include="Filters\GrokFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\IISLog.cs" />
<Compile Include="Inputs\IISW3CInputListener.cs" />
<Compile Include="Inputs\IISW3CLog.cs" />
<Compile Include="Inputs\InputBase.cs" />
<Compile Include="Inputs\InputListener.cs" />
<Compile Include="Inputs\ParameterDefinitions.cs" />
<Compile Include="Inputs\TailFileInput.cs" />
<Compile Include="Inputs\TcpInputListener.cs" />
<Compile Include="Inputs\TailFileInputListener.cs" />
<Compile Include="Inputs\WindowsEvent.cs" />
<Compile Include="Inputs\WindowsEvtInputListener.cs" />
<Compile Include="Manager.cs" />
<Compile Include="Outputs\OutputSender.cs" />
<Compile Include="Outputs\Redis.cs" />
<Compile Include="Parser.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Properties\Resources.Designer.cs">
<AutoGen>True</AutoGen>