TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finished up log tailing.

Browse Source
This commit is contained in:
Eric Fontana
2014-07-22 10:26:04 -04:00
parent 1e271d5592
commit 613faaf49f
19 changed files with 352 additions and 330 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
TimberWinR.ServiceHost/Program.cs
View File

@@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Text;
@@ -20,9 +21,9 @@ namespace TimberWinR.ServiceHost
private static void Main(string[] args)
{
Arguments arguments = new Arguments();
HostFactory.Run(hostConfigurator =>
{
{
string cmdLine = Environment.CommandLine;
hostConfigurator.Service<TimberWinRService>(serviceConfigurator =>
@@ -55,23 +56,23 @@ namespace TimberWinR.ServiceHost
}
}
internal class TimberWinRService
{
readonly CancellationTokenSource _cancellationTokenSource;
readonly CancellationToken _cancellationToken;
readonly Task _serviceTask;
private readonly Arguments _args;
private TcpInputListener _nlogListener;
private TcpInputListener _nlogListener;
public TimberWinRService(Arguments args)
{
_args = args;
_cancellationTokenSource = new CancellationTokenSource();
_cancellationToken = _cancellationTokenSource.Token;
_serviceTask = new Task(RunService, _cancellationToken);
_serviceTask = new Task(RunService, _cancellationToken);
}
public void Start()
{
_serviceTask.Start();
@@ -82,21 +83,16 @@ namespace TimberWinR.ServiceHost
_cancellationTokenSource.Cancel();
_nlogListener.Shutdown();
}
/// <summary>
/// The Main body of the Service Worker Thread
/// </summary>
private void RunService()
{
TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile);
// logaggregator.vistaprint.svc
//var outputRedis = new RedisOutput(new string[] { "tstlexiceapp006.vistaprint.svc", "tstlexiceapp007.vistaprint.svc" }, _cancellationToken);
var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken);
// var outputRedis = new RedisOutput(new string[] { "prdlexicelgs001.vistaprint.svc" }, _cancellationToken);
var outputRedis = new RedisOutput(new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken);
_nlogListener = new TcpInputListener(_cancellationToken, 5140);
outputRedis.Connect(_nlogListener);
@@ -105,18 +101,18 @@ namespace TimberWinR.ServiceHost
var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
foreach (Configuration.WindowsEvent eventConfig in manager.Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
//while (!_cancellationTokenSource.IsCancellationRequested)
//{
// System.Threading.Thread.Sleep(1000);
//}
foreach (var logConfig in manager.Config.Logs)
{
var elistner = new TailFileInputListener(logConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
}
}
}

5
TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
View File

@@ -60,6 +60,11 @@
<Name>TimberWinR</Name>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<Content Include="sampleconf.xml">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

40
TimberWinR.ServiceHost/sampleconf.xml Normal file
View File

@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="utf-8"?>
<TimberWinR>
<Inputs>
<WindowsEvents>
<Event source="System,Application" binaryFormat="PRINT" />
</WindowsEvents>
<Logs>
<Log name="Syslogs" location="C:\Logs1\*.log" />
</Logs>
<IISW3CLogs>
<IISW3CLog name="Default site" location="c:\inetpub\logs\LogFiles\W3SVC1\*" />
</IISW3CLogs>
</Inputs>
<Filters>
<Grok>
<!--Single Tag-->
<Match field="Text" value="%{SYSLOGLINE}" />
<DropIfMatch value="true" />
<!--Multiple Tag allowed -->
<AddField name="field1" value="%{foo}" />
<AddField name="field2" value="%{foo}" />
<RemoveField value="ip1" />
<RemoveField value="ip2" />
<!--Verify field 'name' is unique also target unique -->
<Date field="timestamp" target="@timestamp" convertToUTC="true">
<Pattern>MMM d HH:mm:ss</Pattern>
<Pattern>MMM dd HH:mm:ss</Pattern>
<Pattern>ISO8601</Pattern>
</Date>
<Date field="timestamp2" target="@timestamp2" convertToUTC="false">
<Pattern>MMM d HH:mm:ss</Pattern>
<Pattern>MMM dd HH:mm:ss</Pattern>
<Pattern>ISO8601</Pattern>
</Date>
</Grok>
</Filters>
</TimberWinR>

2
TimberWinR.UnitTests/Configuration.cs
View File

@@ -257,7 +257,7 @@ namespace TimberWinR.UnitTests
bool splitLongLines = false;
string iCheckpoint;
TimberWinR.Configuration.TextLog log = c.Logs.ToArray()[0];
TimberWinR.Configuration.TailFileInput log = c.Logs.ToArray()[0];
Assert.AreEqual(name, log.Name);
Assert.AreEqual(location, log.Location);

3
TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
View File

@@ -47,9 +47,6 @@
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
<None Include="testconf.xsd">
<SubType>Designer</SubType>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

2
TimberWinR.UnitTests/testconf.xml
View File

@@ -17,7 +17,7 @@
<Filters>
<Grok>
<Match value="%{IPAddress:ip1} %{IPAddress:ip2}" />
<Match field="Text" value="%{IPAddress:ip1} %{IPAddress:ip2}" />
<AddField name="field1" value="%{foo}" />
<DropIfMatch value="true" />
<RemoveField value="ip1" />

236
TimberWinR.UnitTests/testconf.xsd
View File

@@ -1,236 +0,0 @@
<xs:schema attributeFormDefault="unqualified" elementFormDefault="qualified" xmlns:xs="http://www.w3.org/2001/XMLSchema">
<xs:element name="TimberWinR">
<xs:complexType>
<xs:sequence>
<xs:element name="Inputs">
<xs:complexType>
<xs:sequence>
<xs:element name="WindowsEvents" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Event" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
iis <xs:element name="Fields" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="name" use="required">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="EventLog|RecordNumber|TimeGenerated|TimeWritten|EventID|EventType|EventTypeName|EventCategory|EventCategoryName|SourceName|Strings|ComputerName|SID|Message|Data" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
<xs:attribute type="xs:string" name="source" use="required"/>
<xs:attribute name="fullText" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="ON|on|OFF|off|true|false" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute name="resolveSIDs" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="ON|on|OFF|off|true|false" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute name="formatMsg" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="ON|on|OFF|off|true|false" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute name="msgErrorMode" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="NULL|null|ERROR|error|MSG|msg" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute name="fullEventCode" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="ON|on|OFF|off|true|false" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute name="direction" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="FW|fw|BW|bw" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute type="xs:string" name="stringsSep" use="optional"/>
<xs:attribute type="xs:string" name="iCheckpoint" use="optional"/>
<xs:attribute name="binaryFormat" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="ASC|asc|PRINT|print|HEX|hex" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="Logs" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Log" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="name" use="required">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="LogFilename|Index|Text" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="location" use="required"/>
<xs:attribute type="xs:int" name="iCodepage" use="optional"/>
<xs:attribute type="xs:int" name="recurse" use="optional"/>
<xs:attribute name="splitLongLines" use="optional">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="ON|on|OFF|off|true|false" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute type="xs:string" name="iCheckpoint" use="optional"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="IISLogs" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="IISLog" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="name" use="required">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="LogFilename|LogRow|UserIP|UserName|Date|Time|ServiceInstance|HostName|ServerIP|TimeTaken|BytesSent|BytesReceived|StatusCode|Win32StatusCode|RequestType|Target|Parameters" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="location" use="required"/>
<xs:attribute type="xs:int" name="iCodepage" use="optional"/>
<xs:attribute type="xs:int" name="recurse" use="optional"/>
<xs:attribute name="minDateMod" use="optional">
<xs:simpleType>
<xs:restriction base="xs:token">
<xs:pattern value="[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute type="xs:string" name="locale" use="optional"/>
<xs:attribute type="xs:string" name="iCheckpoint" use="optional"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="IISW3CLogs" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="IISW3CLog" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="name" use="required">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="LogFilename|LogRow|UserIP|UserName|Date|Time|ServiceInstance|HostName|ServerIP|TimeTaken|BytesSent|BytesReceived|StatusCode|Win32StatusCode|RequestType|Target|Parameters" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="location" use="required"/>
<xs:attribute type="xs:int" name="iCodepage" use="optional"/>
<xs:attribute type="xs:int" name="recurse" use="optional"/>
<xs:attribute name="minDateMod" use="optional">
<xs:simpleType>
<xs:restriction base="xs:token">
<xs:pattern value="[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute type="xs:string" name="locale" use="optional"/>
<xs:attribute type="xs:string" name="iCheckpoint" use="optional"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:schema>

115
TimberWinR/Configuration.cs
View File

@@ -9,6 +9,7 @@ using System.IO;
using System.Globalization;
using TimberWinR.Inputs;
using System.Xml.Schema;
using NLog;
namespace TimberWinR
{
@@ -29,7 +30,7 @@ namespace TimberWinR
public MissingRequiredAttributeException(XElement e, string attributeName)
: base(
string.Format("{0}:{1} Missing required attribute \"{2}\" for element <{3}>", e.Document.BaseUri,
((IXmlLineInfo)e).LineNumber, attributeName, e.Name.ToString()))
((IXmlLineInfo) e).LineNumber, attributeName, e.Name.ToString()))
{
}
}
@@ -39,7 +40,7 @@ namespace TimberWinR
public InvalidAttributeNameException(XAttribute a)
: base(
string.Format("{0}:{1} Invalid Attribute Name <{2} {3}>", a.Document.BaseUri,
((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.Name.ToString()))
((IXmlLineInfo) a).LineNumber, a.Parent.Name, a.Name.ToString()))
{
}
}
@@ -48,8 +49,10 @@ namespace TimberWinR
{
public InvalidAttributeDateValueException(XAttribute a)
: base(
string.Format("{0}:{1} Invalid date format given for attribute. Format must be \"yyyy-MM-dd hh:mm:ss\". <{2} {3}>", a.Document.BaseUri,
((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.ToString()))
string.Format(
"{0}:{1} Invalid date format given for attribute. Format must be \"yyyy-MM-dd hh:mm:ss\". <{2} {3}>",
a.Document.BaseUri,
((IXmlLineInfo) a).LineNumber, a.Parent.Name, a.ToString()))
{
}
}
@@ -59,7 +62,7 @@ namespace TimberWinR
public InvalidAttributeIntegerValueException(XAttribute a)
: base(
string.Format("{0}:{1} Integer value not given for attribute. <{2} {3}>", a.Document.BaseUri,
((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.ToString()))
((IXmlLineInfo) a).LineNumber, a.Parent.Name, a.ToString()))
{
}
}
@@ -69,7 +72,7 @@ namespace TimberWinR
public InvalidAttributeValueException(XAttribute a)
: base(
string.Format("{0}:{1} Invalid Attribute Value <{2} {3}>", a.Document.BaseUri,
((IXmlLineInfo)a).LineNumber, a.Parent.Name, a.ToString()))
((IXmlLineInfo) a).LineNumber, a.Parent.Name, a.ToString()))
{
}
}
@@ -79,25 +82,45 @@ namespace TimberWinR
public InvalidElementNameException(XElement e)
: base(
string.Format("{0}:{1} Invalid Element Name <{2}> <{3}>", e.Document.BaseUri,
((IXmlLineInfo)e).LineNumber, e.Parent.Name, e.ToString()))
((IXmlLineInfo) e).LineNumber, e.Parent.Name, e.ToString()))
{
}
}
private static List<WindowsEvent> _events = new List<WindowsEvent>();
public IEnumerable<WindowsEvent> Events { get { return _events; } }
private static List<TextLog> _logs = new List<TextLog>();
public IEnumerable<TextLog> Logs { get { return _logs; } }
public IEnumerable<WindowsEvent> Events
{
get { return _events; }
}
private static List<TailFileInput> _logs = new List<TailFileInput>();
public IEnumerable<TailFileInput> Logs
{
get { return _logs; }
}
private static List<IISLog> _iislogs = new List<IISLog>();
public IEnumerable<IISLog> IIS { get { return _iislogs; } }
public IEnumerable<IISLog> IIS
{
get { return _iislogs; }
}
private static List<IISW3CLog> _iisw3clogs = new List<IISW3CLog>();
public IEnumerable<IISW3CLog> IISW3C { get { return _iisw3clogs; } }
public IEnumerable<IISW3CLog> IISW3C
{
get { return _iisw3clogs; }
}
private static List<Grok> _groks = new List<Grok>();
public IEnumerable<Grok> Groks { get { return _groks; } }
public IEnumerable<Grok> Groks
{
get { return _groks; }
}
public Configuration(string xmlConfFile)
{
@@ -107,23 +130,46 @@ namespace TimberWinR
parseConfFilter(xmlConfFile);
}
static void validateWithSchema(string xmlConfFile, string xsdSchema)
private static void validateWithSchema(string xmlConfFile, string xsdSchema)
{
XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri);
// Ensure that the xml configuration file provided obeys the xsd schema.
XmlSchemaSet schemas = new XmlSchemaSet();
schemas.Add("", XmlReader.Create(new StringReader(xsdSchema)));
bool errors = false;
bool errorsFound = false;
config.Validate(schemas, (o, e) =>
{
Console.WriteLine("{0}", e.Message);
errors = true;
});
Console.WriteLine("The XML configuration file provided {0}", errors ? "did not validate." : "validated.");
errorsFound = true;
LogManager.GetCurrentClassLogger().Error(e.Message);
}, true);
if (errorsFound)
DumpInvalidNodes(config.Root);
}
static void DumpInvalidNodes(XElement el)
{
if (el.GetSchemaInfo().Validity != XmlSchemaValidity.Valid)
LogManager.GetCurrentClassLogger().Error("Invalid Element {0}",
el.AncestorsAndSelf()
.InDocumentOrder()
.Aggregate("", (s, i) => s + "/" + i.Name.ToString()));
foreach (XAttribute att in el.Attributes())
if (att.GetSchemaInfo().Validity != XmlSchemaValidity.Valid)
LogManager.GetCurrentClassLogger().Error("Invalid Attribute {0}",
att
.Parent
.AncestorsAndSelf()
.InDocumentOrder()
.Aggregate("",
(s, i) => s + "/" + i.Name.ToString()) + "/@" + att.Name.ToString()
);
foreach (XElement child in el.Elements())
DumpInvalidNodes(child);
}
static void parseConfInput(string xmlConfFile)
{
XDocument config = XDocument.Load(xmlConfFile, LoadOptions.SetLineInfo | LoadOptions.SetBaseUri);
@@ -248,7 +294,7 @@ namespace TimberWinR
// Parse parameters.
Params_TextLog args = parseParams_Log(e.Attributes());
TextLog log = new TextLog(name, location, fields, args);
TailFileInput log = new TailFileInput(name, location, fields, args);
_logs.Add(log);
}
@@ -841,7 +887,19 @@ namespace TimberWinR
throw new MissingRequiredAttributeException(e, attributeName);
}
p.WithMatch(val);
attributeName = "field";
try
{
val = e.Attribute(attributeName).Value;
}
catch
{
throw new MissingRequiredAttributeException(e, attributeName);
}
p.WithField(val);
break;
case "AddField":
string name, value;
attributeName = "name";
@@ -964,7 +1022,7 @@ namespace TimberWinR
}
}
public class TextLog
public class TailFileInput
{
public string Name { get; private set; }
public string Location { get; private set; }
@@ -976,7 +1034,7 @@ namespace TimberWinR
public bool SplitLongLines { get; private set; }
public string ICheckpoint { get; private set; }
public TextLog(string name, string location, List<FieldDefinition> fields, Params_TextLog args)
public TailFileInput(string name, string location, List<FieldDefinition> fields, Params_TextLog args)
{
Name = name;
Location = location;
@@ -1421,6 +1479,7 @@ namespace TimberWinR
public class Grok
{
public string Match { get; private set; }
public string Field { get; private set; }
public Pair AddField { get; private set; }
public bool DropIfMatch { get; private set; }
public string RemoveField { get; private set; }
@@ -1428,6 +1487,7 @@ namespace TimberWinR
public Grok(Params_Grok args)
{
Match = args.Match;
Field = args.Field;
AddField = args.AddField;
DropIfMatch = args.DropIfMatch;
RemoveField = args.RemoveField;
@@ -1452,6 +1512,7 @@ namespace TimberWinR
public class Params_Grok
{
public string Match { get; private set; }
public string Field { get; private set; }
public Pair AddField { get; private set; }
public bool DropIfMatch { get; private set; }
public string RemoveField { get; private set; }
@@ -1459,10 +1520,17 @@ namespace TimberWinR
public class Builder
{
private string match;
private string field;
private Pair addField;
private bool dropIfMatch = false;
private string removeField;
public Builder WithField(string value)
{
field = value;
return this;
}
public Builder WithMatch(string value)
{
match = value;
@@ -1492,6 +1560,7 @@ namespace TimberWinR
return new Params_Grok()
{
Match = match,
Field = field,
AddField = addField,
DropIfMatch = dropIfMatch,
RemoveField = removeField

2
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -92,7 +92,7 @@ namespace TimberWinR.Inputs
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-IISLog"));
ProcessJson(json.ToString());
ProcessJson(json);
}
}
// Close the recordset

12
TimberWinR/Inputs/InputListener.cs
View File

@@ -1,4 +1,5 @@
using System;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
@@ -9,7 +10,7 @@ namespace TimberWinR.Inputs
public abstract class InputListener
{
public CancellationToken CancelToken { get; set; }
public event Action<string> OnMessageRecieved;
public event Action<JObject> OnMessageRecieved;
public InputListener(CancellationToken token)
@@ -17,10 +18,11 @@ namespace TimberWinR.Inputs
this.CancelToken = token;
}
protected void ProcessJson(string message)
{
protected void ProcessJson(JObject json)
{
if (OnMessageRecieved != null)
OnMessageRecieved(message);
OnMessageRecieved(json);
}
}
}

105
TimberWinR/Inputs/TailFileInputListener.cs Normal file
View File

@@ -0,0 +1,105 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using NLog;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using TextLineInputFormat = Interop.MSUtil.COMTextLineInputContextClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
namespace TimberWinR.Inputs
{
/// <summary>
/// Tail a file.
/// </summary>
public class TailFileInputListener : InputListener
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Configuration.TailFileInput _arguments;
public TailFileInputListener(TimberWinR.Configuration.TailFileInput arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
{
_arguments = arguments;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
var task = new Task(FileWatcher, cancelToken);
task.Start();
}
private void FileWatcher()
{
var oLogQuery = new LogQuery();
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
var iFmt = new TextLineInputFormat()
{
iCodepage = _arguments.ICodepage,
splitLongLines = _arguments.SplitLongLines,
iCheckpoint = checkpointFileName,
recurse = _arguments.Recurse
};
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Location);
var firstQuery = true;
// Execute the query
while (!CancelToken.IsCancellationRequested)
{
try
{
var rs = oLogQuery.Execute(query, iFmt);
Dictionary<string, int> colMap = new Dictionary<string, int>();
for (int col=0; col<rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
colMap[colName] = col;
}
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
// We want to "tail" the log, so skip the first query results.
if (!firstQuery)
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
{
if (!colMap.ContainsKey(field.Name))
continue;
object v = record.getValue(field.Name);
if (field.FieldType == typeof(DateTime))
v = field.ToDateTime(v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-FileLog"));
ProcessJson(json);
}
}
// Close the recordset
rs.close();
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
}
}
}

3
TimberWinR/Inputs/TcpInputListener.cs
View File

@@ -5,6 +5,7 @@ using System.Text;
using System.Threading;
using System.Net;
using System.Net.Sockets;
using Newtonsoft.Json.Linq;
using NLog;
namespace TimberWinR.Inputs
@@ -82,7 +83,7 @@ namespace TimberWinR.Inputs
var encoder = new ASCIIEncoding();
var encodedMessage = encoder.GetString(message, 0, bytesRead);
ProcessJson(encodedMessage);
ProcessJson(JObject.Parse(encodedMessage));
}
tcpClient.Close();
}

19
TimberWinR/Inputs/TextLine.cs
View File

@@ -1,19 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using TextLineInputFormat = Interop.MSUtil.COMTextLineInputContextClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
namespace TimberWinR.Inputs
{
/// <summary>
/// Tail a file.
/// </summary>
class TextLine
{
}
}

2
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -83,7 +83,7 @@ namespace TimberWinR.Inputs
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-Eventlog"));
ProcessJson(json.ToString());
ProcessJson(json);
}
}
// Close the recordset

3
TimberWinR/Outputs/OutputSender.cs
View File

@@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading;
using Newtonsoft.Json.Linq;
using TimberWinR.Inputs;
namespace TimberWinR.Outputs
@@ -23,6 +24,6 @@ namespace TimberWinR.Outputs
listener.OnMessageRecieved += MessageReceivedHandler;
}
protected abstract void MessageReceivedHandler(string jsonMessage);
protected abstract void MessageReceivedHandler(JObject jsonMessage);
}
}

89
TimberWinR/Outputs/Redis.cs
View File

@@ -6,14 +6,18 @@ using System.Net.Sockets;
using System.Text;
using System.Threading;
using ctstone.Redis;
using Newtonsoft.Json.Linq;
using NLog;
using System.Threading.Tasks;
using RapidRegex.Core;
using System.Text.RegularExpressions;
using System.Globalization;
namespace TimberWinR.Outputs
{
public class RedisOutput : OutputSender
{
private readonly string _logstashIndexName;
{
private readonly string _logstashIndexName;
private readonly int _port;
private readonly int _timeout;
private readonly object _locker = new object();
@@ -21,6 +25,7 @@ namespace TimberWinR.Outputs
readonly Task _consumerTask;
private readonly string[] _redisHosts;
private int _redisHostIndex;
private TimberWinR.Manager _manager;
/// <summary>
/// Get the next client
@@ -30,36 +35,37 @@ namespace TimberWinR.Outputs
{
if (_redisHostIndex >= _redisHosts.Length)
_redisHostIndex = 0;
int numTries = 0;
while (numTries < _redisHosts.Length)
{
try
{
RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port, _timeout);
_redisHostIndex++;
if (_redisHostIndex >= _redisHosts.Length)
_redisHostIndex = 0;
return client;
}
catch (Exception ex)
{
}
numTries++;
}
}
return null;
}
public RedisOutput(string[] redisHosts, CancellationToken cancelToken, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000)
public RedisOutput(TimberWinR.Manager manager, string[] redisHosts, CancellationToken cancelToken, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000)
: base(cancelToken)
{
_manager = manager;
_redisHostIndex = 0;
_redisHosts = redisHosts;
_jsonQueue = new List<string>();
_jsonQueue = new List<string>();
_port = port;
_timeout = timeout;
_logstashIndexName = logstashIndexName;
@@ -67,18 +73,69 @@ namespace TimberWinR.Outputs
_consumerTask.Start();
}
/// <summary>
/// Forward on Json message to Redis Logstash queue
/// </summary>
/// <param name="jsonMessage"></param>
protected override void MessageReceivedHandler(string jsonMessage)
protected override void MessageReceivedHandler(JObject jsonMessage)
{
LogManager.GetCurrentClassLogger().Info(jsonMessage);
if (_manager.Config.Groks != null)
ProcessGroks(jsonMessage);
var message = jsonMessage.ToString();
LogManager.GetCurrentClassLogger().Info(message);
lock (_locker)
{
_jsonQueue.Add(jsonMessage);
_jsonQueue.Add(message);
}
}
private void ProcessGroks(JObject json)
{
foreach (var grok in _manager.Config.Groks)
{
JToken token = null;
if (json.TryGetValue(grok.Field, StringComparison.OrdinalIgnoreCase, out token))
{
string text = token.ToString();
if (!string.IsNullOrEmpty(text))
{
string expr = grok.Match;
var resolver = new RegexGrokResolver();
var pattern = resolver.ResolveToRegex(expr);
var match = Regex.Match(text, pattern);
if (match.Success)
{
var regex = new Regex(pattern);
var namedCaptures = regex.MatchNamedCaptures(text);
foreach (string fieldName in namedCaptures.Keys)
{
if (fieldName == "timestamp")
{
string value = namedCaptures[fieldName];
DateTime ts;
if (DateTime.TryParse(value, out ts))
json.Add(fieldName, ts.ToUniversalTime());
else if (DateTime.TryParseExact(value, new string[]
{
"MMM dd hh:mm:ss",
"MMM dd HH:mm:ss",
"MMM dd h:mm",
"MMM dd hh:mm",
}, CultureInfo.InvariantCulture, DateTimeStyles.None, out ts))
json.Add(fieldName, ts.ToUniversalTime());
else
json.Add(fieldName, (JToken)namedCaptures[fieldName]);
}
else
json.Add(fieldName, (JToken)namedCaptures[fieldName]);
}
}
}
}
}
}
@@ -120,7 +177,7 @@ namespace TimberWinR.Outputs
{
}
}
client.EndPipe();
client.EndPipe();
break;
}
else
@@ -131,9 +188,9 @@ namespace TimberWinR.Outputs
}
}
}
catch(Exception ex)
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
LogManager.GetCurrentClassLogger().Error(ex);
}
}
}

9
TimberWinR/TimberWinR.csproj
View File

@@ -46,7 +46,8 @@
<Reference Include="NLog">
<HintPath>..\packages\NLog.3.1.0.0\lib\net40\NLog.dll</HintPath>
</Reference>
<Reference Include="RapidRegex.Core">
<Reference Include="RapidRegex.Core, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\RapidRegex.Core.1.0.0.0\lib\net40\RapidRegex.Core.dll</HintPath>
</Reference>
<Reference Include="System" />
@@ -68,7 +69,7 @@
<Compile Include="Inputs\InputListener.cs" />
<Compile Include="Inputs\ParameterDefinitions.cs" />
<Compile Include="Inputs\TcpInputListener.cs" />
<Compile Include="Inputs\TextLine.cs" />
<Compile Include="Inputs\TailFileInputListener.cs" />
<Compile Include="Inputs\WindowsEvtInputListener.cs" />
<Compile Include="Manager.cs" />
<Compile Include="Outputs\OutputSender.cs" />
@@ -81,7 +82,9 @@
</Compile>
</ItemGroup>
<ItemGroup>
<Content Include="lib\com-logparser\Interop.MSUtil.dll" />
<Content Include="lib\com-logparser\Interop.MSUtil.dll">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<None Include="configSchema.xsd">

1
TimberWinR/configSchema.xsd
View File

@@ -260,6 +260,7 @@
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute type="xs:string" name="field" use="required"/>
<xs:attribute type="xs:string" name="value" use="required"/>
</xs:extension>
</xs:simpleContent>

BIN
packages/RapidRegex.Core.1.0.0.0/lib/net40/RapidRegex.Core.dll vendored
View File

Binary file not shown.