TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug with IISW3C parser

Browse Source
This commit is contained in:
Eric Fontana
2014-07-18 09:22:32 -04:00
parent 2d0001c190
commit 31df124aff
8 changed files with 181 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
TimberWinR.ServiceHost/Program.cs
View File

@@ -94,6 +94,12 @@ namespace TimberWinR.ServiceHost
_nlogListener = new TcpInputListener(_cancellationToken, 5140);
outputRedis.Connect(_nlogListener);
foreach (Configuration.IISW3CLog iisw3cConfig in manager.Config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
foreach (Configuration.WindowsEvent eventConfig in manager.Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken);

8
TimberWinR.UnitTests/testconf.xml
View File

@@ -2,7 +2,7 @@
<TimberWinR>
<Inputs>
<WindowsEvents>
<Event source="System,Application,Security" binaryFormat="PRINT" />
<Event source="System,Application" binaryFormat="PRINT" />
</WindowsEvents>
<Logs>
<Log name="First Set" location="C:\Logs1\*.log" />
@@ -10,9 +10,9 @@
<Log name="Third Set" location="C:\Logs2\1.log,C:\Logs2\2.log" />
</Logs>
<IISLogs>
</IISLogs>
<IISW3CLogs>
<IISW3CLog name="Default site" location="c:\inetpub\logs\LogFiles\W3SVC1\*" />
</IISW3CLogs>
</Inputs>
</TimberWinR>

52
TimberWinR.UnitTests/testconf.xsd
View File

@@ -8,10 +8,10 @@
<xs:element name="WindowsEvents" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Events" maxOccurs="unbounded" minOccurs="0">
<xs:element name="Event" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">
iis <xs:element name="Fields" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
@@ -138,7 +138,53 @@
<xs:element name="IISLogs" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="IIS" maxOccurs="unbounded" minOccurs="0">
<xs:element name="IISLog" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
<xs:complexType>
<xs:simpleContent>
<xs:extension base="xs:string">
<xs:attribute name="name" use="required">
<xs:simpleType>
<xs:restriction base="xs:string">
<xs:pattern value="LogFilename|LogRow|UserIP|UserName|Date|Time|ServiceInstance|HostName|ServerIP|TimeTaken|BytesSent|BytesReceived|StatusCode|Win32StatusCode|RequestType|Target|Parameters" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
</xs:extension>
</xs:simpleContent>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
</xs:sequence>
<xs:attribute type="xs:string" name="name" use="required"/>
<xs:attribute type="xs:string" name="location" use="required"/>
<xs:attribute type="xs:int" name="iCodepage" use="optional"/>
<xs:attribute type="xs:int" name="recurse" use="optional"/>
<xs:attribute name="minDateMod" use="optional">
<xs:simpleType>
<xs:restriction base="xs:token">
<xs:pattern value="[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}" />
</xs:restriction>
</xs:simpleType>
</xs:attribute>
<xs:attribute type="xs:string" name="locale" use="optional"/>
<xs:attribute type="xs:string" name="iCheckpoint" use="optional"/>
</xs:complexType>
</xs:element>
</xs:sequence>
</xs:complexType>
</xs:element>
<xs:element name="IISW3CLogs" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="IISW3CLog" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">

13
TimberWinR/Configuration.cs
View File

@@ -244,7 +244,7 @@ namespace TimberWinR
IEnumerable<XElement> xml_iisw3c =
from el in inputs.Descendants("IISW3CLogs").Descendants("IISW3CLog")
select el;
foreach (XElement e in xml_iis)
foreach (XElement e in xml_iisw3c)
{
// Required attributes.
string name;
@@ -285,10 +285,7 @@ namespace TimberWinR
IISW3CLog iisw3c = new IISW3CLog(name, location, fields, args);
_iisw3clogs.Add(iisw3c);
}
Console.WriteLine("end");
}
}
static List<FieldDefinition> parseFields_Event(IEnumerable<XElement> xml_fields)
@@ -488,9 +485,9 @@ namespace TimberWinR
{ "time-taken", typeof(int) },
{ "cs-version", typeof(string) },
{ "cs-host", typeof(string) },
{ "cs (User-Agent)", typeof(string) },
{ "cs (Cookie)", typeof(string) },
{ "cs (Referer)", typeof(string) },
{ "cs(User-Agent)", typeof(string) },
{ "cs(Cookie)", typeof(string) },
{ "cs(Referer)", typeof(string) },
{ "s-event", typeof(string) },
{ "s-process-type", typeof(string) },
{ "s-user-time", typeof(double) },

110
TimberWinR/Inputs/IISW3CInputListener.cs Normal file
View File

@@ -0,0 +1,110 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Security.AccessControl;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using NLog;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using IISW3CLogInputFormat = Interop.MSUtil.COMIISW3CInputContextClassClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
namespace TimberWinR.Inputs
{
public class IISW3CInputListener : InputListener
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Configuration.IISW3CLog _arguments;
public IISW3CInputListener(TimberWinR.Configuration.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken)
{
_arguments = arguments;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
var task = new Task(IISW3CWatcher, cancelToken);
task.Start();
}
private void IISW3CWatcher()
{
var oLogQuery = new LogQuery();
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
var iFmt = new IISW3CLogInputFormat()
{
codepage = _arguments.ICodepage,
consolidateLogs = _arguments.ConsolidateLogs,
dirTime = _arguments.DirTime,
dQuotes = _arguments.DQuotes,
iCheckpoint = checkpointFileName,
recurse = _arguments.Recurse,
useDoubleQuotes = _arguments.DQuotes
};
if (!string.IsNullOrEmpty(_arguments.MinDateMod))
iFmt.minDateMod = _arguments.MinDateMod;
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Location);
var firstQuery = true;
// Execute the query
while (!CancelToken.IsCancellationRequested)
{
try
{
var rs = oLogQuery.Execute(query, iFmt);
Dictionary<string, int> colMap = new Dictionary<string, int>();
for (int col=0; col<rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
colMap[colName] = col;
}
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
// We want to "tail" the log, so skip the first query results.
if (!firstQuery)
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
{
if (!colMap.ContainsKey(field.Name))
continue;
object v = record.getValue(field.Name);
if (field.FieldType == typeof(DateTime))
v = field.ToDateTime(v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
}
json.Add(new JProperty("type", "Win32-IISLog"));
ProcessJson(json.ToString());
}
}
// Close the recordset
rs.close();
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
}
}
}

4
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -38,7 +38,7 @@ namespace TimberWinR.Inputs
{
var oLogQuery = new LogQuery();
var fileName = Path.Combine(System.IO.Path.GetTempPath(),
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
// Instantiate the Event Log Input Format object
@@ -52,7 +52,7 @@ namespace TimberWinR.Inputs
msgErrorMode = _arguments.MsgErrorMode,
stringsSep = _arguments.StringsSep,
resolveSIDs = _arguments.ResolveSIDS,
iCheckpoint = fileName,
iCheckpoint = checkpointFileName,
};
// Create the query

4
TimberWinR/TimberWinR.csproj
View File

@@ -61,6 +61,7 @@
<ItemGroup>
<Compile Include="Configuration.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\IISW3CInputListener.cs" />
<Compile Include="Inputs\InputListener.cs" />
<Compile Include="Inputs\ParameterDefinitions.cs" />
<Compile Include="Inputs\TcpInputListener.cs" />
@@ -76,6 +77,9 @@
</ItemGroup>
<ItemGroup>
<None Include="packages.config" />
<None Include="testconf.xsd">
<SubType>Designer</SubType>
</None>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.

2
TimberWinR/testconf.xsd
View File

@@ -8,7 +8,7 @@
<xs:element name="WindowsEvents" minOccurs="0">
<xs:complexType>
<xs:sequence>
<xs:element name="Events" maxOccurs="unbounded" minOccurs="0">
<xs:element name="Event" maxOccurs="unbounded" minOccurs="0">
<xs:complexType mixed="true">
<xs:sequence>
<xs:element name="Fields" minOccurs="0">