TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #13 from gediminasgu/iislog_timestamp_col

Browse Source 
(feat): @timestamp column for IISW3C input now is parsed from date and t...
This commit is contained in:
Eric Fontana
2014-11-30 14:07:39 -05:00
parent 5656f5b03f e25024da9d
commit 85fcf11503
8 changed files with 173 additions and 32 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -154,3 +154,4 @@ $RECYCLE.BIN/
# Mac desktop service store files
.DS_Store
packages

60
TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs Normal file
View File

@@ -0,0 +1,60 @@
namespace TimberWinR.UnitTests.Inputs
{
using System;
using System.Collections.Generic;
using Interop.MSUtil;
using Moq;
using NUnit.Framework;
using TimberWinR.Inputs;
using TimberWinR.Parser;
[TestFixture]
public class IisW3CRowReaderTests : TestBase
{
private IisW3CRowReader reader;
public override void Setup()
{
base.Setup();
var fields = new List<Field>
{
new Field("date", "DateTime"),
new Field("time", "DateTime"),
new Field("uri")
};
this.reader = new IisW3CRowReader(fields);
var recordset = this.GetRecordsetMock();
this.reader.ReadColumnMap(recordset.Object);
}
[Test]
public void GivenValidRowAddsTimestampColumn()
{
var record = this.MockRepository.Create<ILogRecord>();
record.Setup(x => x.getValue("date")).Returns(new DateTime(2014, 11, 30));
record.Setup(x => x.getValue("time")).Returns(new DateTime(1, 1, 1, 18, 45, 37, 590));
record.Setup(x => x.getValue("uri")).Returns("http://somedomain.com/someurl");
var json = this.reader.ReadToJson(record.Object);
Assert.AreEqual("2014-11-30T18:45:37.000Z", json["@timestamp"].ToString());
Assert.AreEqual("http://somedomain.com/someurl", json["uri"].ToString());
}
private Mock<ILogRecordset> GetRecordsetMock()
{
var recordset = this.MockRepository.Create<ILogRecordset>();
recordset.Setup(x => x.getColumnCount()).Returns(3);
recordset.Setup(x => x.getColumnName(0)).Returns("date");
recordset.Setup(x => x.getColumnName(1)).Returns("time");
recordset.Setup(x => x.getColumnName(2)).Returns("uri");
return recordset;
}
}
}

23
TimberWinR.UnitTests/TestBase.cs Normal file
View File

@@ -0,0 +1,23 @@
namespace TimberWinR.UnitTests
{
using Moq;
using NUnit.Framework;
public class TestBase
{
public MockRepository MockRepository { get; private set; }
[SetUp]
public virtual void Setup()
{
this.MockRepository = new MockRepository(MockBehavior.Default);
}
[TearDown]
public virtual void TearDown()
{
this.MockRepository.VerifyAll();
}
}
}

10
TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
View File

@@ -33,6 +33,14 @@
<Prefer32Bit>false</Prefer32Bit>
</PropertyGroup>
<ItemGroup>
<Reference Include="Interop.MSUtil, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<EmbedInteropTypes>False</EmbedInteropTypes>
<HintPath>..\TimberWinR\lib\com-logparser\Interop.MSUtil.dll</HintPath>
</Reference>
<Reference Include="Moq">
<HintPath>..\packages\Moq.4.2.1409.1722\lib\net40\Moq.dll</HintPath>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
@@ -53,9 +61,11 @@
<Compile Include="Configuration.cs" />
<Compile Include="DateFilterTests.cs" />
<Compile Include="GeoIPFilterTests.cs" />
<Compile Include="Inputs\IisW3CRowReaderTests.cs" />
<Compile Include="JsonFilterTests.cs" />
<Compile Include="GrokFilterTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="TestBase.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

1
TimberWinR.UnitTests/packages.config
View File

@@ -1,5 +1,6 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="Moq" version="4.2.1409.1722" targetFramework="net40" />
<package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
<package id="NUnit" version="2.6.3" targetFramework="net40" />
</packages>

40
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -1,18 +1,11 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Security.AccessControl;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using NLog;
using TimberWinR.Parser;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using IISW3CLogInputFormat = Interop.MSUtil.COMIISW3CInputContextClassClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
@@ -23,15 +16,19 @@ namespace TimberWinR.Inputs
public class IISW3CInputListener : InputListener
{
private readonly int _pollingIntervalInSeconds;
private readonly TimberWinR.Parser.IISW3CLog _arguments;
private readonly Parser.IISW3CLog _arguments;
private long _receivedMessages;
public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
private IisW3CRowReader rowReader;
public IISW3CInputListener(Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
: base(cancelToken, "Win32-IISLog")
{
_arguments = arguments;
_receivedMessages = 0;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
this.rowReader = new IisW3CRowReader(_arguments.Fields);
foreach (string loc in _arguments.Location.Split(','))
{
string hive = loc.Trim();
@@ -62,7 +59,6 @@ namespace TimberWinR.Inputs
return json;
}
private void IISW3CWatcher(string location)
{
LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location);
@@ -108,39 +104,19 @@ namespace TimberWinR.Inputs
}
}
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName, lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
var colMap = new Dictionary<string, int>();
for (int col = 0; col < rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
colMap[colName] = col;
}
rowReader.ReadColumnMap(rs);
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in _arguments.Fields)
{
if (!colMap.ContainsKey(field.Name))
continue;
object v = record.getValue(field.Name);
if (field.DataType == typeof (DateTime))
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field.Name, dt));
}
else
json.Add(new JProperty(field.Name, v));
}
var json = rowReader.ReadToJson(record);
ProcessJson(json);
_receivedMessages++;
var lrn = (Int64)record.getValueEx("LogRow");

69
TimberWinR/Inputs/IISW3CRowReader.cs Normal file
View File

@@ -0,0 +1,69 @@
namespace TimberWinR.Inputs
{
using System;
using System.Collections.Generic;
using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using TimberWinR.Parser;
public class IisW3CRowReader
{
private readonly List<Field> fields;
private IDictionary<string, int> columnMap;
public IisW3CRowReader(List<Field> fields)
{
this.fields = fields;
}
public JObject ReadToJson(ILogRecord row)
{
var json = new JObject();
foreach (var field in this.fields)
{
if (this.columnMap.ContainsKey(field.Name))
{
object v = row.getValue(field.Name);
if (field.DataType == typeof(DateTime))
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field.Name, dt));
}
else
{
json.Add(new JProperty(field.Name, v));
}
}
}
AddTimestamp(json);
return json;
}
public void ReadColumnMap(ILogRecordset rs)
{
this.columnMap = new Dictionary<string, int>();
for (int col = 0; col < rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
this.columnMap[colName] = col;
}
}
private static void AddTimestamp(JObject json)
{
if (json["date"] != null && json["time"] != null)
{
var date = DateTime.Parse(json["date"].ToString());
var time = DateTime.Parse(json["time"].ToString());
date = new DateTime(date.Year, date.Month, date.Day, time.Hour, time.Minute, time.Second, time.Millisecond);
json.Add(new JProperty("@timestamp", date.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")));
}
}
}
}

1
TimberWinR/TimberWinR.csproj
View File

@@ -86,6 +86,7 @@
<Compile Include="Filters\JsonFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\IISW3CRowReader.cs" />
<Compile Include="Inputs\UdpInputListener.cs" />
<Compile Include="Inputs\W3CInputListener.cs" />
<Compile Include="Inputs\IISW3CInputListener.cs" />