Merge pull request #13 from gediminasgu/iislog_timestamp_col

(feat): @timestamp column for IISW3C input now is parsed from date and t...

.gitignore

@@ -154,3 +154,4 @@ $RECYCLE.BIN/
 
 # Mac desktop service store files
 .DS_Store
+packages

TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs

@@ -0,0 +1,60 @@
+namespace TimberWinR.UnitTests.Inputs
+{
+    using System;
+    using System.Collections.Generic;
+
+    using Interop.MSUtil;
+
+    using Moq;
+
+    using NUnit.Framework;
+
+    using TimberWinR.Inputs;
+    using TimberWinR.Parser;
+
+    [TestFixture]
+    public class IisW3CRowReaderTests : TestBase
+    {
+        private IisW3CRowReader reader;
+
+        public override void Setup()
+        {
+            base.Setup();
+            var fields = new List<Field>
+                             {
+                                 new Field("date", "DateTime"),
+                                 new Field("time", "DateTime"),
+                                 new Field("uri")
+                             };
+            this.reader = new IisW3CRowReader(fields);
+
+            var recordset = this.GetRecordsetMock();
+            this.reader.ReadColumnMap(recordset.Object);
+        }
+
+        [Test]
+        public void GivenValidRowAddsTimestampColumn()
+        {
+            var record = this.MockRepository.Create<ILogRecord>();
+            record.Setup(x => x.getValue("date")).Returns(new DateTime(2014, 11, 30));
+            record.Setup(x => x.getValue("time")).Returns(new DateTime(1, 1, 1, 18, 45, 37, 590));
+            record.Setup(x => x.getValue("uri")).Returns("http://somedomain.com/someurl");
+
+            var json = this.reader.ReadToJson(record.Object);
+
+            Assert.AreEqual("2014-11-30T18:45:37.000Z", json["@timestamp"].ToString());
+            Assert.AreEqual("http://somedomain.com/someurl", json["uri"].ToString());
+        }
+
+        private Mock<ILogRecordset> GetRecordsetMock()
+        {
+            var recordset = this.MockRepository.Create<ILogRecordset>();
+            recordset.Setup(x => x.getColumnCount()).Returns(3);
+
+            recordset.Setup(x => x.getColumnName(0)).Returns("date");
+            recordset.Setup(x => x.getColumnName(1)).Returns("time");
+            recordset.Setup(x => x.getColumnName(2)).Returns("uri");
+            return recordset;
+        }
+    }
+}

TimberWinR.UnitTests/TestBase.cs

@@ -0,0 +1,23 @@
+namespace TimberWinR.UnitTests
+{
+    using Moq;
+
+    using NUnit.Framework;
+
+    public class TestBase
+    {
+        public MockRepository MockRepository { get; private set; }
+
+        [SetUp]
+        public virtual void Setup()
+        {
+            this.MockRepository = new MockRepository(MockBehavior.Default);
+        }
+
+        [TearDown]
+        public virtual void TearDown()
+        {
+            this.MockRepository.VerifyAll();
+        }
+    }
+}

TimberWinR.UnitTests/TimberWinR.UnitTests.csproj

@@ -33,6 +33,14 @@
     <Prefer32Bit>false</Prefer32Bit>
   </PropertyGroup>
   <ItemGroup>
+    <Reference Include="Interop.MSUtil, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <EmbedInteropTypes>False</EmbedInteropTypes>
+      <HintPath>..\TimberWinR\lib\com-logparser\Interop.MSUtil.dll</HintPath>
+    </Reference>
+    <Reference Include="Moq">
+      <HintPath>..\packages\Moq.4.2.1409.1722\lib\net40\Moq.dll</HintPath>
+    </Reference>
     <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
       <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
@@ -53,9 +61,11 @@
     <Compile Include="Configuration.cs" />
     <Compile Include="DateFilterTests.cs" />
     <Compile Include="GeoIPFilterTests.cs" />
+    <Compile Include="Inputs\IisW3CRowReaderTests.cs" />
     <Compile Include="JsonFilterTests.cs" />
     <Compile Include="GrokFilterTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="TestBase.cs" />
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

TimberWinR.UnitTests/packages.config

@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
+  <package id="Moq" version="4.2.1409.1722" targetFramework="net40" />
   <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
   <package id="NUnit" version="2.6.3" targetFramework="net40" />
 </packages>

TimberWinR/Inputs/IISW3CInputListener.cs

@@ -1,18 +1,11 @@
 using System;
 using System.Collections.Generic;
-using System.ComponentModel;
 using System.Linq;
-using System.Security.AccessControl;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
-using System.IO;
-using Interop.MSUtil;
 
 using Newtonsoft.Json.Linq;
-using Newtonsoft.Json.Serialization;
 using NLog;
-using TimberWinR.Parser;
 using LogQuery = Interop.MSUtil.LogQueryClassClass;
 using IISW3CLogInputFormat = Interop.MSUtil.COMIISW3CInputContextClassClass;
 using LogRecordSet = Interop.MSUtil.ILogRecordset;
@@ -23,15 +16,19 @@ namespace TimberWinR.Inputs
     public class IISW3CInputListener : InputListener
     {
         private readonly int _pollingIntervalInSeconds;
-        private readonly TimberWinR.Parser.IISW3CLog _arguments;
+        private readonly Parser.IISW3CLog _arguments;
         private long _receivedMessages;
 
-        public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
+        private IisW3CRowReader rowReader;
+
+        public IISW3CInputListener(Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
             : base(cancelToken, "Win32-IISLog")
         {
             _arguments = arguments;
             _receivedMessages = 0;
             _pollingIntervalInSeconds = pollingIntervalInSeconds;
+            this.rowReader = new IisW3CRowReader(_arguments.Fields);
+
             foreach (string loc in _arguments.Location.Split(','))
             {
                 string hive = loc.Trim();
@@ -62,7 +59,6 @@ namespace TimberWinR.Inputs
             return json;
         }
 
-
         private void IISW3CWatcher(string location)
         {
             LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location);
@@ -108,39 +104,19 @@ namespace TimberWinR.Inputs
                         }
                     }
 
-                 
                     foreach (string fileName in logFileMaxRecords.Keys.ToList())
                     {
                         var lastRecordNumber = logFileMaxRecords[fileName];
                         var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName, lastRecordNumber);
 
                         var rs = oLogQuery.Execute(query, iFmt);
-                        var colMap = new Dictionary<string, int>();
+                        rowReader.ReadColumnMap(rs);
-                        for (int col = 0; col < rs.getColumnCount(); col++)
-                        {
-                            string colName = rs.getColumnName(col);
-                            colMap[colName] = col;
-                        }
 
                         // Browse the recordset
                         for (; !rs.atEnd(); rs.moveNext())
                         {
                             var record = rs.getRecord();
-                            var json = new JObject();
+                            var json = rowReader.ReadToJson(record);
-                            foreach (var field in _arguments.Fields)
-                            {
-                                if (!colMap.ContainsKey(field.Name))
-                                    continue;
-
-                                object v = record.getValue(field.Name);
-                                if (field.DataType == typeof (DateTime))
-                                {
-                                    DateTime dt = DateTime.Parse(v.ToString());
-                                    json.Add(new JProperty(field.Name, dt));
-                                }
-                                else
-                                    json.Add(new JProperty(field.Name, v));
-                            }
                             ProcessJson(json);
                             _receivedMessages++;
                             var lrn = (Int64)record.getValueEx("LogRow");

TimberWinR/Inputs/IISW3CRowReader.cs

@@ -0,0 +1,69 @@
+namespace TimberWinR.Inputs
+{
+    using System;
+    using System.Collections.Generic;
+
+    using Interop.MSUtil;
+
+    using Newtonsoft.Json.Linq;
+
+    using TimberWinR.Parser;
+
+    public class IisW3CRowReader
+    {
+        private readonly List<Field> fields;
+        private IDictionary<string, int> columnMap;
+
+        public IisW3CRowReader(List<Field> fields)
+        {
+            this.fields = fields;
+        }
+
+        public JObject ReadToJson(ILogRecord row)
+        {
+            var json = new JObject();
+            foreach (var field in this.fields)
+            {
+                if (this.columnMap.ContainsKey(field.Name))
+                {
+                    object v = row.getValue(field.Name);
+                    if (field.DataType == typeof(DateTime))
+                    {
+                        DateTime dt = DateTime.Parse(v.ToString());
+                        json.Add(new JProperty(field.Name, dt));
+                    }
+                    else
+                    {
+                        json.Add(new JProperty(field.Name, v));
+                    }
+                }
+            }
+
+            AddTimestamp(json);
+
+            return json;
+        }
+
+        public void ReadColumnMap(ILogRecordset rs)
+        {
+            this.columnMap = new Dictionary<string, int>();
+            for (int col = 0; col < rs.getColumnCount(); col++)
+            {
+                string colName = rs.getColumnName(col);
+                this.columnMap[colName] = col;
+            }
+        }
+
+        private static void AddTimestamp(JObject json)
+        {
+            if (json["date"] != null && json["time"] != null)
+            {
+                var date = DateTime.Parse(json["date"].ToString());
+                var time = DateTime.Parse(json["time"].ToString());
+                date = new DateTime(date.Year, date.Month, date.Day, time.Hour, time.Minute, time.Second, time.Millisecond);
+
+                json.Add(new JProperty("@timestamp", date.ToString("yyyy-MM-ddTHH:mm:ss.fffZ")));
+            }
+        }
+    }
+}

TimberWinR/TimberWinR.csproj

@@ -86,6 +86,7 @@
     <Compile Include="Filters\JsonFilter.cs" />
     <Compile Include="Filters\MutateFilter.cs" />
     <Compile Include="Inputs\FieldDefinitions.cs" />
+    <Compile Include="Inputs\IISW3CRowReader.cs" />
     <Compile Include="Inputs\UdpInputListener.cs" />
     <Compile Include="Inputs\W3CInputListener.cs" />
     <Compile Include="Inputs\IISW3CInputListener.cs" />