Fixed bug with IISW3C parser

TimberWinR.ServiceHost/Program.cs

@@ -94,6 +94,12 @@ namespace TimberWinR.ServiceHost
             _nlogListener = new TcpInputListener(_cancellationToken, 5140);
             outputRedis.Connect(_nlogListener);
 
+            foreach (Configuration.IISW3CLog iisw3cConfig in manager.Config.IISW3C)
+            {
+                var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken);
+                outputRedis.Connect(elistner);
+            }
+        
             foreach (Configuration.WindowsEvent eventConfig in manager.Config.Events)
             {
                 var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken);

TimberWinR.UnitTests/testconf.xml

@@ -2,7 +2,7 @@
 <TimberWinR>
   <Inputs>
     <WindowsEvents>
-      <Event source="System,Application,Security" binaryFormat="PRINT" />
+      <Event source="System,Application" binaryFormat="PRINT" />
     </WindowsEvents>
     <Logs>
       <Log name="First Set" location="C:\Logs1\*.log" />
@@ -10,9 +10,9 @@
       <Log name="Third Set" location="C:\Logs2\1.log,C:\Logs2\2.log" />
     </Logs>
 
-    <IISLogs>
+    <IISW3CLogs>
-
+      <IISW3CLog name="Default site" location="c:\inetpub\logs\LogFiles\W3SVC1\*" />
-    </IISLogs>
+    </IISW3CLogs>
   </Inputs>
 
 </TimberWinR>

TimberWinR.UnitTests/testconf.xsd

@@ -8,10 +8,10 @@
               <xs:element name="WindowsEvents" minOccurs="0">
                 <xs:complexType>
                   <xs:sequence>
-                    <xs:element name="Events" maxOccurs="unbounded" minOccurs="0">
+                    <xs:element name="Event" maxOccurs="unbounded" minOccurs="0">
                       <xs:complexType mixed="true">
                         <xs:sequence>
-                          <xs:element name="Fields" minOccurs="0">
+             iis             <xs:element name="Fields" minOccurs="0">
                             <xs:complexType>
                               <xs:sequence>
                                 <xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
@@ -138,7 +138,53 @@
               <xs:element name="IISLogs" minOccurs="0">
                 <xs:complexType>
                   <xs:sequence>
-                    <xs:element name="IIS" maxOccurs="unbounded" minOccurs="0">
+                    <xs:element name="IISLog" maxOccurs="unbounded" minOccurs="0">
+                      <xs:complexType mixed="true">
+                        <xs:sequence>
+                          <xs:element name="Fields" minOccurs="0">
+                            <xs:complexType>
+                              <xs:sequence>
+                                <xs:element name="Field" maxOccurs="unbounded" minOccurs="0">
+                                  <xs:complexType>
+                                    <xs:simpleContent>
+                                      <xs:extension base="xs:string">
+                                        <xs:attribute name="name" use="required">
+                                          <xs:simpleType>
+                                            <xs:restriction base="xs:string">
+                                              <xs:pattern value="LogFilename|LogRow|UserIP|UserName|Date|Time|ServiceInstance|HostName|ServerIP|TimeTaken|BytesSent|BytesReceived|StatusCode|Win32StatusCode|RequestType|Target|Parameters" />
+                                            </xs:restriction>
+                                          </xs:simpleType>
+                                        </xs:attribute>
+                                      </xs:extension>
+                                    </xs:simpleContent>
+                                  </xs:complexType>
+                                </xs:element>
+                              </xs:sequence>
+                            </xs:complexType>
+                          </xs:element>
+                        </xs:sequence>
+                        <xs:attribute type="xs:string" name="name" use="required"/>
+                        <xs:attribute type="xs:string" name="location" use="required"/>
+                        <xs:attribute type="xs:int" name="iCodepage" use="optional"/>
+                        <xs:attribute type="xs:int" name="recurse" use="optional"/>
+                        <xs:attribute name="minDateMod" use="optional">
+                          <xs:simpleType>
+                            <xs:restriction base="xs:token">
+                              <xs:pattern value="[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2}" />
+                            </xs:restriction>
+                          </xs:simpleType>
+                        </xs:attribute>
+                        <xs:attribute type="xs:string" name="locale" use="optional"/>
+                        <xs:attribute type="xs:string" name="iCheckpoint" use="optional"/>
+                      </xs:complexType>
+                    </xs:element>
+                  </xs:sequence>
+                </xs:complexType>
+              </xs:element>
+              <xs:element name="IISW3CLogs" minOccurs="0">
+                <xs:complexType>
+                  <xs:sequence>
+                    <xs:element name="IISW3CLog" maxOccurs="unbounded" minOccurs="0">
                       <xs:complexType mixed="true">
                         <xs:sequence>
                           <xs:element name="Fields" minOccurs="0">

TimberWinR/Configuration.cs

@@ -244,7 +244,7 @@ namespace TimberWinR
             IEnumerable<XElement> xml_iisw3c =
                 from el in inputs.Descendants("IISW3CLogs").Descendants("IISW3CLog")
                 select el;
-            foreach (XElement e in xml_iis)
+            foreach (XElement e in xml_iisw3c)
             {
                 // Required attributes.
                 string name;
@@ -285,10 +285,7 @@ namespace TimberWinR
 
                 IISW3CLog iisw3c = new IISW3CLog(name, location, fields, args);
                 _iisw3clogs.Add(iisw3c);
-            }
+            }           
-
-
-            Console.WriteLine("end");
         }
 
         static List<FieldDefinition> parseFields_Event(IEnumerable<XElement> xml_fields)
@@ -488,9 +485,9 @@ namespace TimberWinR
                 { "time-taken", typeof(int) },
                 { "cs-version", typeof(string) },
                 { "cs-host", typeof(string) },
-                { "cs (User-Agent)", typeof(string) },
+                { "cs(User-Agent)", typeof(string) },
-                { "cs (Cookie)", typeof(string) },
+                { "cs(Cookie)", typeof(string) },
-                { "cs (Referer)", typeof(string) },
+                { "cs(Referer)", typeof(string) },
                 { "s-event", typeof(string) },
                 { "s-process-type", typeof(string) },
                 { "s-user-time", typeof(double) },

TimberWinR/Inputs/IISW3CInputListener.cs

@@ -0,0 +1,110 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Security.AccessControl;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.IO;
+using Interop.MSUtil;
+
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+using NLog;
+using LogQuery = Interop.MSUtil.LogQueryClassClass;
+using IISW3CLogInputFormat = Interop.MSUtil.COMIISW3CInputContextClassClass;
+using LogRecordSet = Interop.MSUtil.ILogRecordset;
+
+
+namespace TimberWinR.Inputs
+{
+    public class IISW3CInputListener : InputListener
+    {
+        private int _pollingIntervalInSeconds = 1;
+        private TimberWinR.Configuration.IISW3CLog _arguments;
+
+
+        public IISW3CInputListener(TimberWinR.Configuration.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
+            : base(cancelToken)
+        {
+            _arguments = arguments;
+            _pollingIntervalInSeconds = pollingIntervalInSeconds;
+            var task = new Task(IISW3CWatcher, cancelToken);
+            task.Start();
+        }
+
+        private void IISW3CWatcher()
+        {
+            var oLogQuery = new LogQuery();
+
+            var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
+                string.Format("{0}.lpc", Guid.NewGuid().ToString()));
+
+            var iFmt = new IISW3CLogInputFormat()
+            {
+                codepage = _arguments.ICodepage,
+                consolidateLogs = _arguments.ConsolidateLogs,
+                dirTime = _arguments.DirTime,
+                dQuotes = _arguments.DQuotes,
+                iCheckpoint = checkpointFileName,               
+                recurse = _arguments.Recurse,
+                useDoubleQuotes = _arguments.DQuotes
+            };
+
+            if (!string.IsNullOrEmpty(_arguments.MinDateMod))
+                iFmt.minDateMod = _arguments.MinDateMod;
+
+            // Create the query
+            var query = string.Format("SELECT * FROM {0}", _arguments.Location);
+
+            var firstQuery = true;
+            // Execute the query
+            while (!CancelToken.IsCancellationRequested)
+            {
+                try
+                {
+                    var rs = oLogQuery.Execute(query, iFmt);
+                    Dictionary<string, int> colMap = new Dictionary<string, int>();
+                    for (int col=0; col<rs.getColumnCount(); col++)
+                    {
+                        string colName = rs.getColumnName(col);
+                        colMap[colName] = col;
+                    }
+
+                    // Browse the recordset
+                    for (; !rs.atEnd(); rs.moveNext())
+                    {
+                        // We want to "tail" the log, so skip the first query results.
+                        if (!firstQuery)
+                        {
+                            var record = rs.getRecord();
+                            var json = new JObject();
+                            foreach (var field in _arguments.Fields)
+                            {
+                                if (!colMap.ContainsKey(field.Name))
+                                    continue;
+
+                                object v = record.getValue(field.Name);
+
+                                if (field.FieldType == typeof(DateTime))
+                                    v = field.ToDateTime(v).ToUniversalTime();
+
+                                json.Add(new JProperty(field.Name, v));
+                            }
+                            json.Add(new JProperty("type", "Win32-IISLog"));
+                            ProcessJson(json.ToString());
+                        }
+                    }
+                    // Close the recordset
+                    rs.close();
+                }
+                catch (Exception ex)
+                {
+                    LogManager.GetCurrentClassLogger().Error(ex);
+                }
+                firstQuery = false;
+                System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
+            }
+        }
+    }
+}

TimberWinR/Inputs/WindowsEvtInputListener.cs

@@ -38,7 +38,7 @@ namespace TimberWinR.Inputs
         {
             var oLogQuery = new LogQuery();
 
-            var fileName = Path.Combine(System.IO.Path.GetTempPath(),
+            var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
                 string.Format("{0}.lpc", Guid.NewGuid().ToString()));          
           
             // Instantiate the Event Log Input Format object
@@ -52,7 +52,7 @@ namespace TimberWinR.Inputs
                 msgErrorMode =  _arguments.MsgErrorMode,
                 stringsSep = _arguments.StringsSep,
                 resolveSIDs = _arguments.ResolveSIDS,
-                iCheckpoint = fileName,               
+                iCheckpoint = checkpointFileName,               
             };
 
             // Create the query

TimberWinR/TimberWinR.csproj

@@ -61,6 +61,7 @@
   <ItemGroup>
     <Compile Include="Configuration.cs" />
     <Compile Include="Inputs\FieldDefinitions.cs" />
+    <Compile Include="Inputs\IISW3CInputListener.cs" />
     <Compile Include="Inputs\InputListener.cs" />
     <Compile Include="Inputs\ParameterDefinitions.cs" />
     <Compile Include="Inputs\TcpInputListener.cs" />
@@ -76,6 +77,9 @@
   </ItemGroup>
   <ItemGroup>
     <None Include="packages.config" />
+    <None Include="testconf.xsd">
+      <SubType>Designer</SubType>
+    </None>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 

TimberWinR/testconf.xsd

@@ -8,7 +8,7 @@
               <xs:element name="WindowsEvents" minOccurs="0">
                 <xs:complexType>
                   <xs:sequence>
-                    <xs:element name="Events" maxOccurs="unbounded" minOccurs="0">
+                    <xs:element name="Event" maxOccurs="unbounded" minOccurs="0">
                       <xs:complexType mixed="true">
                         <xs:sequence>
                           <xs:element name="Fields" minOccurs="0">