diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 19afc18..36a0d31 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -70,6 +70,12 @@ namespace TimberWinR get { return _iisw3clogs; } } + private List _w3clogs = new List(); + + public IEnumerable W3C + { + get { return _w3clogs; } + } private List _stdins = new List(); public IEnumerable Stdins @@ -128,6 +134,8 @@ namespace TimberWinR { if (x.TimberWinR.Inputs.WindowsEvents != null) c._events.AddRange(x.TimberWinR.Inputs.WindowsEvents.ToList()); + if (x.TimberWinR.Inputs.W3CLogs != null) + c._w3clogs.AddRange(x.TimberWinR.Inputs.W3CLogs.ToList()); if (x.TimberWinR.Inputs.IISW3CLogs != null) c._iisw3clogs.AddRange(x.TimberWinR.Inputs.IISW3CLogs.ToList()); if (x.TimberWinR.Inputs.Stdins != null) diff --git a/TimberWinR/Filters/GrokFilter.cs b/TimberWinR/Filters/GrokFilter.cs index 65bacb6..290093e 100644 --- a/TimberWinR/Filters/GrokFilter.cs +++ b/TimberWinR/Filters/GrokFilter.cs @@ -19,7 +19,7 @@ namespace TimberWinR.Parser { get { return fields[i]; } set { fields[i] = value; } - } + } public Fields(JObject json) { diff --git a/TimberWinR/Inputs/W3CInputListener.cs b/TimberWinR/Inputs/W3CInputListener.cs new file mode 100644 index 0000000..66b3531 --- /dev/null +++ b/TimberWinR/Inputs/W3CInputListener.cs @@ -0,0 +1,155 @@ +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Security.AccessControl; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.IO; +using Interop.MSUtil; + +using Newtonsoft.Json.Linq; +using Newtonsoft.Json.Serialization; +using NLog; +using TimberWinR.Parser; +using LogQuery = Interop.MSUtil.LogQueryClassClass; +using W3CLogInputFormat = Interop.MSUtil.COMW3CInputContextClassClass; +using LogRecordSet = Interop.MSUtil.ILogRecordset; + + +namespace TimberWinR.Inputs +{ + public class W3CInputListener : InputListener + { + private readonly int _pollingIntervalInSeconds; + private readonly TimberWinR.Parser.W3CLog _arguments; + private long _receivedMessages; + + public W3CInputListener(TimberWinR.Parser.W3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5) + : base(cancelToken, "Win32-W3CLog") + { + _arguments = arguments; + _receivedMessages = 0; + _pollingIntervalInSeconds = pollingIntervalInSeconds; + foreach (string loc in _arguments.Location.Split(',')) + { + string hive = loc.Trim(); + Task.Factory.StartNew(() => IISW3CWatcher(loc)); + } + } + + public override void Shutdown() + { + base.Shutdown(); + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("iisw3c", + new JObject( + new JProperty("messages", _receivedMessages), + new JProperty("location", _arguments.Location), + new JProperty("codepage", _arguments.CodePage), + new JProperty("separator", _arguments.Separator), + new JProperty("dQuotes", _arguments.DoubleQuotes), + new JProperty("dtLines", _arguments.DtLines) + ))); + return json; + } + + + private void IISW3CWatcher(string location) + { + LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location); + + var oLogQuery = new LogQuery(); + + var iFmt = new W3CLogInputFormat() + { + codepage = _arguments.CodePage, + iCodepage = _arguments.CodePage, + doubleQuotedStrings = _arguments.DoubleQuotes, + detectTypesLines = _arguments.DtLines, + dQuotes = _arguments.DoubleQuotes, + separator = _arguments.Separator + }; + + Dictionary logFileMaxRecords = new Dictionary(); + + // Execute the query + while (!CancelToken.IsCancellationRequested) + { + try + { + oLogQuery = new LogQuery(); + + var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", location); + var rsfiles = oLogQuery.Execute(qfiles, iFmt); + for (; !rsfiles.atEnd(); rsfiles.moveNext()) + { + var record = rsfiles.getRecord(); + string fileName = record.getValue("LogFilename") as string; + if (!logFileMaxRecords.ContainsKey(fileName)) + { + var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}", fileName); + var rcount = oLogQuery.Execute(qcount, iFmt); + var qr = rcount.getRecord(); + var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); + logFileMaxRecords[fileName] = lrn; + } + } + + + foreach (string fileName in logFileMaxRecords.Keys.ToList()) + { + var lastRecordNumber = logFileMaxRecords[fileName]; + var query = string.Format("SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName, lastRecordNumber); + var rs = oLogQuery.Execute(query, iFmt); + var colMap = new Dictionary(); + for (int col = 0; col < rs.getColumnCount(); col++) + { + string colName = rs.getColumnName(col); + colMap[colName] = col; + } + + // Browse the recordset + for (; !rs.atEnd(); rs.moveNext()) + { + var record = rs.getRecord(); + var json = new JObject(); + foreach (var field in colMap.Keys) + { + object v = record.getValue(field); + if (field == "date" || field == "time") + { + DateTime dt = DateTime.Parse(v.ToString()); + json.Add(new JProperty(field, dt)); + } + else + json.Add(new JProperty(field, v)); + } + ProcessJson(json); + _receivedMessages++; + var lrn = (Int64)record.getValueEx("RowNumber"); + logFileMaxRecords[fileName] = lrn; + record = null; + json = null; + } + // Close the recordset + rs.close(); + } + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + } + + System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); + } + + Finished(); + } + } +} diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index 7fb8080..c9f5085 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -150,6 +150,14 @@ namespace TimberWinR output.Connect(elistner); } + foreach (Parser.W3CLog iisw3cConfig in Config.W3C) + { + var elistner = new W3CInputListener(iisw3cConfig, cancelToken); + Listeners.Add(elistner); + foreach (var output in Outputs) + output.Connect(elistner); + } + foreach (Parser.WindowsEvent eventConfig in Config.Events) { var elistner = new WindowsEvtInputListener(eventConfig, cancelToken); diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 6480f60..88a5cc6 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -327,7 +327,42 @@ namespace TimberWinR.Parser } } - + + public class W3CLog : IValidateSchema + { + [JsonProperty(PropertyName = "location")] + public string Location { get; set; } + [JsonProperty(PropertyName = "separator")] + public string Separator { get; set; } + [JsonProperty(PropertyName = "iCodepage")] + public int CodePage { get; set; } + [JsonProperty(PropertyName = "dtLines")] + public int DtLines { get; set; } + [JsonProperty(PropertyName = "dQuotes")] + public bool DoubleQuotes { get; set; } + + + [JsonProperty(PropertyName = "fields")] + public List Fields { get; set; } + + public W3CLog() + { + CodePage = 0; + DtLines = 10; + Fields = new List(); + Separator = "auto"; + + Fields.Add(new Field("LogFilename", "string")); + Fields.Add(new Field("RowNumber", "integer")); + } + + public void Validate() + { + + } + } + + public class IISW3CLog : IValidateSchema { [JsonProperty(PropertyName = "location")] @@ -489,6 +524,9 @@ namespace TimberWinR.Parser [JsonProperty("IISW3CLogs")] public IISW3CLog[] IISW3CLogs { get; set; } + [JsonProperty("W3CLogs")] + public W3CLog[] W3CLogs { get; set; } + [JsonProperty("Stdin")] public Stdin[] Stdins { get; set; } } diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index 16f02c7..f8525a6 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -83,6 +83,7 @@ + @@ -119,6 +120,7 @@ + diff --git a/TimberWinR/mdocs/W3CInput.md b/TimberWinR/mdocs/W3CInput.md new file mode 100644 index 0000000..1161fdf --- /dev/null +++ b/TimberWinR/mdocs/W3CInput.md @@ -0,0 +1,48 @@ +# Input: W3CLogs + +The W3C input format parses IIS log files in the W3C Extended Log File Format, and handles custom fields unlike the IISW3C input. + +IIS web sites logging in the W3C Extended format can be configured to log only a specific subset of the available fields. +Log files in this format begin with some informative headers ("directives"), the most important of which is the "#Fields" directive, describing which fields are logged at which position in a log row. +After the directives, the log entries follow. Each log entry is a space-separated list of field values. + +If the logging configuration of an IIS virtual site is updated, the structure of the fields in the file that is currently logged to might change according to the new configuration. In this case, a new "#Fields" directive is logged describing the new fields structure, and the IISW3C input format keeps track of the structure change and parses the new log entries accordingly. + + + +## Parameters +The following parameters are allowed when configuring IISW3CLogs input. + +| Parameter | Type | Description | Details | Default | +| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | +| *location* | string |Location of log files(s) to monitor | Path to text file(s) including wildcards, may be separated by commas | | +| *iCodepage* | integer |Codepage of the text file. | 0 is the system codepage, -1 is UNICODE. | 0 | +| *dtLines* | integer |Number of lines examined to determine field types at run time. | This parameter specifies the number of initial log lines that the W3C input format examines to determine the data type of the input record fields. If the value is zero, all fields will be assumed to be of the STRING data type. | false | +| *dQuotes* | boolean |Specifies that string values in the log are double-quoted. | Log processors might generate W3C logs whose string values are enclosed in double-quotes. | false | +| *separator* | string |Use the value of the "#Date" directive for the "date" and/or "time" field values when these fields are not logged. | When a log file is configured to not log the "date" and/or "time" fields, specifying "true" for this parameters causes the IISW3C input format to generate "date" and "time" values using the value of the last seen "#Date" directive. | false | + +Example Input: +```json +{ + "TimberWinR": { + "Inputs": { + "W3CLogs": [ + { + "location": "C:\\inetpub\\logs\\LogFiles\\W3SVC1\\*" + } + ] + } + } +} +``` + + +## Fields +After a successful parse of an event, the following fields are added [(if configured to be logged)](http://technet.microsoft.com/en-us/library/cc754702(v=ws.10).aspx) + +| Name | Type | Description | +| ---- |:-----| :-----------------------------------------------------------------------| +|LogFilename| STRING | Full path of the log file containing this entry | +|LogRow | INTEGER | Line in the log file containing this entry | + +Custom fields to follow..