TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added W3C input to handle custom fields

Browse Source
This commit is contained in:
Eric Fontana
2014-09-22 13:40:41 -04:00
parent 69c142c63b
commit 21c4f8e3b8
7 changed files with 261 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
TimberWinR/Configuration.cs
View File

@@ -70,6 +70,12 @@ namespace TimberWinR
get { return _iisw3clogs; }
}
private List<W3CLog> _w3clogs = new List<W3CLog>();
public IEnumerable<W3CLog> W3C
{
get { return _w3clogs; }
}
private List<Stdin> _stdins = new List<Stdin>();
public IEnumerable<Stdin> Stdins
@@ -128,6 +134,8 @@ namespace TimberWinR
{
if (x.TimberWinR.Inputs.WindowsEvents != null)
c._events.AddRange(x.TimberWinR.Inputs.WindowsEvents.ToList());
if (x.TimberWinR.Inputs.W3CLogs != null)
c._w3clogs.AddRange(x.TimberWinR.Inputs.W3CLogs.ToList());
if (x.TimberWinR.Inputs.IISW3CLogs != null)
c._iisw3clogs.AddRange(x.TimberWinR.Inputs.IISW3CLogs.ToList());
if (x.TimberWinR.Inputs.Stdins != null)

2
TimberWinR/Filters/GrokFilter.cs
View File

@@ -19,7 +19,7 @@ namespace TimberWinR.Parser
{
get { return fields[i]; }
set { fields[i] = value; }
}
}
public Fields(JObject json)
{

155
TimberWinR/Inputs/W3CInputListener.cs Normal file
View File

@@ -0,0 +1,155 @@
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Linq;
using System.Security.AccessControl;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using NLog;
using TimberWinR.Parser;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using W3CLogInputFormat = Interop.MSUtil.COMW3CInputContextClassClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
namespace TimberWinR.Inputs
{
public class W3CInputListener : InputListener
{
private readonly int _pollingIntervalInSeconds;
private readonly TimberWinR.Parser.W3CLog _arguments;
private long _receivedMessages;
public W3CInputListener(TimberWinR.Parser.W3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
: base(cancelToken, "Win32-W3CLog")
{
_arguments = arguments;
_receivedMessages = 0;
_pollingIntervalInSeconds = pollingIntervalInSeconds;
foreach (string loc in _arguments.Location.Split(','))
{
string hive = loc.Trim();
Task.Factory.StartNew(() => IISW3CWatcher(loc));
}
}
public override void Shutdown()
{
base.Shutdown();
}
public override JObject ToJson()
{
JObject json = new JObject(
new JProperty("iisw3c",
new JObject(
new JProperty("messages", _receivedMessages),
new JProperty("location", _arguments.Location),
new JProperty("codepage", _arguments.CodePage),
new JProperty("separator", _arguments.Separator),
new JProperty("dQuotes", _arguments.DoubleQuotes),
new JProperty("dtLines", _arguments.DtLines)
)));
return json;
}
private void IISW3CWatcher(string location)
{
LogManager.GetCurrentClassLogger().Info("IISW3Listener Ready For {0}", location);
var oLogQuery = new LogQuery();
var iFmt = new W3CLogInputFormat()
{
codepage = _arguments.CodePage,
iCodepage = _arguments.CodePage,
doubleQuotedStrings = _arguments.DoubleQuotes,
detectTypesLines = _arguments.DtLines,
dQuotes = _arguments.DoubleQuotes,
separator = _arguments.Separator
};
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
// Execute the query
while (!CancelToken.IsCancellationRequested)
{
try
{
oLogQuery = new LogQuery();
var qfiles = string.Format("SELECT Distinct [LogFilename] FROM {0}", location);
var rsfiles = oLogQuery.Execute(qfiles, iFmt);
for (; !rsfiles.atEnd(); rsfiles.moveNext())
{
var record = rsfiles.getRecord();
string fileName = record.getValue("LogFilename") as string;
if (!logFileMaxRecords.ContainsKey(fileName))
{
var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}", fileName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
logFileMaxRecords[fileName] = lrn;
}
}
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName, lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
var colMap = new Dictionary<string, int>();
for (int col = 0; col < rs.getColumnCount(); col++)
{
string colName = rs.getColumnName(col);
colMap[colName] = col;
}
// Browse the recordset
for (; !rs.atEnd(); rs.moveNext())
{
var record = rs.getRecord();
var json = new JObject();
foreach (var field in colMap.Keys)
{
object v = record.getValue(field);
if (field == "date" || field == "time")
{
DateTime dt = DateTime.Parse(v.ToString());
json.Add(new JProperty(field, dt));
}
else
json.Add(new JProperty(field, v));
}
ProcessJson(json);
_receivedMessages++;
var lrn = (Int64)record.getValueEx("RowNumber");
logFileMaxRecords[fileName] = lrn;
record = null;
json = null;
}
// Close the recordset
rs.close();
}
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
Finished();
}
}
}

8
TimberWinR/Manager.cs
View File

@@ -150,6 +150,14 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (Parser.W3CLog iisw3cConfig in Config.W3C)
{
var elistner = new W3CInputListener(iisw3cConfig, cancelToken);
Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
foreach (Parser.WindowsEvent eventConfig in Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);

40
TimberWinR/Parser.cs
View File

@@ -327,7 +327,42 @@ namespace TimberWinR.Parser
}
}
public class W3CLog : IValidateSchema
{
[JsonProperty(PropertyName = "location")]
public string Location { get; set; }
[JsonProperty(PropertyName = "separator")]
public string Separator { get; set; }
[JsonProperty(PropertyName = "iCodepage")]
public int CodePage { get; set; }
[JsonProperty(PropertyName = "dtLines")]
public int DtLines { get; set; }
[JsonProperty(PropertyName = "dQuotes")]
public bool DoubleQuotes { get; set; }
[JsonProperty(PropertyName = "fields")]
public List<Field> Fields { get; set; }
public W3CLog()
{
CodePage = 0;
DtLines = 10;
Fields = new List<Field>();
Separator = "auto";
Fields.Add(new Field("LogFilename", "string"));
Fields.Add(new Field("RowNumber", "integer"));
}
public void Validate()
{
}
}
public class IISW3CLog : IValidateSchema
{
[JsonProperty(PropertyName = "location")]
@@ -489,6 +524,9 @@ namespace TimberWinR.Parser
[JsonProperty("IISW3CLogs")]
public IISW3CLog[] IISW3CLogs { get; set; }
[JsonProperty("W3CLogs")]
public W3CLog[] W3CLogs { get; set; }
[JsonProperty("Stdin")]
public Stdin[] Stdins { get; set; }
}

2
TimberWinR/TimberWinR.csproj
View File

@@ -83,6 +83,7 @@
<Compile Include="Filters\JsonFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\W3CInputListener.cs" />
<Compile Include="Inputs\IISW3CInputListener.cs" />
<Compile Include="Inputs\InputBase.cs" />
<Compile Include="Inputs\InputListener.cs" />
@@ -119,6 +120,7 @@
<None Include="mdocs\DateFilter.md" />
<None Include="mdocs\Filters.md" />
<None Include="mdocs\GeoIPFilter.md" />
<None Include="mdocs\W3CInput.md" />
<None Include="mdocs\JsonFilter.md" />
<None Include="mdocs\GrokFilter.md" />
<None Include="mdocs\ElasticsearchOutput.md" />

48
TimberWinR/mdocs/W3CInput.md Normal file
View File

@@ -0,0 +1,48 @@
# Input: W3CLogs
The W3C input format parses IIS log files in the W3C Extended Log File Format, and handles custom fields unlike the IISW3C input.
IIS web sites logging in the W3C Extended format can be configured to log only a specific subset of the available fields.
Log files in this format begin with some informative headers ("directives"), the most important of which is the "#Fields" directive, describing which fields are logged at which position in a log row.
After the directives, the log entries follow. Each log entry is a space-separated list of field values.
If the logging configuration of an IIS virtual site is updated, the structure of the fields in the file that is currently logged to might change according to the new configuration. In this case, a new "#Fields" directive is logged describing the new fields structure, and the IISW3C input format keeps track of the structure change and parses the new log entries accordingly.
## Parameters
The following parameters are allowed when configuring IISW3CLogs input.
| Parameter | Type | Description | Details | Default |
| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- |
| *location* | string |Location of log files(s) to monitor | Path to text file(s) including wildcards, may be separated by commas | |
| *iCodepage* | integer |Codepage of the text file. | 0 is the system codepage, -1 is UNICODE. | 0 |
| *dtLines* | integer |Number of lines examined to determine field types at run time. | This parameter specifies the number of initial log lines that the W3C input format examines to determine the data type of the input record fields. If the value is zero, all fields will be assumed to be of the STRING data type. | false |
| *dQuotes* | boolean |Specifies that string values in the log are double-quoted. | Log processors might generate W3C logs whose string values are enclosed in double-quotes. | false |
| *separator* | string |Use the value of the "#Date" directive for the "date" and/or "time" field values when these fields are not logged. | When a log file is configured to not log the "date" and/or "time" fields, specifying "true" for this parameters causes the IISW3C input format to generate "date" and "time" values using the value of the last seen "#Date" directive. | false |
Example Input:
```json
{
"TimberWinR": {
"Inputs": {
"W3CLogs": [
{
"location": "C:\\inetpub\\logs\\LogFiles\\W3SVC1\\*"
}
]
}
}
}
```
## Fields
After a successful parse of an event, the following fields are added [(if configured to be logged)](http://technet.microsoft.com/en-us/library/cc754702(v=ws.10).aspx)
| Name | Type | Description |
| ---- |:-----| :-----------------------------------------------------------------------|
|LogFilename| STRING | Full path of the log file containing this entry |
|LogRow | INTEGER | Line in the log file containing this entry |
Custom fields to follow..