TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Last changes

Browse Source
This commit is contained in:
Eric Fontana
2014-07-30 10:19:19 -04:00
parent 783c549e53
commit 19e07770d8
5 changed files with 118 additions and 38 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
TimberWinR.ServiceHost/config.json
View File

@@ -72,7 +72,7 @@
"match": [
"Text",
"%{SYSLOGLINE}"
],
],
"add_tag": [
"rn_%{RecordNumber}",
"bar"
@@ -84,12 +84,10 @@
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
],
"add_field": [
"UtcTimestamp"
],
"target": "UtcTimestamp",
"convertToUTC": true
}
},

31
TimberWinR/Filters/DateFilter.cs
View File

@@ -20,34 +20,12 @@ namespace TimberWinR.Parser
return false;
if (Matches(json))
{
ApplyFilter(json);
{
AddFields(json);
}
return true;
}
private void ApplyFilter(JObject json)
{
string text = json.ToString();
if (!string.IsNullOrEmpty(text))
{
DateTime ts;
if (Patterns == null || Patterns.Length == 0)
{
if (DateTime.TryParse(text, out ts))
AddOrModify(json, ts);
}
else
{
if (DateTime.TryParseExact(text, Patterns.ToArray(), CultureInfo.InvariantCulture,
DateTimeStyles.None, out ts))
AddOrModify(json, ts);
}
}
}
}
// copy_field "field1" -> "field2"
private void AddFields(Newtonsoft.Json.Linq.JObject json)
@@ -80,13 +58,13 @@ namespace TimberWinR.Parser
{
DateTime ts;
var exprArray = Match.Skip(1).ToArray();
var resolver = new RegexGrokResolver();
var resolver = new RegexGrokResolver();
for (int i=0; i<exprArray.Length; i++)
{
var pattern = resolver.ResolveToRegex(exprArray[i]);
exprArray[i] = pattern;
}
if (DateTime.TryParseExact(text, exprArray, ci,DateTimeStyles.None, out ts))
if (DateTime.TryParseExact(text, exprArray, ci, DateTimeStyles.None, out ts))
AddOrModify(json, ts);
}
return true; // Empty field is no match
@@ -94,7 +72,6 @@ namespace TimberWinR.Parser
return false; // Not specified is failure
}
private void AddOrModify(JObject json, DateTime ts)
{
if (ConvertToUTC)

7
TimberWinR/Parser.cs
View File

@@ -153,7 +153,7 @@ namespace TimberWinR.Parser
public string To { get; set; }
[JsonProperty(PropertyName = "type")]
public string FieldType { get; set; }
public Type DataType
{
@@ -505,9 +505,6 @@ namespace TimberWinR.Parser
[JsonProperty("convertToUTC")]
public bool ConvertToUTC { get; set; }
[JsonProperty("pattern")]
public string[] Patterns { get; set; }
[JsonProperty("add_field")]
public string[] AddField { get; set; }
@@ -522,7 +519,7 @@ namespace TimberWinR.Parser
public DateFilter()
{
Target = "timestamp";
Target = "@timestamp";
Locale = "en-US";
}
}

1
TimberWinR/TimberWinR.csproj
View File

@@ -101,6 +101,7 @@
<None Include="mdocs\DateFilter.md" />
<None Include="mdocs\Filters.md" />
<None Include="mdocs\GrokFilter.md" />
<None Include="mdocs\RedisOutput.md" />
<None Include="mdocs\TcpInput.md" />
<None Include="mdocs\MutateFilter.md" />
<None Include="mdocs\Logs.md" />

107
TimberWinR/mdocs/DateFilter.md
View File

@@ -1 +1,108 @@
# Date Filter
The date filter is used for parsing dates from fields, and then using that date or timestamp as the logstash timestamp for the event.
For example, syslog events usually have timestamps like this:
```
"Apr 17 09:32:01"
```
You would use the date format "MMM dd HH:mm:ss" to parse this.
The date filter is especially important for sorting events and for backfilling old data. If you don't
get the date correct in your event, then searching for them later will likely sort out of order.
In the absence of this filter, TimberWinR will choose a timestamp based on the first time it sees
the event (at input time), if the timestamp is not already set in the event. For example, with
file input, the timestamp is set to the time of each read.
## Date Parameters
The following parameters and operations are allowed when using the Date filter.
| Operation | Type | Description | Default
| :---------------|:----------------|:-----------------------------------------------------------------------|
| *add_field* | array |If the filter is successful, add an arbitrary field to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax. | |
| *condition* | string |C# expression | |
| *convertToUTC* | boolean |Converts time to UTC | false |
| *match* | [string] |Required field and pattern must match before any subsequent date operations are executed. | |
| *locale* | string | Specify a locale to be used for date parsing | en-US |
| *target* | string | Store the matching timestamp into the given target field. If not provided, default to updating the @timestamp field of the event. | @timestamp |
## Parameter Details
### match
The date formats allowed are anything allowed by [C# DateTime Format](http://msdn.microsoft.com/en-us/library/8kb3ddd4(v=vs.110).aspx). You can see the docs for this format here:
Given this configuration
```json
"Filters": [
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
],
"add_field": [
"UtcTimestamp"
],
"convertToUTC": true
}
}
]
```
### condition "C# expression"
If present, the condition must evaluate to true in order for the remaining operations to be performed. If there is no condition specified
then the operation(s) will be executed in order.
```json
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-EventLog\""
"add_field": [
"ComputerName", "%{Host}"
]
}
}
]
```
The above example will add a field ComputerName set to the value of Host only for Win32-EventLog types.
### add_field ["fieldName", "fieldValue", ...]
The fields must be in pairs with fieldName first and value second.
```json
"Filters": [
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
],
"add_field": [
"UtcTimestamp"
]
}
}
]
```
### convertToUTC "true|false"
If true and the filter matches, the time parsed will be converted to UTC
```json
"Filters": [
{
"date": {
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
],
"add_field": [
"UtcTimestamp"
],
"convertToUTC": true
}
}
]
```