TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added GeoIP Filter

Browse Source
This commit is contained in:
Eric Fontana
2014-09-22 11:19:11 -04:00
parent e42d99147e
commit 37f09601dd
13 changed files with 471 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -28,6 +28,7 @@ The current list of supported filters are:
2. [Mutate](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md)
3. [Date](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md)
4. [Json](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md)
5. [GeoIP](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GeoIPFilter.md)
## JSON
Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.2.2.0")]
[assembly: AssemblyFileVersion("1.2.2.0")]
[assembly: AssemblyVersion("1.3.0.0")]
[assembly: AssemblyFileVersion("1.3.0.0")]

4
TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
View File

@@ -72,7 +72,9 @@
<None Include="loopback.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
<None Include="packages.config" />
<None Include="packages.config">
<SubType>Designer</SubType>
</None>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

58
TimberWinR.UnitTests/GeoIPFilterTests.cs Normal file
View File

@@ -0,0 +1,58 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Newtonsoft.Json;
using NUnit.Framework;
using TimberWinR.Parser;
using Newtonsoft.Json.Linq;
namespace TimberWinR.UnitTests
{
[TestFixture]
public class GeoIPFilterTests
{
[Test]
public void TestDropConditions()
{
JObject jsonInputLine1 = new JObject
{
{"type", "Win32-FileLog"},
{"IP", "8.8.8.8"}
};
string jsonFilter = @"{
""TimberWinR"":{
""Filters"":[
{
""geoip"":{
""type"": ""Win32-FileLog"",
""target"": ""mygeoip"",
""source"": ""IP""
}
}]
}
}";
// Positive Tests
Configuration c = Configuration.FromString(jsonFilter);
GeoIP jf = c.Filters.First() as GeoIP;
Assert.IsTrue(jf.Apply(jsonInputLine1));
JObject stuff = jsonInputLine1["mygeoip"] as JObject;
Assert.IsNotNull(stuff);
Assert.AreEqual("8.8.8.8", stuff["ip"].ToString());
Assert.AreEqual("US", stuff["country_code2"].ToString());
Assert.AreEqual("United States", stuff["country_name"].ToString());
Assert.AreEqual("CA", stuff["region_name"].ToString());
Assert.AreEqual("Mountain View", stuff["city_name"].ToString());
Assert.AreEqual("California", stuff["real_region_name"].ToString());
Assert.AreEqual(37.386f, (float)stuff["latitude"]);
Assert.AreEqual(-122.0838f, (float) stuff["longitude"]);
}
}
}

1
TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
View File

@@ -52,6 +52,7 @@
<ItemGroup>
<Compile Include="Configuration.cs" />
<Compile Include="DateFilterTests.cs" />
<Compile Include="GeoIPFilterTests.cs" />
<Compile Include="JsonFilterTests.cs" />
<Compile Include="GrokFilterTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />

187
TimberWinR/Filters/GeoIPFilter.cs Normal file
View File

@@ -0,0 +1,187 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Reflection;
using System.Text;
using System.Xml.Linq;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using MaxMind.GeoIP2;
using MaxMind.Db;
using MaxMind.GeoIP2.Exceptions;
using NLog;
namespace TimberWinR.Parser
{
public partial class GeoIP : LogstashFilter
{
private string DatabaseFileName { get; set; }
private DatabaseReader dr;
public override JObject ToJson()
{
JObject json = new JObject(
new JProperty("geoip",
new JObject(
new JProperty("source", Source),
new JProperty("condition", Condition),
new JProperty("target", Target)
)));
return json;
}
public GeoIP()
{
Target = "geoip";
DatabaseFileName = Path.Combine(AssemblyDirectory, "GeoLite2City.mmdb");
dr = new DatabaseReader(DatabaseFileName);
}
private static string AssemblyDirectory
{
get
{
string codeBase = Assembly.GetExecutingAssembly().CodeBase;
UriBuilder uri = new UriBuilder(codeBase);
string path = Uri.UnescapeDataString(uri.Path);
return Path.GetDirectoryName(path);
}
}
public override bool Apply(JObject json)
{
if (!string.IsNullOrEmpty(Type))
{
JToken json_type = json["type"];
if (json_type != null && json_type.ToString() != Type)
return true; // Filter does not apply.
}
if (Condition != null)
{
var expr = EvaluateCondition(json, Condition);
if (!expr)
return true;
}
var source = json[Source];
if (source != null && !string.IsNullOrEmpty(source.ToString()))
{
try
{
var l = dr.City(source.ToString());
if (l != null)
{
JObject geo_json = new JObject(
new JProperty(Target,
new JObject(
new JProperty("ip", source.ToString()),
new JProperty("country_code2", l.Country.IsoCode),
new JProperty("country_name", l.Country.Name),
new JProperty("continent_code", l.Continent.Code),
new JProperty("region_name", l.MostSpecificSubdivision.IsoCode),
new JProperty("city_name", l.City.Name),
new JProperty("postal_code", l.Postal.Code),
new JProperty("latitude", l.Location.Latitude),
new JProperty("longitude", l.Location.Longitude),
new JProperty("dma_code", l.Location.MetroCode),
new JProperty("timezone", l.Location.TimeZone),
new JProperty("real_region_name", l.MostSpecificSubdivision.Name),
new JProperty("location",
new JArray(l.Location.Longitude, l.Location.Latitude)
))));
json.Merge(geo_json, new JsonMergeSettings
{
MergeArrayHandling = MergeArrayHandling.Union
});
}
else
{
json["_geoiperror"] = string.Format("IP Address not found: {0}", source.ToString());
}
}
catch (Exception ex)
{
json["_geoiperror"] = string.Format("IP Address not found: {0} ({1})", source.ToString(), ex.ToString());
return true;
}
}
AddFields(json);
AddTags(json);
RemoveFields(json);
RemoveTags(json);
return true;
}
private void AddFields(Newtonsoft.Json.Linq.JObject json)
{
if (AddField != null && AddField.Length > 0)
{
for (int i = 0; i < AddField.Length; i += 2)
{
string fieldName = ExpandField(AddField[i], json);
string fieldValue = ExpandField(AddField[i + 1], json);
AddOrModify(json, fieldName, fieldValue);
}
}
}
private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
{
if (RemoveField != null && RemoveField.Length > 0)
{
for (int i = 0; i < RemoveField.Length; i++)
{
string fieldName = ExpandField(RemoveField[i], json);
RemoveProperties(json, new string[] { fieldName });
}
}
}
private void AddTags(Newtonsoft.Json.Linq.JObject json)
{
if (AddTag != null && AddTag.Length > 0)
{
for (int i = 0; i < AddTag.Length; i++)
{
string value = ExpandField(AddTag[i], json);
JToken tags = json["tags"];
if (tags == null)
json.Add("tags", new JArray(value));
else
{
JArray a = tags as JArray;
a.Add(value);
}
}
}
}
private void RemoveTags(Newtonsoft.Json.Linq.JObject json)
{
if (RemoveTag != null && RemoveTag.Length > 0)
{
JToken tags = json["tags"];
if (tags != null)
{
List<JToken> children = tags.Children().ToList();
for (int i = 0; i < RemoveTag.Length; i++)
{
string tagName = ExpandField(RemoveTag[i], json);
foreach (JToken token in children)
{
if (token.ToString() == tagName)
token.Remove();
}
}
}
}
}
}
}

BIN
TimberWinR/GeoLite2City.mmdb Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 31 MiB

54
TimberWinR/Parser.cs
View File

@@ -623,6 +623,58 @@ namespace TimberWinR.Parser
}
}
public partial class GeoIP : LogstashFilter
{
public class GeoIPMissingSourceException : Exception
{
public GeoIPMissingSourceException()
: base("GeoIP filter source is required")
{
}
}
public class GeoIPAddFieldException : Exception
{
public GeoIPAddFieldException()
: base("GeoIP filter add_field requires tuples")
{
}
}
[JsonProperty("type")]
public string Type { get; set; }
[JsonProperty("condition")]
public string Condition { get; set; }
[JsonProperty("source")]
public string Source { get; set; }
[JsonProperty("target")]
public string Target { get; set; }
[JsonProperty("add_tag")]
public string[] AddTag { get; set; }
[JsonProperty("add_field")]
public string[] AddField { get; set; }
[JsonProperty("remove_field")]
public string[] RemoveField { get; set; }
[JsonProperty("remove_tag")]
public string[] RemoveTag { get; set; }
public override void Validate()
{
if (string.IsNullOrEmpty(Source))
throw new GeoIPMissingSourceException();
if (AddField != null && AddField.Length % 2 != 0)
throw new GeoIPAddFieldException();
}
}
public partial class Json : LogstashFilter
{
public class JsonMissingSourceException : Exception
@@ -692,6 +744,8 @@ namespace TimberWinR.Parser
[JsonProperty("json")]
public Json Json { get; set; }
[JsonProperty("geoip")]
public GeoIP GeoIP { get; set; }
}
public class TimberWinR

13
TimberWinR/TimberWinR.csproj
View File

@@ -39,6 +39,14 @@
<EmbedInteropTypes>False</EmbedInteropTypes>
<HintPath>lib\com-logparser\Interop.MSUtil.dll</HintPath>
</Reference>
<Reference Include="MaxMind.Db">
<HintPath>..\packages\MaxMind.Db.0.2.3.0\lib\net40\MaxMind.Db.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="MaxMind.GeoIP2">
<HintPath>..\packages\MaxMind.GeoIP2.0.4.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
<SpecificVersion>False</SpecificVersion>
<HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
@@ -71,6 +79,7 @@
<Compile Include="Filters\DateFilter.cs" />
<Compile Include="Filters\FilterBase.cs" />
<Compile Include="Filters\GrokFilter.cs" />
<Compile Include="Filters\GeoIPFilter.cs" />
<Compile Include="Filters\JsonFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
@@ -104,8 +113,12 @@
<None Include="configSchema.xsd">
<SubType>Designer</SubType>
</None>
<Content Include="GeoLite2City.mmdb">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<None Include="mdocs\DateFilter.md" />
<None Include="mdocs\Filters.md" />
<None Include="mdocs\GeoIPFilter.md" />
<None Include="mdocs\JsonFilter.md" />
<None Include="mdocs\GrokFilter.md" />
<None Include="mdocs\ElasticsearchOutput.md" />

137
TimberWinR/mdocs/GeoIPFilter.md Normal file
View File

@@ -0,0 +1,137 @@
# GeoIP Filter
The Json filter allows you to parse a single line of Json into its corresponding fields. This is
particularly useful when parsing log files.
## GeoIP Operations
The following operations are allowed when mutating a field.
| Operation | Type | Description
| :---------------|:----------------|:-----------------------------------------------------------------------|
| *type* | property:string |Type to which this filter applies, if empty, applies to all types.
| *condition* | property:string |C# expression, if the expression is true, continue, otherwise, ignore
| *source* | property:string |Required field indicates which field contains the IP address to be parsed
| *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target*, default=geoip
| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
| *add_tag* | property:array |If the filter is successful, add an arbitrary tag to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax.
| *remove_tag* | property:array |If the filter is successful, remove arbitrary tags from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
## Operation Details
### source
The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second
argument. In the below example, the message is spected to be something like this from a fictional sample log:
Given this input configuration:
Lets assume that a newline such as the following is appended to foo.jlog:
```
{"type": "Win32-FileLog", "IP": "8.8.8.8" }
```
```json
"Inputs": {
"Logs": [
{
"location": "C:\\Logs1\\foo.jlog",
"recurse": -1
}
]
},
"Filters":[
{
"geoip":{
"type": "Win32-FileLog",
"source": "IP"
}
}]
}
```
In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed
to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff*
The resulting output would be:
```
{
"type": "Win32-FileLog",
"IP": "8.8.8.8",
"mygeoip": {
"ip": "8.8.8.8",
"country_code2": "US",
"country_name": "United States",
"continent_code": "NA",
"region_name": "CA",
"city_name": "Mountain View",
"postal_code": null,
"latitude": 37.386,
"longitude": -122.0838,
"dma_code": 807,
"timezone": "America/Los_Angeles",
"real_region_name": "California",
"location": [
-122.0838,
37.386
]
}
```
### add_field ["fieldName", "fieldValue", ...]
The fields must be in pairs with fieldName first and value second.
```json
"Filters": [
{
"json": {
"add_field": [
"ComputerName", "Host",
"Username", "%{SID}"
]
}
}
]
```
### remove_field ["tag1", "tag2", ...]
Remove the fields. More than one field can be specified at a time.
```json
"Filters": [
{
"json": {
"remove_tag": [
"static_tag1",
"Computer_%{Host}"
]
}
}
]
```
### add_tag ["tag1", "tag2", ...]
Adds the tag(s) to the tag array.
```json
"Filters": [
{
"json": {
"add_tag": [
"foo_%{Host}",
"static_tag1"
]
}
}
]
```
### remove_tag ["tag1", "tag2", ...]
Remove the tag(s) to the tag array. More than one tag can be specified at a time.
```json
"Filters": [
{
"json": {
"remove_tag": [
"static_tag1",
"Username"
]
}
}
]
```

2
TimberWinR/packages.config
View File

@@ -1,6 +1,8 @@
<?xml version="1.0" encoding="utf-8"?>
<packages>
<package id="csredis" version="1.4.7.1" targetFramework="net40" />
<package id="MaxMind.Db" version="0.2.3.0" targetFramework="net40" />
<package id="MaxMind.GeoIP2" version="0.4.0.0" targetFramework="net40" />
<package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
<package id="NLog" version="3.1.0.0" targetFramework="net40" />
<package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />

5
TimberWix/Product.wxs
View File

@@ -40,7 +40,7 @@
<Fragment>
<ComponentGroup Id="ProductComponents" Directory="INSTALLFOLDER">
<Component Id="ProductComponent" Guid="1BDEC5F3-5E9F-4E2E-8B1B-30E7968A99E1">
<File Id="TimberWinR.ServiceHost.exe" Name="$(var.TimberWinR.ServiceHost.TargetFileName)" Source="$(var.TimberWinR.ServiceHost.TargetPath)" />
<File Id="TimberWinR.ServiceHost.exe" Name="$(var.TimberWinR.ServiceHost.TargetFileName)" Source="$(var.TimberWinR.ServiceHost.TargetPath)" />
<File Id="TimberWinR.ServiceHost.exe.config" Source="$(var.TimberWinR.ServiceHost.TargetDir)\TimberWinR.ServiceHost.exe.config" />
<File Id="Interop.MSUtil.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\Interop.MSUtil.dll" />
<File Id="csredis.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\csredis.dll" />
@@ -51,6 +51,9 @@
<File Id="RapidRegex.Core.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\RapidRegex.Core.dll" />
<File Id="TimberWinR.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\TimberWinR.dll" />
<File Id="Topshelf.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\Topshelf.dll" />
<File Id="MaxMind.Db.dll" Source="$(var.TimberWinR.TargetDir)\MaxMind.db.dll" />
<File Id="MaxMind.GeoIP2.dll" Source="$(var.TimberWinR.TargetDir)\MaxMind.GeoIP2.dll" />
<File Id="GeoLite2City.mmdb" Source="$(var.TimberWinR.TargetDir)\GeoLite2City.mmdb" />
</Component>
</ComponentGroup>

8
TimberWix/TimberWinR.Wix.wixproj
View File

@@ -43,6 +43,14 @@
<RefProjectOutputGroups>Binaries;Content;Satellites</RefProjectOutputGroups>
<RefTargetDir>INSTALLFOLDER</RefTargetDir>
</ProjectReference>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">
<Name>TimberWinR</Name>
<Project>{4ef96a08-21db-4178-be44-70dae594632c}</Project>
<Private>True</Private>
<DoNotHarvest>True</DoNotHarvest>
<RefProjectOutputGroups>Binaries;Content;Satellites</RefProjectOutputGroups>
<RefTargetDir>INSTALLFOLDER</RefTargetDir>
</ProjectReference>
</ItemGroup>
<ItemGroup>
<WixExtension Include="WixUIExtension">