diff --git a/README.md b/README.md
index 2e87af1..4c8324b 100644
--- a/README.md
+++ b/README.md
@@ -28,6 +28,7 @@ The current list of supported filters are:
2. [Mutate](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md)
3. [Date](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md)
4. [Json](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md)
+ 5. [GeoIP](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/GeoIPFilter.md)
## JSON
Since TimberWinR only ships to Redis, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be
diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
index b89e9ae..28b8434 100644
--- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
+++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.2.2.0")]
-[assembly: AssemblyFileVersion("1.2.2.0")]
+[assembly: AssemblyVersion("1.3.0.0")]
+[assembly: AssemblyFileVersion("1.3.0.0")]
diff --git a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
index cd76fd0..1bbaa5b 100644
--- a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
+++ b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
@@ -72,7 +72,9 @@
PreserveNewest
-
+
+ Designer
+
diff --git a/TimberWinR.UnitTests/GeoIPFilterTests.cs b/TimberWinR.UnitTests/GeoIPFilterTests.cs
new file mode 100644
index 0000000..23025e4
--- /dev/null
+++ b/TimberWinR.UnitTests/GeoIPFilterTests.cs
@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TimberWinR.Parser;
+using Newtonsoft.Json.Linq;
+
+namespace TimberWinR.UnitTests
+{
+ [TestFixture]
+ public class GeoIPFilterTests
+ {
+ [Test]
+ public void TestDropConditions()
+ {
+ JObject jsonInputLine1 = new JObject
+ {
+ {"type", "Win32-FileLog"},
+ {"IP", "8.8.8.8"}
+ };
+
+
+ string jsonFilter = @"{
+ ""TimberWinR"":{
+ ""Filters"":[
+ {
+ ""geoip"":{
+ ""type"": ""Win32-FileLog"",
+ ""target"": ""mygeoip"",
+ ""source"": ""IP""
+ }
+ }]
+ }
+ }";
+
+ // Positive Tests
+ Configuration c = Configuration.FromString(jsonFilter);
+ GeoIP jf = c.Filters.First() as GeoIP;
+ Assert.IsTrue(jf.Apply(jsonInputLine1));
+
+ JObject stuff = jsonInputLine1["mygeoip"] as JObject;
+ Assert.IsNotNull(stuff);
+
+ Assert.AreEqual("8.8.8.8", stuff["ip"].ToString());
+ Assert.AreEqual("US", stuff["country_code2"].ToString());
+ Assert.AreEqual("United States", stuff["country_name"].ToString());
+ Assert.AreEqual("CA", stuff["region_name"].ToString());
+ Assert.AreEqual("Mountain View", stuff["city_name"].ToString());
+ Assert.AreEqual("California", stuff["real_region_name"].ToString());
+ Assert.AreEqual(37.386f, (float)stuff["latitude"]);
+ Assert.AreEqual(-122.0838f, (float) stuff["longitude"]);
+ }
+ }
+}
diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
index bfae22b..7cec546 100644
--- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
+++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
@@ -52,6 +52,7 @@
+
diff --git a/TimberWinR/Filters/GeoIPFilter.cs b/TimberWinR/Filters/GeoIPFilter.cs
new file mode 100644
index 0000000..9eee0eb
--- /dev/null
+++ b/TimberWinR/Filters/GeoIPFilter.cs
@@ -0,0 +1,187 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Reflection;
+using System.Text;
+using System.Xml.Linq;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using MaxMind.GeoIP2;
+using MaxMind.Db;
+using MaxMind.GeoIP2.Exceptions;
+using NLog;
+
+namespace TimberWinR.Parser
+{
+ public partial class GeoIP : LogstashFilter
+ {
+ private string DatabaseFileName { get; set; }
+ private DatabaseReader dr;
+ public override JObject ToJson()
+ {
+ JObject json = new JObject(
+ new JProperty("geoip",
+ new JObject(
+ new JProperty("source", Source),
+ new JProperty("condition", Condition),
+ new JProperty("target", Target)
+ )));
+ return json;
+ }
+
+ public GeoIP()
+ {
+ Target = "geoip";
+ DatabaseFileName = Path.Combine(AssemblyDirectory, "GeoLite2City.mmdb");
+ dr = new DatabaseReader(DatabaseFileName);
+ }
+
+ private static string AssemblyDirectory
+ {
+ get
+ {
+ string codeBase = Assembly.GetExecutingAssembly().CodeBase;
+ UriBuilder uri = new UriBuilder(codeBase);
+ string path = Uri.UnescapeDataString(uri.Path);
+ return Path.GetDirectoryName(path);
+ }
+ }
+
+ public override bool Apply(JObject json)
+ {
+ if (!string.IsNullOrEmpty(Type))
+ {
+ JToken json_type = json["type"];
+ if (json_type != null && json_type.ToString() != Type)
+ return true; // Filter does not apply.
+ }
+
+ if (Condition != null)
+ {
+ var expr = EvaluateCondition(json, Condition);
+ if (!expr)
+ return true;
+ }
+
+ var source = json[Source];
+
+ if (source != null && !string.IsNullOrEmpty(source.ToString()))
+ {
+ try
+ {
+ var l = dr.City(source.ToString());
+ if (l != null)
+ {
+ JObject geo_json = new JObject(
+ new JProperty(Target,
+ new JObject(
+ new JProperty("ip", source.ToString()),
+ new JProperty("country_code2", l.Country.IsoCode),
+ new JProperty("country_name", l.Country.Name),
+ new JProperty("continent_code", l.Continent.Code),
+ new JProperty("region_name", l.MostSpecificSubdivision.IsoCode),
+ new JProperty("city_name", l.City.Name),
+ new JProperty("postal_code", l.Postal.Code),
+ new JProperty("latitude", l.Location.Latitude),
+ new JProperty("longitude", l.Location.Longitude),
+ new JProperty("dma_code", l.Location.MetroCode),
+ new JProperty("timezone", l.Location.TimeZone),
+ new JProperty("real_region_name", l.MostSpecificSubdivision.Name),
+ new JProperty("location",
+ new JArray(l.Location.Longitude, l.Location.Latitude)
+ ))));
+
+ json.Merge(geo_json, new JsonMergeSettings
+ {
+ MergeArrayHandling = MergeArrayHandling.Union
+ });
+ }
+ else
+ {
+ json["_geoiperror"] = string.Format("IP Address not found: {0}", source.ToString());
+ }
+ }
+ catch (Exception ex)
+ {
+ json["_geoiperror"] = string.Format("IP Address not found: {0} ({1})", source.ToString(), ex.ToString());
+ return true;
+ }
+ }
+
+ AddFields(json);
+ AddTags(json);
+ RemoveFields(json);
+ RemoveTags(json);
+
+ return true;
+ }
+
+ private void AddFields(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (AddField != null && AddField.Length > 0)
+ {
+ for (int i = 0; i < AddField.Length; i += 2)
+ {
+ string fieldName = ExpandField(AddField[i], json);
+ string fieldValue = ExpandField(AddField[i + 1], json);
+ AddOrModify(json, fieldName, fieldValue);
+ }
+ }
+ }
+
+ private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (RemoveField != null && RemoveField.Length > 0)
+ {
+ for (int i = 0; i < RemoveField.Length; i++)
+ {
+ string fieldName = ExpandField(RemoveField[i], json);
+ RemoveProperties(json, new string[] { fieldName });
+ }
+ }
+ }
+
+ private void AddTags(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (AddTag != null && AddTag.Length > 0)
+ {
+ for (int i = 0; i < AddTag.Length; i++)
+ {
+ string value = ExpandField(AddTag[i], json);
+
+ JToken tags = json["tags"];
+ if (tags == null)
+ json.Add("tags", new JArray(value));
+ else
+ {
+ JArray a = tags as JArray;
+ a.Add(value);
+ }
+ }
+ }
+ }
+
+ private void RemoveTags(Newtonsoft.Json.Linq.JObject json)
+ {
+ if (RemoveTag != null && RemoveTag.Length > 0)
+ {
+ JToken tags = json["tags"];
+ if (tags != null)
+ {
+ List children = tags.Children().ToList();
+ for (int i = 0; i < RemoveTag.Length; i++)
+ {
+ string tagName = ExpandField(RemoveTag[i], json);
+ foreach (JToken token in children)
+ {
+ if (token.ToString() == tagName)
+ token.Remove();
+ }
+ }
+ }
+ }
+ }
+
+ }
+}
diff --git a/TimberWinR/GeoLite2City.mmdb b/TimberWinR/GeoLite2City.mmdb
new file mode 100644
index 0000000..ba4aa7a
Binary files /dev/null and b/TimberWinR/GeoLite2City.mmdb differ
diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs
index d66e33e..6480f60 100644
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -623,6 +623,58 @@ namespace TimberWinR.Parser
}
}
+ public partial class GeoIP : LogstashFilter
+ {
+ public class GeoIPMissingSourceException : Exception
+ {
+ public GeoIPMissingSourceException()
+ : base("GeoIP filter source is required")
+ {
+ }
+ }
+
+ public class GeoIPAddFieldException : Exception
+ {
+ public GeoIPAddFieldException()
+ : base("GeoIP filter add_field requires tuples")
+ {
+ }
+ }
+
+ [JsonProperty("type")]
+ public string Type { get; set; }
+
+ [JsonProperty("condition")]
+ public string Condition { get; set; }
+
+ [JsonProperty("source")]
+ public string Source { get; set; }
+
+ [JsonProperty("target")]
+ public string Target { get; set; }
+
+ [JsonProperty("add_tag")]
+ public string[] AddTag { get; set; }
+
+ [JsonProperty("add_field")]
+ public string[] AddField { get; set; }
+
+ [JsonProperty("remove_field")]
+ public string[] RemoveField { get; set; }
+
+ [JsonProperty("remove_tag")]
+ public string[] RemoveTag { get; set; }
+
+ public override void Validate()
+ {
+ if (string.IsNullOrEmpty(Source))
+ throw new GeoIPMissingSourceException();
+
+ if (AddField != null && AddField.Length % 2 != 0)
+ throw new GeoIPAddFieldException();
+ }
+ }
+
public partial class Json : LogstashFilter
{
public class JsonMissingSourceException : Exception
@@ -691,7 +743,9 @@ namespace TimberWinR.Parser
[JsonProperty("json")]
public Json Json { get; set; }
-
+
+ [JsonProperty("geoip")]
+ public GeoIP GeoIP { get; set; }
}
public class TimberWinR
diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj
index b69a6f1..16f02c7 100644
--- a/TimberWinR/TimberWinR.csproj
+++ b/TimberWinR/TimberWinR.csproj
@@ -39,6 +39,14 @@
False
lib\com-logparser\Interop.MSUtil.dll
+
+ ..\packages\MaxMind.Db.0.2.3.0\lib\net40\MaxMind.Db.dll
+ True
+
+
+ ..\packages\MaxMind.GeoIP2.0.4.0.0\lib\net40\MaxMind.GeoIP2.dll
+ True
+
False
..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll
@@ -71,6 +79,7 @@
+
@@ -104,8 +113,12 @@
Designer
+
+ PreserveNewest
+
+
diff --git a/TimberWinR/mdocs/GeoIPFilter.md b/TimberWinR/mdocs/GeoIPFilter.md
new file mode 100644
index 0000000..d07da73
--- /dev/null
+++ b/TimberWinR/mdocs/GeoIPFilter.md
@@ -0,0 +1,137 @@
+# GeoIP Filter
+The Json filter allows you to parse a single line of Json into its corresponding fields. This is
+particularly useful when parsing log files.
+
+## GeoIP Operations
+The following operations are allowed when mutating a field.
+
+| Operation | Type | Description
+| :---------------|:----------------|:-----------------------------------------------------------------------|
+| *type* | property:string |Type to which this filter applies, if empty, applies to all types.
+| *condition* | property:string |C# expression, if the expression is true, continue, otherwise, ignore
+| *source* | property:string |Required field indicates which field contains the IP address to be parsed
+| *target* | property:string |If suppled, the parsed json will be contained underneath a propery named *target*, default=geoip
+| *add_field* | property:array |If the filter is successful, add an arbitrary field to this event. Field names can be dynamic and include parts of the event using the %{field} syntax. This property must be specified in pairs.
+| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
+| *add_tag* | property:array |If the filter is successful, add an arbitrary tag to this event. Tag names can be dynamic and include parts of the event using the %{field} syntax.
+| *remove_tag* | property:array |If the filter is successful, remove arbitrary tags from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
+
+## Operation Details
+### source
+The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second
+argument. In the below example, the message is spected to be something like this from a fictional sample log:
+
+Given this input configuration:
+
+Lets assume that a newline such as the following is appended to foo.jlog:
+```
+ {"type": "Win32-FileLog", "IP": "8.8.8.8" }
+```
+
+```json
+ "Inputs": {
+ "Logs": [
+ {
+ "location": "C:\\Logs1\\foo.jlog",
+ "recurse": -1
+ }
+ ]
+ },
+ "Filters":[
+ {
+ "geoip":{
+ "type": "Win32-FileLog",
+ "source": "IP"
+ }
+ }]
+ }
+```
+
+In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed
+to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff*
+
+The resulting output would be:
+```
+ {
+ "type": "Win32-FileLog",
+ "IP": "8.8.8.8",
+ "mygeoip": {
+ "ip": "8.8.8.8",
+ "country_code2": "US",
+ "country_name": "United States",
+ "continent_code": "NA",
+ "region_name": "CA",
+ "city_name": "Mountain View",
+ "postal_code": null,
+ "latitude": 37.386,
+ "longitude": -122.0838,
+ "dma_code": 807,
+ "timezone": "America/Los_Angeles",
+ "real_region_name": "California",
+ "location": [
+ -122.0838,
+ 37.386
+ ]
+ }
+```
+
+### add_field ["fieldName", "fieldValue", ...]
+The fields must be in pairs with fieldName first and value second.
+```json
+ "Filters": [
+ {
+ "json": {
+ "add_field": [
+ "ComputerName", "Host",
+ "Username", "%{SID}"
+ ]
+ }
+ }
+ ]
+```
+
+### remove_field ["tag1", "tag2", ...]
+Remove the fields. More than one field can be specified at a time.
+```json
+ "Filters": [
+ {
+ "json": {
+ "remove_tag": [
+ "static_tag1",
+ "Computer_%{Host}"
+ ]
+ }
+ }
+ ]
+```
+
+
+### add_tag ["tag1", "tag2", ...]
+Adds the tag(s) to the tag array.
+```json
+ "Filters": [
+ {
+ "json": {
+ "add_tag": [
+ "foo_%{Host}",
+ "static_tag1"
+ ]
+ }
+ }
+ ]
+```
+
+### remove_tag ["tag1", "tag2", ...]
+Remove the tag(s) to the tag array. More than one tag can be specified at a time.
+```json
+ "Filters": [
+ {
+ "json": {
+ "remove_tag": [
+ "static_tag1",
+ "Username"
+ ]
+ }
+ }
+ ]
+```
diff --git a/TimberWinR/packages.config b/TimberWinR/packages.config
index a7ff76b..ef85fad 100644
--- a/TimberWinR/packages.config
+++ b/TimberWinR/packages.config
@@ -1,6 +1,8 @@
+
+
diff --git a/TimberWix/Product.wxs b/TimberWix/Product.wxs
index da98305..04e183c 100644
--- a/TimberWix/Product.wxs
+++ b/TimberWix/Product.wxs
@@ -40,7 +40,7 @@
-
+
@@ -51,6 +51,9 @@
+
+
+
diff --git a/TimberWix/TimberWinR.Wix.wixproj b/TimberWix/TimberWinR.Wix.wixproj
index aab7f92..872b660 100644
--- a/TimberWix/TimberWinR.Wix.wixproj
+++ b/TimberWix/TimberWinR.Wix.wixproj
@@ -43,6 +43,14 @@
Binaries;Content;Satellites
INSTALLFOLDER
+
+ TimberWinR
+ {4ef96a08-21db-4178-be44-70dae594632c}
+ True
+ True
+ Binaries;Content;Satellites
+ INSTALLFOLDER
+