From b35e195b49d114d1743d68d424cb638a85742cd2 Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Tue, 18 Nov 2014 13:31:33 -0500 Subject: [PATCH] Fixed memory leak with conditions. --- .../Properties/AssemblyInfo.cs | 4 +- TimberWinR/Inputs/TcpInputListener.cs | 2 + TimberWinR/Inputs/UdpInputListener.cs | 6 +- TimberWinR/Parser.cs | 198 +++++++----------- TimberWinR/TimberWinR.csproj | 3 + TimberWinR/packages.config | 1 + chocolateyUninstall.ps1.template | 8 +- 7 files changed, 96 insertions(+), 126 deletions(-) diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 669faa6..d5c99bf 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.3.10.0")] -[assembly: AssemblyFileVersion("1.3.10.0")] +[assembly: AssemblyVersion("1.3.11.0")] +[assembly: AssemblyFileVersion("1.3.11.0")] diff --git a/TimberWinR/Inputs/TcpInputListener.cs b/TimberWinR/Inputs/TcpInputListener.cs index b4d9870..d09c18d 100644 --- a/TimberWinR/Inputs/TcpInputListener.cs +++ b/TimberWinR/Inputs/TcpInputListener.cs @@ -37,6 +37,7 @@ namespace TimberWinR.Inputs LogManager.GetCurrentClassLogger().Info("Tcp Input(v4/v6) on Port {0} Ready", _port); + _receivedMessages = 0; _tcpListenerV6 = new System.Net.Sockets.TcpListener(IPAddress.IPv6Any, port); _tcpListenerV4 = new System.Net.Sockets.TcpListener(IPAddress.Any, port); @@ -104,6 +105,7 @@ namespace TimberWinR.Inputs if (CancelToken.IsCancellationRequested) break; JObject json = JObject.Load(reader); ProcessJson(json); + _receivedMessages++; } } } diff --git a/TimberWinR/Inputs/UdpInputListener.cs b/TimberWinR/Inputs/UdpInputListener.cs index 122da64..9cbaf3d 100644 --- a/TimberWinR/Inputs/UdpInputListener.cs +++ b/TimberWinR/Inputs/UdpInputListener.cs @@ -46,7 +46,9 @@ namespace TimberWinR.Inputs _port = port; LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port); - + + _receivedMessages = 0; + _udpListener = new System.Net.Sockets.UdpClient(port); _listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener)); @@ -76,7 +78,9 @@ namespace TimberWinR.Inputs var data = Encoding.ASCII.GetString(bytes, 0, bytes.Length); JObject json = JObject.Parse(data); ProcessJson(json); + _receivedMessages++; } + _udpListener.Close(); } catch (Exception ex) { diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index e0a7966..34b04cb 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -2,6 +2,8 @@ using System.Collections.Generic; using System.Configuration; using System.Linq; +using System.Linq.Expressions; +using System.Linq.Dynamic; using System.Reflection; using System.Runtime.Remoting.Channels; using System.Text; @@ -11,7 +13,7 @@ using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; using TimberWinR.Outputs; - +using System.CodeDom.Compiler; namespace TimberWinR.Parser { @@ -19,11 +21,12 @@ namespace TimberWinR.Parser { void Validate(); } + public abstract class LogstashFilter : IValidateSchema { public abstract bool Apply(JObject json); - + protected void RenameProperty(JObject json, string oldName, string newName) { JToken token = json[oldName]; @@ -37,67 +40,22 @@ namespace TimberWinR.Parser } public abstract JObject ToJson(); - + protected bool EvaluateCondition(JObject json, string condition) { - // Create a new instance of the C# compiler + var cond = condition; - IList keys = json.Properties().Select(p => p.Name).ToList(); + IList keys = json.Properties().Select(pn => pn.Name).ToList(); foreach (string key in keys) cond = cond.Replace(string.Format("[{0}]", key), string.Format("{0}", json[key].ToString())); - var compiler = new CSharpCodeProvider(); + var p = Expression.Parameter(typeof (JObject), ""); + var e = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] {p}, null, cond); - // Create some parameters for the compiler - var parms = new System.CodeDom.Compiler.CompilerParameters - { - GenerateExecutable = false, - GenerateInMemory = true - }; - parms.ReferencedAssemblies.Add("System.dll"); - parms.ReferencedAssemblies.Add("System.Core.dll"); - parms.ReferencedAssemblies.Add("Newtonsoft.Json.dll"); + var result = e.Compile().DynamicInvoke(json); - var code = string.Format(@" using System; - using System.Linq; - using Newtonsoft.Json; - using Newtonsoft.Json.Linq; - - class EvaluatorClass - {{ - public JObject json {{ get; set; }} - public bool Evaluate() - {{ - return {0}; - }} - }}", cond); - - // Try to compile the string into an assembly - var results = compiler.CompileAssemblyFromSource(parms, new string[] { code }); - - // If there weren't any errors get an instance of "MyClass" and invoke - // the "Message" method on it - if (results.Errors.Count == 0) - { - var evClass = results.CompiledAssembly.CreateInstance("EvaluatorClass"); - evClass.GetType().GetProperty("json").SetValue(evClass, json, null); - - var result = evClass.GetType(). - GetMethod("Evaluate"). - Invoke(evClass, null); - return bool.Parse(result.ToString()); - } - else - { - foreach (var e in results.Errors) - { - LogManager.GetCurrentClassLogger().Error(e); - LogManager.GetCurrentClassLogger().Error("Bad Code: {0}", code); - } - } - - return false; + return (bool) result; } protected void RemoveProperties(JToken token, string[] fields) { @@ -105,6 +63,7 @@ namespace TimberWinR.Parser if (container == null) return; List removeList = new List(); + foreach (JToken el in container.Children()) { JProperty p = el as JProperty; @@ -119,6 +78,7 @@ namespace TimberWinR.Parser { el.Remove(); } + } protected void ReplaceProperty(JObject json, string propertyName, string propertyValue) @@ -155,7 +115,7 @@ namespace TimberWinR.Parser } public abstract void Validate(); - + } [JsonObject(MemberSerialization.OptIn)] @@ -204,7 +164,7 @@ namespace TimberWinR.Parser To = to; } } - + public class WindowsEvent : IValidateSchema { public enum FormatKinds @@ -247,7 +207,7 @@ namespace TimberWinR.Parser public bool FormatMsg { get; set; } [JsonProperty(PropertyName = "interval")] public int Interval { get; set; } - + public WindowsEvent() { Interval = 60; // Every minute @@ -257,7 +217,7 @@ namespace TimberWinR.Parser FullText = true; BinaryFormat = FormatKinds.ASC; FullEventCode = false; - + Fields = new List(); Fields.Add(new Field("EventLog", "string")); Fields.Add(new Field("RecordNumber", "int")); @@ -273,7 +233,7 @@ namespace TimberWinR.Parser Fields.Add(new Field("ComputerName", "string")); Fields.Add(new Field("SID", "string")); Fields.Add(new Field("Message", "string")); - Fields.Add(new Field("Data", "string")); + Fields.Add(new Field("Data", "string")); } public void Validate() @@ -281,12 +241,12 @@ namespace TimberWinR.Parser } } - + public class Stdin : IValidateSchema { public void Validate() { - + } } @@ -313,7 +273,7 @@ namespace TimberWinR.Parser public void Validate() { - + } } @@ -329,7 +289,7 @@ namespace TimberWinR.Parser public void Validate() { - + } } @@ -361,7 +321,7 @@ namespace TimberWinR.Parser public int DtLines { get; set; } [JsonProperty(PropertyName = "dQuotes")] public bool DoubleQuotes { get; set; } - + [JsonProperty(PropertyName = "fields")] public List Fields { get; set; } @@ -374,7 +334,7 @@ namespace TimberWinR.Parser Separator = "auto"; Fields.Add(new Field("LogFilename", "string")); - Fields.Add(new Field("RowNumber", "integer")); + Fields.Add(new Field("RowNumber", "integer")); } public void Validate() @@ -385,7 +345,7 @@ namespace TimberWinR.Parser public class IISW3CLog : IValidateSchema - { + { [JsonProperty(PropertyName = "location")] public string Location { get; set; } [JsonProperty(PropertyName = "iCodepage")] @@ -411,42 +371,42 @@ namespace TimberWinR.Parser Fields = new List(); Fields.Add(new Field("LogFilename", "string")); - Fields.Add(new Field("LogRow", "integer" )); - Fields.Add(new Field("date", "DateTime" )); - Fields.Add(new Field("time", "DateTime" )); - Fields.Add(new Field("c-ip", "string" )); - Fields.Add(new Field("cs-username", "string" )); - Fields.Add(new Field("s-sitename", "string" )); - Fields.Add(new Field("s-computername", "integer" )); - Fields.Add(new Field("s-ip", "string" )); - Fields.Add(new Field("s-port", "integer" )); - Fields.Add(new Field("cs-method", "string" )); - Fields.Add(new Field("cs-uri-stem", "string" )); - Fields.Add(new Field("cs-uri-query", "string" )); - Fields.Add(new Field("sc-status", "integer" )); - Fields.Add(new Field("sc-substatus", "integer" )); - Fields.Add(new Field("sc-win32-status", "integer" )); - Fields.Add(new Field("sc-bytes", "integer" )); - Fields.Add(new Field("cs-bytes", "integer" )); - Fields.Add(new Field("time-taken", "integer" )); - Fields.Add(new Field("cs-version", "string" )); - Fields.Add(new Field("cs-host", "string" )); - Fields.Add(new Field("cs(User-Agent)", "string" )); - Fields.Add(new Field("cs(Cookie)", "string" )); - Fields.Add(new Field("cs(Referer)", "string" )); - Fields.Add(new Field("s-event", "string" )); - Fields.Add(new Field("s-process-type", "string" )); - Fields.Add(new Field("s-user-time", "double" )); - Fields.Add(new Field("s-kernel-time", "double" )); - Fields.Add(new Field("s-page-faults", "integer" )); - Fields.Add(new Field("s-total-procs", "integer" )); - Fields.Add(new Field("s-active-procs", "integer" )); + Fields.Add(new Field("LogRow", "integer")); + Fields.Add(new Field("date", "DateTime")); + Fields.Add(new Field("time", "DateTime")); + Fields.Add(new Field("c-ip", "string")); + Fields.Add(new Field("cs-username", "string")); + Fields.Add(new Field("s-sitename", "string")); + Fields.Add(new Field("s-computername", "integer")); + Fields.Add(new Field("s-ip", "string")); + Fields.Add(new Field("s-port", "integer")); + Fields.Add(new Field("cs-method", "string")); + Fields.Add(new Field("cs-uri-stem", "string")); + Fields.Add(new Field("cs-uri-query", "string")); + Fields.Add(new Field("sc-status", "integer")); + Fields.Add(new Field("sc-substatus", "integer")); + Fields.Add(new Field("sc-win32-status", "integer")); + Fields.Add(new Field("sc-bytes", "integer")); + Fields.Add(new Field("cs-bytes", "integer")); + Fields.Add(new Field("time-taken", "integer")); + Fields.Add(new Field("cs-version", "string")); + Fields.Add(new Field("cs-host", "string")); + Fields.Add(new Field("cs(User-Agent)", "string")); + Fields.Add(new Field("cs(Cookie)", "string")); + Fields.Add(new Field("cs(Referer)", "string")); + Fields.Add(new Field("s-event", "string")); + Fields.Add(new Field("s-process-type", "string")); + Fields.Add(new Field("s-user-time", "double")); + Fields.Add(new Field("s-kernel-time", "double")); + Fields.Add(new Field("s-page-faults", "integer")); + Fields.Add(new Field("s-total-procs", "integer")); + Fields.Add(new Field("s-active-procs", "integer")); Fields.Add(new Field("s-stopped-procs", "integer")); } public void Validate() { - + } } @@ -466,21 +426,21 @@ namespace TimberWinR.Parser public string Protocol { get; set; } [JsonProperty(PropertyName = "interval")] public int Interval { get; set; } - + public ElasticsearchOutput() { Protocol = "http"; Port = 9200; Index = ""; Host = new string[] { "localhost" }; - Timeout = 10000; + Timeout = 10000; NumThreads = 1; Interval = 1000; } } public class RedisOutput - { + { [JsonProperty(PropertyName = "host")] public string[] Host { get; set; } [JsonProperty(PropertyName = "index")] @@ -500,7 +460,7 @@ namespace TimberWinR.Parser { Port = 6379; Index = "logstash"; - Host = new string[] {"localhost"}; + Host = new string[] { "localhost" }; Timeout = 10000; BatchCount = 10; NumThreads = 1; @@ -554,13 +514,13 @@ namespace TimberWinR.Parser [JsonProperty("Stdin")] public Stdin[] Stdins { get; set; } } - + public partial class Grok : LogstashFilter, IValidateSchema { public class GrokFilterException : Exception { public GrokFilterException() - : base("Grok filter missing required match, must be 2 array entries.") + : base("Grok filter missing required match, must be 2 array entries.") { } } @@ -589,10 +549,10 @@ namespace TimberWinR.Parser public string[] AddTag { get; set; } [JsonProperty("add_field")] - public string[] AddField { get; set; } - + public string[] AddField { get; set; } + [JsonProperty("remove_field")] - public string[] RemoveField { get; set; } + public string[] RemoveField { get; set; } [JsonProperty("remove_tag")] public string[] RemoveTag { get; set; } @@ -602,7 +562,7 @@ namespace TimberWinR.Parser if (Match == null || Match.Length != 2) throw new GrokFilterException(); - if (AddTag != null && AddTag.Length%2 != 0) + if (AddTag != null && AddTag.Length % 2 != 0) throw new GrokAddTagException(); } } @@ -644,7 +604,7 @@ namespace TimberWinR.Parser public bool ConvertToUTC { get; set; } [JsonProperty("add_field")] - public string[] AddField { get; set; } + public string[] AddField { get; set; } public override void Validate() { @@ -681,7 +641,7 @@ namespace TimberWinR.Parser public override void Validate() { - + } } @@ -726,14 +686,14 @@ namespace TimberWinR.Parser [JsonProperty("remove_tag")] public string[] RemoveTag { get; set; } - + public override void Validate() { if (string.IsNullOrEmpty(Source)) throw new GeoIPMissingSourceException(); if (AddField != null && AddField.Length % 2 != 0) - throw new GeoIPAddFieldException(); + throw new GeoIPAddFieldException(); } } @@ -746,7 +706,7 @@ namespace TimberWinR.Parser { } } - + public class JsonAddFieldException : Exception { public JsonAddFieldException() @@ -788,7 +748,7 @@ namespace TimberWinR.Parser [JsonProperty("promote")] public string Promote { get; set; } - + public override void Validate() { @@ -796,11 +756,11 @@ namespace TimberWinR.Parser throw new JsonMissingSourceException(); if (AddField != null && AddField.Length % 2 != 0) - throw new JsonAddFieldException(); + throw new JsonAddFieldException(); } } - + public class Filter { @@ -815,11 +775,11 @@ namespace TimberWinR.Parser [JsonProperty("json")] public Json Json { get; set; } - + [JsonProperty("geoip")] public GeoIP GeoIP { get; set; } } - + public class TimberWinR { [JsonProperty("Inputs")] @@ -827,7 +787,7 @@ namespace TimberWinR.Parser [JsonProperty("Filters")] public List Filters { get; set; } [JsonProperty("Outputs")] - public OutputTargets Outputs { get; set; } + public OutputTargets Outputs { get; set; } public LogstashFilter[] AllFilters { @@ -849,7 +809,7 @@ namespace TimberWinR.Parser } } } - + public class RootObject { public TimberWinR TimberWinR { get; set; } diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index 4f87b97..435954a 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -62,6 +62,9 @@ + + ..\packages\System.Linq.Dynamic.1.0.3\lib\net40\System.Linq.Dynamic.dll + diff --git a/TimberWinR/packages.config b/TimberWinR/packages.config index ef85fad..9424c65 100644 --- a/TimberWinR/packages.config +++ b/TimberWinR/packages.config @@ -7,4 +7,5 @@ + \ No newline at end of file diff --git a/chocolateyUninstall.ps1.template b/chocolateyUninstall.ps1.template index b5b5131..f41e7e6 100644 --- a/chocolateyUninstall.ps1.template +++ b/chocolateyUninstall.ps1.template @@ -1,8 +1,8 @@ -$packageName = 'TimberWinR-${version}' -$installerType = 'msi' -$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' +$packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages +$installerType = 'msi' #only one of these: exe, msi, msu +$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url $silentArgs = '{86CA5678-4687-4352-99EB-783CDB7C8D82} /quiet' -$validExitCodes = @(0) +$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes