From 8694ede85d5ea213df1b85c420d0666fa18f1ff3 Mon Sep 17 00:00:00 2001 From: Greg Lutz Date: Fri, 19 Jun 2015 11:33:56 -0400 Subject: [PATCH 1/5] redis last output time and refactored diagnostics --- TimberWinR/Diagnostics/Diagnostics.cs | 61 +++++++++++++------- TimberWinR/Diagnostics/IDiagnosable.cs | 13 +++++ TimberWinR/Inputs/IISW3CInputListener.cs | 6 ++ TimberWinR/Inputs/InputListener.cs | 3 +- TimberWinR/Inputs/WindowsEvtInputListener.cs | 31 ++++++++-- TimberWinR/Outputs/OutputSender.cs | 3 +- TimberWinR/Outputs/Redis.cs | 3 + TimberWinR/Parser.cs | 3 +- 8 files changed, 94 insertions(+), 29 deletions(-) create mode 100644 TimberWinR/Diagnostics/IDiagnosable.cs diff --git a/TimberWinR/Diagnostics/Diagnostics.cs b/TimberWinR/Diagnostics/Diagnostics.cs index f009047..a8b64c2 100644 --- a/TimberWinR/Diagnostics/Diagnostics.cs +++ b/TimberWinR/Diagnostics/Diagnostics.cs @@ -8,7 +8,7 @@ using System.Threading; using System.Net; using System.Net.Sockets; using System.IO; - +using System.Linq.Expressions; using Newtonsoft.Json.Linq; using NLog; @@ -52,29 +52,48 @@ namespace TimberWinR.Diagnostics public JObject DiagnosticsOutput() { JObject json = new JObject( - new JProperty("timberwinr", - new JObject( - new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()), - new JProperty("messages", Manager.NumMessages), - new JProperty("startedon", Manager.StartedOn), - new JProperty("configfile", Manager.JsonConfig), - new JProperty("logdir", Manager.LogfileDir), - new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()), - new JProperty("inputs", - new JArray( - from i in Manager.Listeners - select new JObject(i.ToJson()))), - new JProperty("filters", - new JArray( - from f in Manager.Config.Filters - select new JObject(f.ToJson()))), - new JProperty("outputs", - new JArray( - from o in Manager.Outputs - select new JObject(o.ToJson())))))); + new JProperty("timberwinr", + new JObject( + new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()), + new JProperty("messages", Manager.NumMessages), + new JProperty("startedon", Manager.StartedOn), + new JProperty("configfile", Manager.JsonConfig), + new JProperty("logdir", Manager.LogfileDir), + new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()) + ))); + AddDiagnosis(json); return json; } + protected void AddDiagnosis(JObject wrapper) + { + wrapper.Add("inputs", GetDiagnosisByType("inputs", Manager.Listeners.ToList())); + wrapper.Add("filters", GetDiagnosisByType("inputs", Manager.Config.Filters.ToList())); + wrapper.Add("outputs", GetDiagnosisByType("inputs", Manager.Outputs.ToList())); + } + + protected JObject GetDiagnosisByType(String type, List diags) + { + JObject category = new JObject(); + foreach(IDiagnosable diag in diags) + { + JArray array = GetTypeArray(diag.GetType().Name.ToString(), category); + array.Add(diag.ToJson()); + } + return category; + } + + protected JArray GetTypeArray(String name, JObject category) + { + JArray ret = (JArray)category.GetValue(name); + if (ret == null) + { + ret = new JArray(); + category.Add(new JProperty(name, ret)); + } + return ret; + } + private void DiagnosticCallback(IAsyncResult result) { if (web == null) diff --git a/TimberWinR/Diagnostics/IDiagnosable.cs b/TimberWinR/Diagnostics/IDiagnosable.cs new file mode 100644 index 0000000..8067e41 --- /dev/null +++ b/TimberWinR/Diagnostics/IDiagnosable.cs @@ -0,0 +1,13 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using Newtonsoft.Json.Linq; + +namespace TimberWinR.Diagnostics +{ + public interface IDiagnosable + { + JObject ToJson(); + } +} diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index 6c92a19..cbda47d 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -145,6 +145,12 @@ namespace TimberWinR.Inputs catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); + try + { + if (!Stop) + syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); + } + catch (Exception) { } } } } diff --git a/TimberWinR/Inputs/InputListener.cs b/TimberWinR/Inputs/InputListener.cs index c29ae5e..2eb49bc 100644 --- a/TimberWinR/Inputs/InputListener.cs +++ b/TimberWinR/Inputs/InputListener.cs @@ -8,10 +8,11 @@ using System.Linq; using System.Text; using System.Threading; using NLog; +using TimberWinR.Diagnostics; namespace TimberWinR.Inputs { - public abstract class InputListener + public abstract class InputListener: IDiagnosable { public CancellationToken CancelToken { get; set; } public event Action OnMessageRecieved; diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index 18d73ba..9381e3f 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -8,6 +8,7 @@ using TimberWinR.Parser; using LogQuery = Interop.MSUtil.LogQueryClassClass; using EventLogInputFormat = Interop.MSUtil.COMEventLogInputContextClassClass; using LogRecordSet = Interop.MSUtil.ILogRecordset; +using System.IO; namespace TimberWinR.Inputs { @@ -97,12 +98,13 @@ namespace TimberWinR.Inputs // Execute the query if (!CancelToken.IsCancellationRequested) { + var oLogQuery = new LogQuery(); try - { - var oLogQuery = new LogQuery(); + { var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location); var rsfiles = oLogQuery.Execute(qfiles, iFmt); + for (; !rsfiles.atEnd(); rsfiles.moveNext()) { var record = rsfiles.getRecord(); @@ -113,7 +115,7 @@ namespace TimberWinR.Inputs logName); var rcount = oLogQuery.Execute(qcount, iFmt); var qr = rcount.getRecord(); - var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); + var lrn = (Int64) qr.getValueEx("MaxRecordNumber"); logFileMaxRecords[logName] = lrn; } } @@ -137,12 +139,13 @@ namespace TimberWinR.Inputs object v = record.getValue(field.Name); if (field.Name == "Data") v = ToPrintable(v.ToString()); - if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && field.DataType == typeof (DateTime)) + if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && + field.DataType == typeof (DateTime)) v = ((DateTime) v).ToUniversalTime(); json.Add(new JProperty(field.Name, v)); } - var lrn = (Int64)record.getValueEx("RecordNumber"); + var lrn = (Int64) record.getValueEx("RecordNumber"); logFileMaxRecords[fileName] = lrn; ProcessJson(json); @@ -163,6 +166,24 @@ namespace TimberWinR.Inputs { LogManager.GetCurrentClassLogger().Error(ex); } + finally + { + try + { + oLogQuery = null; + // Sleep + if (!Stop) + syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); + } + catch (OperationCanceledException) + { + } + catch (Exception ex1) + { + LogManager.GetCurrentClassLogger().Warn(ex1); + } + + } } } Finished(); diff --git a/TimberWinR/Outputs/OutputSender.cs b/TimberWinR/Outputs/OutputSender.cs index 0e3cc2d..549e0d0 100644 --- a/TimberWinR/Outputs/OutputSender.cs +++ b/TimberWinR/Outputs/OutputSender.cs @@ -4,11 +4,12 @@ using System.Linq; using System.Text; using System.Threading; using Newtonsoft.Json.Linq; +using TimberWinR.Diagnostics; using TimberWinR.Inputs; namespace TimberWinR.Outputs { - public abstract class OutputSender + public abstract class OutputSender : IDiagnosable { public CancellationToken CancelToken { get; private set; } private List _inputs; diff --git a/TimberWinR/Outputs/Redis.cs b/TimberWinR/Outputs/Redis.cs index 1dd0fb8..23a3782 100644 --- a/TimberWinR/Outputs/Redis.cs +++ b/TimberWinR/Outputs/Redis.cs @@ -132,6 +132,7 @@ namespace TimberWinR.Outputs private long _errorCount; private long _redisDepth; private DateTime? _lastErrorTimeUTC; + private DateTime? _lastSentTimeUTC; private readonly int _maxQueueSize; private readonly bool _queueOverflowDiscardOldest; private BatchCounter _batchCounter; @@ -180,6 +181,7 @@ namespace TimberWinR.Outputs new JProperty("host", string.Join(",", _redisHosts)), new JProperty("errors", _errorCount), new JProperty("lastErrorTimeUTC", _lastErrorTimeUTC), + new JProperty("lastSentTimeUTC", _lastSentTimeUTC), new JProperty("redisQueueDepth", _redisDepth), new JProperty("sentMessageCount", _sentMessages), new JProperty("queuedMessageCount", _jsonQueue.Count), @@ -348,6 +350,7 @@ namespace TimberWinR.Outputs Interlocked.Add(ref _sentMessages, messages.Length); client.EndPipe(); sentSuccessfully = true; + _lastSentTimeUTC = DateTime.UtcNow; if (messages.Length > 0) _manager.IncrementMessageCount(messages.Length); } diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 765d0b9..38d3c93 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -15,6 +15,7 @@ using NLog; using NLog.Config; using TimberWinR.Outputs; using System.CodeDom.Compiler; +using TimberWinR.Diagnostics; namespace TimberWinR.Parser { @@ -26,7 +27,7 @@ namespace TimberWinR.Parser } - public abstract class LogstashFilter : IValidateSchema + public abstract class LogstashFilter : IValidateSchema, IDiagnosable { public abstract bool Apply(JObject json); From 097ffd34c39e61b487fc57bd810126ce5c5a4163 Mon Sep 17 00:00:00 2001 From: Greg Lutz Date: Tue, 23 Jun 2015 06:54:36 -0400 Subject: [PATCH 2/5] version 2 --- TimberWinR.ServiceHost/Properties/AssemblyInfo.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 5c22a1d..6a4e683 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.3.26.0")] -[assembly: AssemblyFileVersion("1.3.26.0")] +[assembly: AssemblyVersion("2.0.0.0")] +[assembly: AssemblyFileVersion("2.0.0.0")] From 45e4e804883223a49e5beb827d13243a7b2d98cb Mon Sep 17 00:00:00 2001 From: Greg Lutz Date: Tue, 23 Jun 2015 09:59:56 -0400 Subject: [PATCH 3/5] typo aka sloppy cut and paste --- TimberWinR/Diagnostics/Diagnostics.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TimberWinR/Diagnostics/Diagnostics.cs b/TimberWinR/Diagnostics/Diagnostics.cs index a8b64c2..aa49369 100644 --- a/TimberWinR/Diagnostics/Diagnostics.cs +++ b/TimberWinR/Diagnostics/Diagnostics.cs @@ -68,7 +68,7 @@ namespace TimberWinR.Diagnostics protected void AddDiagnosis(JObject wrapper) { wrapper.Add("inputs", GetDiagnosisByType("inputs", Manager.Listeners.ToList())); - wrapper.Add("filters", GetDiagnosisByType("inputs", Manager.Config.Filters.ToList())); + wrapper.Add("filters", GetDiagnosisByType("filters", Manager.Config.Filters.ToList())); wrapper.Add("outputs", GetDiagnosisByType("inputs", Manager.Outputs.ToList())); } From 7d43c2cb6708247ca61236db87ab71a555393e19 Mon Sep 17 00:00:00 2001 From: Greg Lutz Date: Thu, 25 Jun 2015 06:48:12 -0400 Subject: [PATCH 4/5] backing off 2.0 plan --- TimberWinR.ServiceHost/Properties/AssemblyInfo.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 6a4e683..051152a 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.0.0.0")] -[assembly: AssemblyFileVersion("2.0.0.0")] +[assembly: AssemblyVersion("1.3.27.0")] +[assembly: AssemblyFileVersion("1.3.27.0")] From 2c9d998794f21e65321debe3f0ea3388046efd41 Mon Sep 17 00:00:00 2001 From: Greg Lutz Date: Thu, 25 Jun 2015 07:32:11 -0400 Subject: [PATCH 5/5] moved wait to finally block --- TimberWinR/Inputs/IISW3CInputListener.cs | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index cbda47d..31c5ca2 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -135,8 +135,6 @@ namespace TimberWinR.Inputs rs.close(); GC.Collect(); } - if (!Stop) - syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } catch (OperationCanceledException) { @@ -145,13 +143,18 @@ namespace TimberWinR.Inputs catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); + } + finally + { try { if (!Stop) syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken); } - catch (Exception) { } - } + catch (Exception) + { + } + } } } }