Merge branch 'rel-1.3.27.0' of https://github.com/Cimpress-MCP/TimberWinR into rel-2.0.0

Backing off plan for rel 2.0.0

TimberWinR.ServiceHost/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.26.0")]
+[assembly: AssemblyVersion("1.3.27.0")]
-[assembly: AssemblyFileVersion("1.3.26.0")]
+[assembly: AssemblyFileVersion("1.3.27.0")]

TimberWinR/Diagnostics/Diagnostics.cs

@@ -8,7 +8,7 @@ using System.Threading;
 using System.Net;
 using System.Net.Sockets;
 using System.IO;
-
+using System.Linq.Expressions;
 using Newtonsoft.Json.Linq;
 
 using NLog;
@@ -52,29 +52,48 @@ namespace TimberWinR.Diagnostics
         public JObject DiagnosticsOutput()
         {
             JObject json = new JObject(
-                           new JProperty("timberwinr",
+                new JProperty("timberwinr",
-                               new JObject(
+                    new JObject(
-                                   new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                        new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
-                                   new JProperty("messages", Manager.NumMessages),
+                        new JProperty("messages", Manager.NumMessages),
-                                   new JProperty("startedon", Manager.StartedOn),
+                        new JProperty("startedon", Manager.StartedOn),
-                                   new JProperty("configfile", Manager.JsonConfig),
+                        new JProperty("configfile", Manager.JsonConfig),
-                                   new JProperty("logdir", Manager.LogfileDir),
+                        new JProperty("logdir", Manager.LogfileDir),
-                                   new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
+                        new JProperty("logginglevel", LogManager.GlobalThreshold.ToString())
-                                   new JProperty("inputs",
+                        )));
-                                       new JArray(
+            AddDiagnosis(json);
-                                           from i in Manager.Listeners
-                                           select new JObject(i.ToJson()))),
-                                   new JProperty("filters",
-                                       new JArray(
-                                           from f in Manager.Config.Filters
-                                           select new JObject(f.ToJson()))),
-                                   new JProperty("outputs",
-                                       new JArray(
-                                           from o in Manager.Outputs
-                                           select new JObject(o.ToJson()))))));
             return json;
         }
 
+        protected void AddDiagnosis(JObject wrapper)
+        {
+            wrapper.Add("inputs", GetDiagnosisByType("inputs", Manager.Listeners.ToList<IDiagnosable>()));
+            wrapper.Add("filters", GetDiagnosisByType("filters", Manager.Config.Filters.ToList<IDiagnosable>()));
+            wrapper.Add("outputs", GetDiagnosisByType("inputs", Manager.Outputs.ToList<IDiagnosable>()));
+        }
+
+        protected JObject GetDiagnosisByType(String type, List<IDiagnosable> diags)
+        {
+            JObject category = new JObject();
+            foreach(IDiagnosable diag in diags)
+            {
+                JArray array = GetTypeArray(diag.GetType().Name.ToString(), category);
+                array.Add(diag.ToJson());
+            }
+            return category;
+        }
+
+        protected JArray GetTypeArray(String name, JObject category)
+        {
+            JArray ret = (JArray)category.GetValue(name);
+            if (ret == null)
+            {
+                ret = new JArray();
+                category.Add(new JProperty(name, ret));
+            }
+            return ret;
+        }
+
         private void DiagnosticCallback(IAsyncResult result)
         {
             if (web == null)

TimberWinR/Diagnostics/IDiagnosable.cs

@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+
+namespace TimberWinR.Diagnostics
+{
+    public interface IDiagnosable
+    {
+        JObject ToJson();
+    }
+}

TimberWinR/Inputs/IISW3CInputListener.cs

@@ -135,8 +135,6 @@ namespace TimberWinR.Inputs
                                 rs.close();
                                 GC.Collect();
                             }
-                            if (!Stop)
-                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                         }
                         catch (OperationCanceledException)
                         {
@@ -145,7 +143,18 @@ namespace TimberWinR.Inputs
                         catch (Exception ex)
                         {
                             LogManager.GetCurrentClassLogger().Error(ex);
-                        }                       
+                        }
+                        finally
+                        {
+                            try
+                            {
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                            }
+                            catch (Exception)
+                            {
+                            }
+                        }
                     }
                 }
             }

TimberWinR/Inputs/InputListener.cs

@@ -8,10 +8,11 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using NLog;
+using TimberWinR.Diagnostics;
 
 namespace TimberWinR.Inputs
 {
-    public abstract class InputListener
+    public abstract class InputListener: IDiagnosable
     {
         public CancellationToken CancelToken { get; set; }
         public event Action<JObject> OnMessageRecieved;

TimberWinR/Inputs/WindowsEvtInputListener.cs

@@ -8,6 +8,7 @@ using TimberWinR.Parser;
 using LogQuery = Interop.MSUtil.LogQueryClassClass;
 using EventLogInputFormat = Interop.MSUtil.COMEventLogInputContextClassClass;
 using LogRecordSet = Interop.MSUtil.ILogRecordset;
+using System.IO;
 
 namespace TimberWinR.Inputs
 {
@@ -97,12 +98,13 @@ namespace TimberWinR.Inputs
                     // Execute the query
                     if (!CancelToken.IsCancellationRequested)
                     {
+                        var oLogQuery = new LogQuery();
                         try
-                        {                          
+                        {
-                            var oLogQuery = new LogQuery();
 
                             var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
                             var rsfiles = oLogQuery.Execute(qfiles, iFmt);
+
                             for (; !rsfiles.atEnd(); rsfiles.moveNext())
                             {
                                 var record = rsfiles.getRecord();
@@ -113,7 +115,7 @@ namespace TimberWinR.Inputs
                                         logName);
                                     var rcount = oLogQuery.Execute(qcount, iFmt);
                                     var qr = rcount.getRecord();
-                                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
+                                    var lrn = (Int64) qr.getValueEx("MaxRecordNumber");
                                     logFileMaxRecords[logName] = lrn;
                                 }
                             }
@@ -137,12 +139,13 @@ namespace TimberWinR.Inputs
                                         object v = record.getValue(field.Name);
                                         if (field.Name == "Data")
                                             v = ToPrintable(v.ToString());
-                                        if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && field.DataType == typeof (DateTime))
+                                        if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") &&
+                                            field.DataType == typeof (DateTime))
                                             v = ((DateTime) v).ToUniversalTime();
                                         json.Add(new JProperty(field.Name, v));
                                     }
 
-                                    var lrn = (Int64)record.getValueEx("RecordNumber");
+                                    var lrn = (Int64) record.getValueEx("RecordNumber");
                                     logFileMaxRecords[fileName] = lrn;
 
                                     ProcessJson(json);
@@ -163,6 +166,24 @@ namespace TimberWinR.Inputs
                         {
                             LogManager.GetCurrentClassLogger().Error(ex);
                         }
+                        finally
+                        {
+                            try
+                            {
+                                oLogQuery = null;
+                                // Sleep 
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                            }
+                            catch (OperationCanceledException)
+                            {
+                            }
+                            catch (Exception ex1)
+                            {
+                                LogManager.GetCurrentClassLogger().Warn(ex1);
+                            }
+                            
+                        }
                     }
                 }
                 Finished();

TimberWinR/Outputs/OutputSender.cs

@@ -4,11 +4,12 @@ using System.Linq;
 using System.Text;
 using System.Threading;
 using Newtonsoft.Json.Linq;
+using TimberWinR.Diagnostics;
 using TimberWinR.Inputs;
 
 namespace TimberWinR.Outputs
 {
-    public abstract class OutputSender
+    public abstract class OutputSender : IDiagnosable
     {
         public CancellationToken CancelToken { get; private set; }        
         private List<InputListener> _inputs;

TimberWinR/Outputs/Redis.cs

@@ -132,6 +132,7 @@ namespace TimberWinR.Outputs
         private long _errorCount;
         private long _redisDepth;
         private DateTime? _lastErrorTimeUTC;
+        private DateTime? _lastSentTimeUTC;
         private readonly int _maxQueueSize;
         private readonly bool _queueOverflowDiscardOldest;
         private BatchCounter _batchCounter;
@@ -180,6 +181,7 @@ namespace TimberWinR.Outputs
                         new JProperty("host", string.Join(",", _redisHosts)),
                         new JProperty("errors", _errorCount),
                         new JProperty("lastErrorTimeUTC", _lastErrorTimeUTC),
+                        new JProperty("lastSentTimeUTC", _lastSentTimeUTC),
                         new JProperty("redisQueueDepth", _redisDepth),
                         new JProperty("sentMessageCount", _sentMessages),
                         new JProperty("queuedMessageCount", _jsonQueue.Count),
@@ -317,9 +319,13 @@ namespace TimberWinR.Outputs
                                 _batchCounter.SampleQueueDepth(_jsonQueue.Count);
                                 // Re-compute current batch size
 
-                                LogManager.GetCurrentClassLogger().Trace("{0}: Average Queue Depth: {1}, Current Length: {2}", Thread.CurrentThread.ManagedThreadId, _batchCounter.AverageQueueDepth(), _jsonQueue.Count); 
+                                LogManager.GetCurrentClassLogger()
-       
+                                    .Trace("{0}: Average Queue Depth: {1}, Current Length: {2}",
-                                _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count, _currentBatchCount);
+                                        Thread.CurrentThread.ManagedThreadId, _batchCounter.AverageQueueDepth(),
+                                        _jsonQueue.Count);
+
+                                _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count,
+                                    _currentBatchCount);
 
                                 messages = _jsonQueue.Take(_currentBatchCount).ToArray();
                                 _jsonQueue.RemoveRange(0, messages.Length);
@@ -340,7 +346,9 @@ namespace TimberWinR.Outputs
                                             {
                                                 client.StartPipe();
                                                 LogManager.GetCurrentClassLogger()
-                                                    .Debug("{0}: Sending {1} Messages to {2}", Thread.CurrentThread.ManagedThreadId, messages.Length, client.Host);
+                                                    .Debug("{0}: Sending {1} Messages to {2}",
+                                                        Thread.CurrentThread.ManagedThreadId, messages.Length,
+                                                        client.Host);
 
                                                 try
                                                 {
@@ -348,6 +356,7 @@ namespace TimberWinR.Outputs
                                                     Interlocked.Add(ref _sentMessages, messages.Length);
                                                     client.EndPipe();
                                                     sentSuccessfully = true;
+                                                    _lastSentTimeUTC = DateTime.UtcNow;
                                                     if (messages.Length > 0)
                                                         _manager.IncrementMessageCount(messages.Length);
                                                 }
@@ -391,9 +400,7 @@ namespace TimberWinR.Outputs
                                         _jsonQueue.InsertRange(0, messages);
                                     }
                                 }
-                            }                          
+                            }
-                            if (!Stop)
-                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                         }
                         catch (OperationCanceledException)
                         {
@@ -409,6 +416,17 @@ namespace TimberWinR.Outputs
                             Interlocked.Increment(ref _errorCount);
                             LogManager.GetCurrentClassLogger().Error(ex);
                         }
+                        finally
+                        {
+                            try
+                            {
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
+                            }
+                            catch (Exception)
+                            {
+                            }
+                        }
                     }
                 }
             }

TimberWinR/Parser.cs

@@ -15,6 +15,7 @@ using NLog;
 using NLog.Config;
 using TimberWinR.Outputs;
 using System.CodeDom.Compiler;
+using TimberWinR.Diagnostics;
 
 namespace TimberWinR.Parser
 {
@@ -26,7 +27,7 @@ namespace TimberWinR.Parser
     }
 
 
-    public abstract class LogstashFilter : IValidateSchema
+    public abstract class LogstashFilter : IValidateSchema, IDiagnosable
     {
         public abstract bool Apply(JObject json);