Merge pull request #4 from efontana/rel1.x

Rel1.x

TimberWinR.ServiceHost/Program.cs

@@ -38,7 +38,8 @@ namespace TimberWinR.ServiceHost
                
                 hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c);
                 hostConfigurator.AddCommandLineDefinition("logLevel", c => arguments.LogLevel = c);
-                hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c);    
+                hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c);
+                hostConfigurator.AddCommandLineDefinition("diagnosticPort", c => arguments.DiagnosticPort = int.Parse(c));    
 
                 hostConfigurator.ApplyCommandLine();
                 hostConfigurator.RunAsLocalSystem();
@@ -59,6 +60,8 @@ namespace TimberWinR.ServiceHost
                         AddServiceParameter("-configFile", arguments.ConfigFile);
                         AddServiceParameter("-logLevel", arguments.LogLevel);
                         AddServiceParameter("-logDir", arguments.LogfileDir);
+                        if (arguments.DiagnosticPort > 0)
+                            AddServiceParameter("-diagnosticPort", arguments.DiagnosticPort);
                     }
                 });
             });
@@ -77,6 +80,21 @@ namespace TimberWinR.ServiceHost
                 Registry.SetValue(keyPath, keyName, currentValue);               
             }
         }
+
+        private static void AddServiceParameter(string paramName, int value)
+        {
+            string keyPath = @"HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\TimberWinR";
+            string keyName = "ImagePath";
+
+            string currentValue = Registry.GetValue(keyPath, keyName, "").ToString();
+
+            if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
+            {
+                currentValue += string.Format(" {0}:{1}", paramName, value);
+                Registry.SetValue(keyPath, keyName, currentValue);
+            }
+        }
+
     }
 
     internal class Arguments
@@ -84,9 +102,11 @@ namespace TimberWinR.ServiceHost
         public string ConfigFile { get; set; }
         public string LogLevel { get; set; }
         public string LogfileDir { get; set; }
+        public int DiagnosticPort { get; set; }
 
         public Arguments()
         {
+            DiagnosticPort = 5141;
             ConfigFile = "default.json";
             LogLevel = "Info";
             LogfileDir = @"C:\logs";
@@ -100,7 +120,7 @@ namespace TimberWinR.ServiceHost
         readonly CancellationToken _cancellationToken;
         readonly Task _serviceTask;
         private readonly Arguments _args;
-      
+        private TimberWinR.Diagnostics.Diagnostics _diags;
         private TimberWinR.Manager _manager;
 
         public TimberWinRService(Arguments args)
@@ -108,7 +128,7 @@ namespace TimberWinR.ServiceHost
             _args = args;
             _cancellationTokenSource = new CancellationTokenSource();
             _cancellationToken = _cancellationTokenSource.Token;
-            _serviceTask = new Task(RunService, _cancellationToken);
+            _serviceTask = new Task(RunService, _cancellationToken);           
         }
 
         public void Start()
@@ -118,8 +138,10 @@ namespace TimberWinR.ServiceHost
 
         public void Stop()
         {
-            _cancellationTokenSource.Cancel();          
+            _cancellationTokenSource.Cancel();
-            
+            if (_diags != null)
+             _diags.Shutdown();
+
             if (_manager != null)
                 _manager.Shutdown();
         }
@@ -130,6 +152,8 @@ namespace TimberWinR.ServiceHost
         private void RunService()
         {
             _manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _cancellationToken);
+            if (_args.DiagnosticPort > 0)
+                _diags = new Diagnostics.Diagnostics(_manager, _cancellationToken, _args.DiagnosticPort);
         }
     }
 }

TimberWinR/Configuration.cs

@@ -104,6 +104,7 @@ namespace TimberWinR
         {           
             if (!string.IsNullOrEmpty(jsonConfFile))
             {
+                LogManager.GetCurrentClassLogger().Info("Reading Configuration From {0}", jsonConfFile);
                 string json = File.ReadAllText(jsonConfFile);
 
                 return FromString(json, c);

TimberWinR/Inputs/IISW3CInputListener.cs

@@ -22,12 +22,13 @@ namespace TimberWinR.Inputs
     {
         private int _pollingIntervalInSeconds = 1;
         private TimberWinR.Parser.IISW3CLog _arguments;
-
+        private long _receivedMessages;
 
         public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
             : base(cancelToken, "Win32-IISLog")
         {
             _arguments = arguments;
+            _receivedMessages = 0;
             _pollingIntervalInSeconds = pollingIntervalInSeconds;
             var task = new Task(IISW3CWatcher, cancelToken);
             task.Start();
@@ -36,19 +37,37 @@ namespace TimberWinR.Inputs
         public override void Shutdown()
         {
             base.Shutdown();
-        }       
+        }
+
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("iisw3c",
+                    new JObject(
+                        new JProperty("messages", _receivedMessages),
+                        new JProperty("location", _arguments.Location),
+                        new JProperty("codepage", _arguments.CodePage),
+                        new JProperty("consolidateLogs", _arguments.ConsolidateLogs),
+                        new JProperty("dirTime", _arguments.DirTime),
+                        new JProperty("dQuotes", _arguments.DoubleQuotes),
+                        new JProperty("recurse", _arguments.Recurse),
+                        new JProperty("useDoubleQuotes", _arguments.DoubleQuotes)
+                        )));
+            return json;
+        }
+
 
         private void IISW3CWatcher()
         {
             var oLogQuery = new LogQuery();
-           
+
             var iFmt = new IISW3CLogInputFormat()
             {
                 codepage = _arguments.CodePage,
                 consolidateLogs = _arguments.ConsolidateLogs,
                 dirTime = _arguments.DirTime,
                 dQuotes = _arguments.DoubleQuotes,
-                iCheckpoint = CheckpointFileName,           
+                iCheckpoint = CheckpointFileName,
                 recurse = _arguments.Recurse,
                 useDoubleQuotes = _arguments.DoubleQuotes
             };
@@ -67,7 +86,7 @@ namespace TimberWinR.Inputs
                 {
                     var rs = oLogQuery.Execute(query, iFmt);
                     Dictionary<string, int> colMap = new Dictionary<string, int>();
-                    for (int col=0; col<rs.getColumnCount(); col++)
+                    for (int col = 0; col < rs.getColumnCount(); col++)
                     {
                         string colName = rs.getColumnName(col);
                         colMap[colName] = col;
@@ -86,16 +105,17 @@ namespace TimberWinR.Inputs
                                 if (!colMap.ContainsKey(field.Name))
                                     continue;
 
-                                object v = record.getValue(field.Name);                               
+                                object v = record.getValue(field.Name);
                                 if (field.DataType == typeof(DateTime))
                                 {
                                     DateTime dt = DateTime.Parse(v.ToString());
                                     json.Add(new JProperty(field.Name, dt));
-                                }  
+                                }
-                                else 
+                                else
                                     json.Add(new JProperty(field.Name, v));
-                            }                            
+                            }
                             ProcessJson(json);
+                            _receivedMessages++;
                         }
                     }
                     // Close the recordset

TimberWinR/Inputs/InputListener.cs

@@ -6,6 +6,7 @@ using System.Collections.Generic;
 using System.Linq;
 using System.Text;
 using System.Threading;
+using NLog;
 
 namespace TimberWinR.Inputs
 {
@@ -17,7 +18,14 @@ namespace TimberWinR.Inputs
         private string _typeName;
         public AutoResetEvent FinishedEvent { get; set; }
         public string CheckpointFileName { get; set; }
-       
+
+        public string InputType
+        {
+            get { return _typeName; }
+        }
+
+        public abstract JObject ToJson();
+
         public InputListener(CancellationToken token, string typeName)
         {
             CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString()));          
@@ -60,8 +68,9 @@ namespace TimberWinR.Inputs
                 if (File.Exists(CheckpointFileName))
                     File.Delete(CheckpointFileName);
             }
-            catch (Exception)
+            catch (Exception ex)
-            {               
+            {
+                LogManager.GetCurrentClassLogger().Error("Error Deleting Checkpoint File", ex);
             }          
         }
 

TimberWinR/Inputs/LogsListener.cs

@@ -24,10 +24,12 @@ namespace TimberWinR.Inputs
     {
         private int _pollingIntervalInSeconds;
         private TimberWinR.Parser.Log _arguments;
-
+        private long _receivedMessages;
+       
         public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 3)
             : base(cancelToken, "Win32-FileLog")
         {
+            _receivedMessages = 0;
             _arguments = arguments;
             _pollingIntervalInSeconds = pollingIntervalInSeconds;
             var task = new Task(FileWatcher, cancelToken);
@@ -39,6 +41,20 @@ namespace TimberWinR.Inputs
             base.Shutdown();
         }
 
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("log",
+                    new JObject(
+                        new JProperty("messages", _receivedMessages),
+                        new JProperty("location", _arguments.Location),
+                        new JProperty("codepage", _arguments.CodePage),
+                        new JProperty("splitLongLines", _arguments.SplitLongLines),                     
+                        new JProperty("recurse", _arguments.Recurse)
+                        )));
+            return json;
+        }
+
         private void FileWatcher()
         {                      
             var iFmt = new TextLineInputFormat()
@@ -81,17 +97,20 @@ namespace TimberWinR.Inputs
                                     continue;
 
                                 object v = record.getValue(field.Name);
-                                if (field.DataType == typeof(DateTime))
+                                if (field.DataType == typeof (DateTime))
                                 {
                                     DateTime dt = DateTime.Parse(v.ToString());
                                     json.Add(new JProperty(field.Name, dt));
                                 }
                                 else
-                                    json.Add(new JProperty(field.Name, v));                                
+                                    json.Add(new JProperty(field.Name, v));
                             }
                             string msg = json["Text"].ToString();
                             if (!string.IsNullOrEmpty(msg))
+                            {
                                 ProcessJson(json);
+                                _receivedMessages++;
+                            }
                         }
                     }
                     // Close the recordset

TimberWinR/Inputs/StdinListener.cs

@@ -20,6 +20,13 @@ namespace TimberWinR.Inputs
             _listenThread.Start();
         }
 
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("stdin", "enabled"));                 
+            return json;
+        }
+
         public override void Shutdown()
         {           
             base.Shutdown();

TimberWinR/Inputs/TcpInputListener.cs

@@ -13,52 +13,79 @@ namespace TimberWinR.Inputs
 {
     public class TcpInputListener : InputListener
     {
-        private readonly System.Net.Sockets.TcpListener _tcpListener;
+        private readonly System.Net.Sockets.TcpListener _tcpListenerV4;
-        private Thread _listenThread;       
+        private readonly System.Net.Sockets.TcpListener _tcpListenerV6;
+        private Thread _listenThreadV4;
+        private Thread _listenThreadV6;
         private readonly int _port;
+        private long _receivedMessages;
+
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("tcp",
+                    new JObject(
+                        new JProperty("port", _port),
+                        new JProperty("messages", _receivedMessages)
+                        )));
+
+            return json;
+        }
 
         public TcpInputListener(CancellationToken cancelToken, int port = 5140)
             : base(cancelToken, "Win32-Tcp")
         {
             _port = port;
-            _tcpListener = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
+          
-            _listenThread = new Thread(new ThreadStart(ListenForClients));
+            LogManager.GetCurrentClassLogger().Info("Tcp Input(v4/v6) on Port {0} Ready", _port);
-            _listenThread.Start();
+
+
+            _tcpListenerV6 = new System.Net.Sockets.TcpListener(IPAddress.IPv6Any, port);
+            _tcpListenerV4 = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
+
+            _listenThreadV4 = new Thread(new ParameterizedThreadStart(ListenForClients));
+            _listenThreadV4.Start(_tcpListenerV4);
+
+            _listenThreadV6 = new Thread(new ParameterizedThreadStart(ListenForClients));
+            _listenThreadV6.Start(_tcpListenerV6);
         }
 
 
         public override void Shutdown()
         {
-            this._tcpListener.Stop();
+            this._tcpListenerV4.Stop();
+            this._tcpListenerV6.Stop();
+
             Finished();
-            base.Shutdown();            
+            base.Shutdown();
-        }           
+        }
-        
 
-        private void ListenForClients()
+
+        private void ListenForClients(object olistener)
         {
-            this._tcpListener.Start();
+            System.Net.Sockets.TcpListener listener = olistener as System.Net.Sockets.TcpListener;
 
-            LogManager.GetCurrentClassLogger().Info("Tcp Input on Port {0} Ready", _port);
+            listener.Start();
 
+          
             while (!CancelToken.IsCancellationRequested)
             {
                 try
                 {
                     //blocks until a client has connected to the server
-                    TcpClient client = this._tcpListener.AcceptTcpClient();                   
+                    TcpClient client = listener.AcceptTcpClient();
-                  
+
                     // Wait for a client, spin up a thread.
                     var clientThread = new Thread(new ParameterizedThreadStart(HandleNewClient));
                     clientThread.Start(client);
-                }                  
+                }
                 catch (SocketException ex)
                 {
-                    if (ex.SocketErrorCode == SocketError.Interrupted)                        
+                    if (ex.SocketErrorCode == SocketError.Interrupted)
                         break;
-                    else                    
+                    else
-                        LogManager.GetCurrentClassLogger().Error(ex);                    
+                        LogManager.GetCurrentClassLogger().Error(ex);
-                }               
+                }
             }
         }
 
@@ -66,8 +93,9 @@ namespace TimberWinR.Inputs
         {
             var tcpClient = (TcpClient)client;
             NetworkStream clientStream = null;
-            do
+
-            {             
+            try
+            {
                 clientStream = tcpClient.GetStream();
                 var stream = new StreamReader(clientStream);
                 string line;
@@ -77,6 +105,7 @@ namespace TimberWinR.Inputs
                     {
                         JObject json = JObject.Parse(line);
                         ProcessJson(json);
+                        _receivedMessages++;
                     }
                     catch (Exception ex)
                     {
@@ -85,10 +114,16 @@ namespace TimberWinR.Inputs
                     if (CancelToken.IsCancellationRequested)
                         break;
                 }
-            } while (!CancelToken.IsCancellationRequested);          
+            }
-            
+            catch (Exception ex)
-            clientStream.Close(); 
+            {
-            tcpClient.Close();          
+                LogManager.GetCurrentClassLogger().Error("Tcp Exception", ex);
+            }
+
+            if (clientStream != null)
+                clientStream.Close();
+
+            tcpClient.Close();
             Finished();
         }
     }

TimberWinR/Inputs/WindowsEvtInputListener.cs

@@ -21,13 +21,14 @@ namespace TimberWinR.Inputs
     /// Listen to Windows Event Log
     /// </summary>
     public class WindowsEvtInputListener : InputListener
-    {      
+    {
         private int _pollingIntervalInSeconds = 1;
         private TimberWinR.Parser.WindowsEvent _arguments;
-      
+        private long _receivedMessages;
+
         public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
             : base(cancelToken, "Win32-Eventlog")
-        {          
+        {
             _arguments = arguments;
             _pollingIntervalInSeconds = pollingIntervalInSeconds;
             var task = new Task(EventWatcher, cancelToken);
@@ -36,7 +37,26 @@ namespace TimberWinR.Inputs
 
         public override void Shutdown()
         {
-            base.Shutdown();           
+            base.Shutdown();
+        }
+
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("windows_events",
+                    new JObject(
+                        new JProperty("messages", _receivedMessages),
+                        new JProperty("binaryFormat", _arguments.BinaryFormat.ToString()),
+                        new JProperty("direction", _arguments.Direction.ToString()),
+                        new JProperty("formatMsg", _arguments.FormatMsg),
+                        new JProperty("fullEventCode", _arguments.FullEventCode),
+                        new JProperty("fullText", _arguments.FullText),
+                        new JProperty("msgErrorMode", _arguments.MsgErrorMode.ToString()),
+                        new JProperty("stringsSep", _arguments.StringsSep),
+                        new JProperty("resolveSIDs", _arguments.ResolveSIDS),
+                        new JProperty("iCheckpoint", CheckpointFileName),
+                        new JProperty("source", _arguments.Source))));
+            return json;
         }
 
         private void EventWatcher()
@@ -53,12 +73,12 @@ namespace TimberWinR.Inputs
                 formatMsg = _arguments.FormatMsg,
                 fullEventCode = _arguments.FullEventCode,
                 fullText = _arguments.FullText,
-                msgErrorMode =  _arguments.MsgErrorMode.ToString(),
+                msgErrorMode = _arguments.MsgErrorMode.ToString(),
                 stringsSep = _arguments.StringsSep,
                 resolveSIDs = _arguments.ResolveSIDS,
-                iCheckpoint = CheckpointFileName,               
+                iCheckpoint = CheckpointFileName,
             };
-         
+
 
             // Create the query
             var query = string.Format("SELECT * FROM {0}", _arguments.Source);
@@ -69,7 +89,7 @@ namespace TimberWinR.Inputs
             {
                 try
                 {
-                    var rs = oLogQuery.Execute(query, iFmt);                 
+                    var rs = oLogQuery.Execute(query, iFmt);
                     // Browse the recordset
                     for (; !rs.atEnd(); rs.moveNext())
                     {
@@ -82,11 +102,12 @@ namespace TimberWinR.Inputs
                             {
                                 object v = record.getValue(field.Name);
                                 if (field.Name == "Data")
-                                    v = ToPrintable(v.ToString());                               
+                                    v = ToPrintable(v.ToString());
                                 json.Add(new JProperty(field.Name, v));
                             }
-                                                      
+
                             ProcessJson(json);
+                            _receivedMessages++;
                         }
                     }
                     // Close the recordset
@@ -98,11 +119,11 @@ namespace TimberWinR.Inputs
                     LogManager.GetCurrentClassLogger().Error("WindowsEventListener", ex);
                     firstQuery = true;
                     oLogQuery = new LogQuery();
-                }               
+                }
                 System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
             }
 
             Finished();
-        }       
+        }
     }
 }

TimberWinR/Manager.cs

@@ -21,7 +21,24 @@ namespace TimberWinR
         public Configuration Config { get; set; }
         public List<OutputSender> Outputs { get; set; }
         public List<TcpInputListener> Tcps { get; set; }
-        public List<InputListener> Listeners { get; set;  } 
+        public List<InputListener> Listeners { get; set;  }
+        public DateTime StartedOn { get; set; }
+        public string JsonConfig { get; set; }
+        public string LogfileDir { get; set; }
+
+        public int NumConnections {
+            get { return numConnections; }
+        }
+
+        public int NumMessages
+        {
+            get { return numMessages; }
+        }
+
+        private static int numConnections;
+        private static int numMessages;
+
+     
         public void Shutdown()
         {
             LogManager.GetCurrentClassLogger().Info("Shutting Down");
@@ -30,8 +47,22 @@ namespace TimberWinR
                 listener.Shutdown();
         }
 
+
+        public void IncrementMessageCount(int count = 1)
+        {            
+            Interlocked.Add(ref numMessages, count);
+        }
+    
         public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken)
         {
+            StartedOn = DateTime.UtcNow;
+
+            JsonConfig = jsonConfigFile;
+            LogfileDir = logfileDir;
+
+            numMessages = 0;
+            numConnections = 0;
+
             Outputs = new List<OutputSender>();           
             Listeners = new List<InputListener>();
            
@@ -59,15 +90,16 @@ namespace TimberWinR
             // Is it a directory?
             if (Directory.Exists(jsonConfigFile))
             {
-                LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", jsonConfigFile);
+                DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
+                LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
                 Config = Configuration.FromDirectory(jsonConfigFile);              
             }
             else
             {
-                LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", jsonConfigFile);
-
                 var fi = new FileInfo(jsonConfigFile);
 
+                LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", fi.FullName);
+               
                 if (!fi.Exists)
                     throw new FileNotFoundException("Missing config file", jsonConfigFile);
 

TimberWinR/Outputs/Elasticsearch.cs

@@ -23,10 +23,16 @@ namespace TimberWinR.Outputs
         private readonly int _timeout;
         private readonly object _locker = new object();
         private readonly List<JObject> _jsonQueue;
+        private readonly int _numThreads;
+        private long _sentMessages;
+        private long _errorCount;
 
         public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken)
             : base(cancelToken)
         {
+            _sentMessages = 0;
+            _errorCount = 0;
+
             _protocol = eo.Protocol;
             _timeout = eo.Timeout;
             _manager = manager;
@@ -36,6 +42,8 @@ namespace TimberWinR.Outputs
             _index = eo.Index;
             _hostIndex = 0;
             _jsonQueue = new List<JObject>();
+            _numThreads = eo.NumThreads;
+
             for (int i = 0; i < eo.NumThreads; i++)
             {
                 var elsThread = new Task(ElasticsearchSender, cancelToken);
@@ -43,6 +51,25 @@ namespace TimberWinR.Outputs
             }
         }
 
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("elasticsearch",
+                    new JObject(
+                        new JProperty("host", string.Join(",", _host)),
+                        new JProperty("errors", _errorCount),                       
+                        new JProperty("sent_messages", _sentMessages),
+                        new JProperty("queued_messages", _jsonQueue.Count),
+                        new JProperty("port", _port),
+                        new JProperty("interval", _interval),
+                        new JProperty("threads", _numThreads),                                             
+                        new JProperty("hosts",
+                            new JArray(
+                                from h in _host
+                                select new JObject(
+                                    new JProperty("host", h)))))));
+            return json;
+        }
         // 
         // Pull off messages from the Queue, batch them up and send them all across
         // 
@@ -55,6 +82,8 @@ namespace TimberWinR.Outputs
                 {
                     messages = _jsonQueue.Take(1).ToArray();
                     _jsonQueue.RemoveRange(0, messages.Length);
+                    if (messages.Length > 0)
+                        _manager.IncrementMessageCount(messages.Length);
                 }
 
                 if (messages.Length > 0)
@@ -96,12 +125,18 @@ namespace TimberWinR.Outputs
                                             {
                                                 LogManager.GetCurrentClassLogger()
                                                     .Error("Failed to send: {0}", response.ErrorMessage);
+                                                Interlocked.Increment(ref _errorCount);
+                                            }
+                                            else
+                                            {
+                                                _sentMessages++;
                                             }
                                         });
                                     }
                                     catch (Exception error)
                                     {
                                         LogManager.GetCurrentClassLogger().Error(error);
+                                        Interlocked.Increment(ref _errorCount);
                                     }
                                 }
                             }
@@ -110,12 +145,14 @@ namespace TimberWinR.Outputs
                                 LogManager.GetCurrentClassLogger()
                                     .Fatal("Unable to connect with any Elasticsearch hosts, {0}",
                                         String.Join(",", _host));
+                                Interlocked.Increment(ref _errorCount);
                             }
                           
                         }
                         catch (Exception ex)
                         {
                             LogManager.GetCurrentClassLogger().Error(ex);
+                            Interlocked.Increment(ref _errorCount);
                         }
                     }
                 }

TimberWinR/Outputs/OutputSender.cs

@@ -24,6 +24,7 @@ namespace TimberWinR.Outputs
             listener.OnMessageRecieved += MessageReceivedHandler;            
         }
 
+        public abstract JObject ToJson();
         protected abstract void MessageReceivedHandler(JObject jsonMessage);
     }
 }

TimberWinR/Outputs/Redis.cs

@@ -29,6 +29,11 @@ namespace TimberWinR.Outputs
         private TimberWinR.Manager _manager;
         private readonly int _batchCount;
         private readonly int _interval;
+        private readonly int _numThreads;
+
+        private long _sentMessages;
+        private long _errorCount;
+        private long _redisDepth;
 
         /// <summary>
         /// Get the next client
@@ -61,9 +66,33 @@ namespace TimberWinR.Outputs
             return null;
         }
 
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("redis",
+                    new JObject(
+                        new JProperty("host", string.Join(",", _redisHosts)),
+                        new JProperty("errors", _errorCount),
+                        new JProperty("redis_depth", _redisDepth),
+                        new JProperty("sent_messages", _sentMessages),
+                        new JProperty("queued_messages", _jsonQueue.Count),
+                        new JProperty("port", _port),
+                        new JProperty("interval", _interval),
+                        new JProperty("threads", _numThreads),
+                        new JProperty("batchcount", _batchCount),
+                        new JProperty("index", _logstashIndexName),
+                        new JProperty("hosts",
+                            new JArray(
+                                from h in _redisHosts
+                                select new JObject(
+                                    new JProperty("host", h)))))));
+            return json;
+        }
+
         public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken)
             : base(cancelToken)
         {
+            _redisDepth = 0;
             _batchCount = ro.BatchCount;
             _manager = manager;
             _redisHostIndex = 0;
@@ -73,6 +102,8 @@ namespace TimberWinR.Outputs
             _timeout = ro.Timeout;
             _logstashIndexName = ro.Index;
             _interval = ro.Interval;
+            _numThreads = ro.NumThreads;
+            _errorCount = 0;
 
             for (int i = 0; i < ro.NumThreads; i++)
             {
@@ -81,6 +112,11 @@ namespace TimberWinR.Outputs
             }
         }
 
+        public override string ToString()
+        {
+            return string.Format("Redis Host: {0} Port: {1}, Threads: {2}, Interval: {3}, BatchCount: {4}", string.Join(",", _redisHosts) , _port, _numThreads, _interval, _batchCount);
+        }
+
         /// <summary>
         /// Forward on Json message to Redis Logstash queue
         /// </summary>
@@ -119,6 +155,8 @@ namespace TimberWinR.Outputs
                 {
                     messages = _jsonQueue.Take(_batchCount).ToArray();
                     _jsonQueue.RemoveRange(0, messages.Length);
+                    if (messages.Length > 0)
+                        _manager.IncrementMessageCount(messages.Length);
                 }
 
                 if (messages.Length > 0)
@@ -141,11 +179,13 @@ namespace TimberWinR.Outputs
                                     {
                                         try
                                         {
-                                            client.RPush(_logstashIndexName, jsonMessage);
+                                           _redisDepth = client.RPush(_logstashIndexName, jsonMessage);
+                                           _sentMessages++;
                                         }
                                         catch (SocketException ex)
                                         {
                                             LogManager.GetCurrentClassLogger().Warn(ex);
+                                            Interlocked.Increment(ref _errorCount);
                                         }
                                     }
                                     client.EndPipe();
@@ -153,6 +193,7 @@ namespace TimberWinR.Outputs
                                 }
                                 else
                                 {
+                                    Interlocked.Increment(ref _errorCount);
                                     LogManager.GetCurrentClassLogger()
                                         .Fatal("Unable to connect with any Redis hosts, {0}",
                                             String.Join(",", _redisHosts));
@@ -162,6 +203,7 @@ namespace TimberWinR.Outputs
                         catch (Exception ex)
                         {
                             LogManager.GetCurrentClassLogger().Error(ex);
+                            Interlocked.Increment(ref _errorCount);
                         }
                     }
                 }

TimberWinR/Outputs/Stdout.cs

@@ -14,10 +14,12 @@ namespace TimberWinR.Outputs
         private readonly int _interval;
         private readonly object _locker = new object();
         private readonly List<JObject> _jsonQueue;
+        private long _sentMessages;
 
         public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken)
             : base(cancelToken)
         {
+            _sentMessages = 0;
             _manager = manager;
             _interval = eo.Interval;
             _jsonQueue = new List<JObject>();
@@ -26,6 +28,16 @@ namespace TimberWinR.Outputs
             elsThread.Start();
         }
 
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("stdout",
+                    new JObject(
+                        new JProperty("sent_messages", _sentMessages))));
+                      
+            return json;
+        }
+
         // 
         // Pull off messages from the Queue, batch them up and send them all across
         // 
@@ -37,7 +49,7 @@ namespace TimberWinR.Outputs
                 lock (_locker)
                 {
                     messages = _jsonQueue.Take(1).ToArray();
-                    _jsonQueue.RemoveRange(0, messages.Length);
+                    _jsonQueue.RemoveRange(0, messages.Length);                  
                 }
 
                 if (messages.Length > 0)
@@ -47,6 +59,7 @@ namespace TimberWinR.Outputs
                         foreach (JObject obj in messages)
                         {
                             Console.WriteLine(obj.ToString());
+                            _sentMessages++;
                         }
                     }
                     catch (Exception ex)

TimberWinR/TimberWinR.csproj

@@ -67,6 +67,7 @@
   <ItemGroup>
     <Compile Include="Configuration.cs" />
     <Compile Include="ConfigurationErrors.cs" />
+    <Compile Include="Diagnostics\Diagnostics.cs" />
     <Compile Include="Filters\DateFilter.cs" />
     <Compile Include="Filters\FilterBase.cs" />
     <Compile Include="Filters\GrokFilter.cs" />
@@ -121,9 +122,7 @@
       <LastGenOutput>Resources.Designer.cs</LastGenOutput>
     </EmbeddedResource>
   </ItemGroup>
-  <ItemGroup>
+  <ItemGroup />
-    <Folder Include="Parsing\" />
-  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.

TimberWix/Product.wxs

@@ -5,7 +5,7 @@
     <Property Id="CONFIGFILE">default.json</Property>
     <Property Id="LOGDIR">c:\logs</Property>
     <Property Id="LOGLEVEL">Info</Property>
-    
+    <Property Id="DIAGPORT">5141</Property>
     <!--
 				We need to be able to uninstall a newer version from an older version.
 				The default reinstallmode is "omus", of which the 'o' means "reinstall if missing or older"
@@ -58,7 +58,7 @@
       Directory="INSTALLFOLDER"
       Impersonate="yes"
       Execute="deferred"
-      ExeCommand='"[INSTALLFOLDER]TimberWinR.ServiceHost.exe" install --autostart -configFile "[CONFIGFILE]" -logDir "[LOGDIR]" -logLevel "[LOGLEVEL]"'
+      ExeCommand='"[INSTALLFOLDER]TimberWinR.ServiceHost.exe" install --autostart -configFile "[CONFIGFILE]" -logDir "[LOGDIR]" -logLevel "[LOGLEVEL]" -diagnosticPort:[DIAGPORT]'
       Return='check'>      
     </CustomAction>