diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs index 9d9b190..d3c1b69 100644 --- a/TimberWinR.ServiceHost/Program.cs +++ b/TimberWinR.ServiceHost/Program.cs @@ -38,7 +38,8 @@ namespace TimberWinR.ServiceHost hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c); hostConfigurator.AddCommandLineDefinition("logLevel", c => arguments.LogLevel = c); - hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c); + hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c); + hostConfigurator.AddCommandLineDefinition("diagnosticPort", c => arguments.DiagnosticPort = int.Parse(c)); hostConfigurator.ApplyCommandLine(); hostConfigurator.RunAsLocalSystem(); @@ -59,6 +60,8 @@ namespace TimberWinR.ServiceHost AddServiceParameter("-configFile", arguments.ConfigFile); AddServiceParameter("-logLevel", arguments.LogLevel); AddServiceParameter("-logDir", arguments.LogfileDir); + if (arguments.DiagnosticPort > 0) + AddServiceParameter("-diagnosticPort", arguments.DiagnosticPort); } }); }); @@ -77,6 +80,21 @@ namespace TimberWinR.ServiceHost Registry.SetValue(keyPath, keyName, currentValue); } } + + private static void AddServiceParameter(string paramName, int value) + { + string keyPath = @"HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\services\TimberWinR"; + string keyName = "ImagePath"; + + string currentValue = Registry.GetValue(keyPath, keyName, "").ToString(); + + if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName))) + { + currentValue += string.Format(" {0}:{1}", paramName, value); + Registry.SetValue(keyPath, keyName, currentValue); + } + } + } internal class Arguments @@ -84,9 +102,11 @@ namespace TimberWinR.ServiceHost public string ConfigFile { get; set; } public string LogLevel { get; set; } public string LogfileDir { get; set; } + public int DiagnosticPort { get; set; } public Arguments() { + DiagnosticPort = 5141; ConfigFile = "default.json"; LogLevel = "Info"; LogfileDir = @"C:\logs"; @@ -100,7 +120,7 @@ namespace TimberWinR.ServiceHost readonly CancellationToken _cancellationToken; readonly Task _serviceTask; private readonly Arguments _args; - + private TimberWinR.Diagnostics.Diagnostics _diags; private TimberWinR.Manager _manager; public TimberWinRService(Arguments args) @@ -108,7 +128,7 @@ namespace TimberWinR.ServiceHost _args = args; _cancellationTokenSource = new CancellationTokenSource(); _cancellationToken = _cancellationTokenSource.Token; - _serviceTask = new Task(RunService, _cancellationToken); + _serviceTask = new Task(RunService, _cancellationToken); } public void Start() @@ -118,8 +138,10 @@ namespace TimberWinR.ServiceHost public void Stop() { - _cancellationTokenSource.Cancel(); - + _cancellationTokenSource.Cancel(); + if (_diags != null) + _diags.Shutdown(); + if (_manager != null) _manager.Shutdown(); } @@ -130,6 +152,8 @@ namespace TimberWinR.ServiceHost private void RunService() { _manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _cancellationToken); + if (_args.DiagnosticPort > 0) + _diags = new Diagnostics.Diagnostics(_manager, _cancellationToken, _args.DiagnosticPort); } } } diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 62c0512..4497bdd 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -104,6 +104,7 @@ namespace TimberWinR { if (!string.IsNullOrEmpty(jsonConfFile)) { + LogManager.GetCurrentClassLogger().Info("Reading Configuration From {0}", jsonConfFile); string json = File.ReadAllText(jsonConfFile); return FromString(json, c); diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs index 767c170..d5ef6af 100644 --- a/TimberWinR/Inputs/IISW3CInputListener.cs +++ b/TimberWinR/Inputs/IISW3CInputListener.cs @@ -22,12 +22,13 @@ namespace TimberWinR.Inputs { private int _pollingIntervalInSeconds = 1; private TimberWinR.Parser.IISW3CLog _arguments; - + private long _receivedMessages; public IISW3CInputListener(TimberWinR.Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) : base(cancelToken, "Win32-IISLog") { _arguments = arguments; + _receivedMessages = 0; _pollingIntervalInSeconds = pollingIntervalInSeconds; var task = new Task(IISW3CWatcher, cancelToken); task.Start(); @@ -36,19 +37,37 @@ namespace TimberWinR.Inputs public override void Shutdown() { base.Shutdown(); - } + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("iisw3c", + new JObject( + new JProperty("messages", _receivedMessages), + new JProperty("location", _arguments.Location), + new JProperty("codepage", _arguments.CodePage), + new JProperty("consolidateLogs", _arguments.ConsolidateLogs), + new JProperty("dirTime", _arguments.DirTime), + new JProperty("dQuotes", _arguments.DoubleQuotes), + new JProperty("recurse", _arguments.Recurse), + new JProperty("useDoubleQuotes", _arguments.DoubleQuotes) + ))); + return json; + } + private void IISW3CWatcher() { var oLogQuery = new LogQuery(); - + var iFmt = new IISW3CLogInputFormat() { codepage = _arguments.CodePage, consolidateLogs = _arguments.ConsolidateLogs, dirTime = _arguments.DirTime, dQuotes = _arguments.DoubleQuotes, - iCheckpoint = CheckpointFileName, + iCheckpoint = CheckpointFileName, recurse = _arguments.Recurse, useDoubleQuotes = _arguments.DoubleQuotes }; @@ -67,7 +86,7 @@ namespace TimberWinR.Inputs { var rs = oLogQuery.Execute(query, iFmt); Dictionary colMap = new Dictionary(); - for (int col=0; col public class WindowsEvtInputListener : InputListener - { + { private int _pollingIntervalInSeconds = 1; private TimberWinR.Parser.WindowsEvent _arguments; - + private long _receivedMessages; + public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1) : base(cancelToken, "Win32-Eventlog") - { + { _arguments = arguments; _pollingIntervalInSeconds = pollingIntervalInSeconds; var task = new Task(EventWatcher, cancelToken); @@ -36,7 +37,26 @@ namespace TimberWinR.Inputs public override void Shutdown() { - base.Shutdown(); + base.Shutdown(); + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("windows_events", + new JObject( + new JProperty("messages", _receivedMessages), + new JProperty("binaryFormat", _arguments.BinaryFormat.ToString()), + new JProperty("direction", _arguments.Direction.ToString()), + new JProperty("formatMsg", _arguments.FormatMsg), + new JProperty("fullEventCode", _arguments.FullEventCode), + new JProperty("fullText", _arguments.FullText), + new JProperty("msgErrorMode", _arguments.MsgErrorMode.ToString()), + new JProperty("stringsSep", _arguments.StringsSep), + new JProperty("resolveSIDs", _arguments.ResolveSIDS), + new JProperty("iCheckpoint", CheckpointFileName), + new JProperty("source", _arguments.Source)))); + return json; } private void EventWatcher() @@ -53,12 +73,12 @@ namespace TimberWinR.Inputs formatMsg = _arguments.FormatMsg, fullEventCode = _arguments.FullEventCode, fullText = _arguments.FullText, - msgErrorMode = _arguments.MsgErrorMode.ToString(), + msgErrorMode = _arguments.MsgErrorMode.ToString(), stringsSep = _arguments.StringsSep, resolveSIDs = _arguments.ResolveSIDS, - iCheckpoint = CheckpointFileName, + iCheckpoint = CheckpointFileName, }; - + // Create the query var query = string.Format("SELECT * FROM {0}", _arguments.Source); @@ -69,7 +89,7 @@ namespace TimberWinR.Inputs { try { - var rs = oLogQuery.Execute(query, iFmt); + var rs = oLogQuery.Execute(query, iFmt); // Browse the recordset for (; !rs.atEnd(); rs.moveNext()) { @@ -82,11 +102,12 @@ namespace TimberWinR.Inputs { object v = record.getValue(field.Name); if (field.Name == "Data") - v = ToPrintable(v.ToString()); + v = ToPrintable(v.ToString()); json.Add(new JProperty(field.Name, v)); } - + ProcessJson(json); + _receivedMessages++; } } // Close the recordset @@ -98,11 +119,11 @@ namespace TimberWinR.Inputs LogManager.GetCurrentClassLogger().Error("WindowsEventListener", ex); firstQuery = true; oLogQuery = new LogQuery(); - } + } System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); } Finished(); - } + } } } diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index 83d9e3f..9a0d14e 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -21,7 +21,24 @@ namespace TimberWinR public Configuration Config { get; set; } public List Outputs { get; set; } public List Tcps { get; set; } - public List Listeners { get; set; } + public List Listeners { get; set; } + public DateTime StartedOn { get; set; } + public string JsonConfig { get; set; } + public string LogfileDir { get; set; } + + public int NumConnections { + get { return numConnections; } + } + + public int NumMessages + { + get { return numMessages; } + } + + private static int numConnections; + private static int numMessages; + + public void Shutdown() { LogManager.GetCurrentClassLogger().Info("Shutting Down"); @@ -30,8 +47,22 @@ namespace TimberWinR listener.Shutdown(); } + + public void IncrementMessageCount(int count = 1) + { + Interlocked.Add(ref numMessages, count); + } + public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken) { + StartedOn = DateTime.UtcNow; + + JsonConfig = jsonConfigFile; + LogfileDir = logfileDir; + + numMessages = 0; + numConnections = 0; + Outputs = new List(); Listeners = new List(); @@ -59,15 +90,16 @@ namespace TimberWinR // Is it a directory? if (Directory.Exists(jsonConfigFile)) { - LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", jsonConfigFile); + DirectoryInfo di = new DirectoryInfo(jsonConfigFile); + LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName); Config = Configuration.FromDirectory(jsonConfigFile); } else { - LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", jsonConfigFile); - var fi = new FileInfo(jsonConfigFile); + LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", fi.FullName); + if (!fi.Exists) throw new FileNotFoundException("Missing config file", jsonConfigFile); diff --git a/TimberWinR/Outputs/Elasticsearch.cs b/TimberWinR/Outputs/Elasticsearch.cs index cee2694..6e6d76c 100644 --- a/TimberWinR/Outputs/Elasticsearch.cs +++ b/TimberWinR/Outputs/Elasticsearch.cs @@ -23,10 +23,16 @@ namespace TimberWinR.Outputs private readonly int _timeout; private readonly object _locker = new object(); private readonly List _jsonQueue; + private readonly int _numThreads; + private long _sentMessages; + private long _errorCount; public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken) : base(cancelToken) { + _sentMessages = 0; + _errorCount = 0; + _protocol = eo.Protocol; _timeout = eo.Timeout; _manager = manager; @@ -36,6 +42,8 @@ namespace TimberWinR.Outputs _index = eo.Index; _hostIndex = 0; _jsonQueue = new List(); + _numThreads = eo.NumThreads; + for (int i = 0; i < eo.NumThreads; i++) { var elsThread = new Task(ElasticsearchSender, cancelToken); @@ -43,6 +51,25 @@ namespace TimberWinR.Outputs } } + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("elasticsearch", + new JObject( + new JProperty("host", string.Join(",", _host)), + new JProperty("errors", _errorCount), + new JProperty("sent_messages", _sentMessages), + new JProperty("queued_messages", _jsonQueue.Count), + new JProperty("port", _port), + new JProperty("interval", _interval), + new JProperty("threads", _numThreads), + new JProperty("hosts", + new JArray( + from h in _host + select new JObject( + new JProperty("host", h))))))); + return json; + } // // Pull off messages from the Queue, batch them up and send them all across // @@ -55,6 +82,8 @@ namespace TimberWinR.Outputs { messages = _jsonQueue.Take(1).ToArray(); _jsonQueue.RemoveRange(0, messages.Length); + if (messages.Length > 0) + _manager.IncrementMessageCount(messages.Length); } if (messages.Length > 0) @@ -96,12 +125,18 @@ namespace TimberWinR.Outputs { LogManager.GetCurrentClassLogger() .Error("Failed to send: {0}", response.ErrorMessage); + Interlocked.Increment(ref _errorCount); + } + else + { + _sentMessages++; } }); } catch (Exception error) { LogManager.GetCurrentClassLogger().Error(error); + Interlocked.Increment(ref _errorCount); } } } @@ -110,12 +145,14 @@ namespace TimberWinR.Outputs LogManager.GetCurrentClassLogger() .Fatal("Unable to connect with any Elasticsearch hosts, {0}", String.Join(",", _host)); + Interlocked.Increment(ref _errorCount); } } catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); + Interlocked.Increment(ref _errorCount); } } } diff --git a/TimberWinR/Outputs/OutputSender.cs b/TimberWinR/Outputs/OutputSender.cs index 8445cbb..37ef9d2 100644 --- a/TimberWinR/Outputs/OutputSender.cs +++ b/TimberWinR/Outputs/OutputSender.cs @@ -24,6 +24,7 @@ namespace TimberWinR.Outputs listener.OnMessageRecieved += MessageReceivedHandler; } + public abstract JObject ToJson(); protected abstract void MessageReceivedHandler(JObject jsonMessage); } } diff --git a/TimberWinR/Outputs/Redis.cs b/TimberWinR/Outputs/Redis.cs index 5960eef..415a1e9 100644 --- a/TimberWinR/Outputs/Redis.cs +++ b/TimberWinR/Outputs/Redis.cs @@ -29,6 +29,11 @@ namespace TimberWinR.Outputs private TimberWinR.Manager _manager; private readonly int _batchCount; private readonly int _interval; + private readonly int _numThreads; + + private long _sentMessages; + private long _errorCount; + private long _redisDepth; /// /// Get the next client @@ -61,9 +66,33 @@ namespace TimberWinR.Outputs return null; } + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("redis", + new JObject( + new JProperty("host", string.Join(",", _redisHosts)), + new JProperty("errors", _errorCount), + new JProperty("redis_depth", _redisDepth), + new JProperty("sent_messages", _sentMessages), + new JProperty("queued_messages", _jsonQueue.Count), + new JProperty("port", _port), + new JProperty("interval", _interval), + new JProperty("threads", _numThreads), + new JProperty("batchcount", _batchCount), + new JProperty("index", _logstashIndexName), + new JProperty("hosts", + new JArray( + from h in _redisHosts + select new JObject( + new JProperty("host", h))))))); + return json; + } + public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken) : base(cancelToken) { + _redisDepth = 0; _batchCount = ro.BatchCount; _manager = manager; _redisHostIndex = 0; @@ -73,6 +102,8 @@ namespace TimberWinR.Outputs _timeout = ro.Timeout; _logstashIndexName = ro.Index; _interval = ro.Interval; + _numThreads = ro.NumThreads; + _errorCount = 0; for (int i = 0; i < ro.NumThreads; i++) { @@ -81,6 +112,11 @@ namespace TimberWinR.Outputs } } + public override string ToString() + { + return string.Format("Redis Host: {0} Port: {1}, Threads: {2}, Interval: {3}, BatchCount: {4}", string.Join(",", _redisHosts) , _port, _numThreads, _interval, _batchCount); + } + /// /// Forward on Json message to Redis Logstash queue /// @@ -119,6 +155,8 @@ namespace TimberWinR.Outputs { messages = _jsonQueue.Take(_batchCount).ToArray(); _jsonQueue.RemoveRange(0, messages.Length); + if (messages.Length > 0) + _manager.IncrementMessageCount(messages.Length); } if (messages.Length > 0) @@ -141,11 +179,13 @@ namespace TimberWinR.Outputs { try { - client.RPush(_logstashIndexName, jsonMessage); + _redisDepth = client.RPush(_logstashIndexName, jsonMessage); + _sentMessages++; } catch (SocketException ex) { LogManager.GetCurrentClassLogger().Warn(ex); + Interlocked.Increment(ref _errorCount); } } client.EndPipe(); @@ -153,6 +193,7 @@ namespace TimberWinR.Outputs } else { + Interlocked.Increment(ref _errorCount); LogManager.GetCurrentClassLogger() .Fatal("Unable to connect with any Redis hosts, {0}", String.Join(",", _redisHosts)); @@ -162,6 +203,7 @@ namespace TimberWinR.Outputs catch (Exception ex) { LogManager.GetCurrentClassLogger().Error(ex); + Interlocked.Increment(ref _errorCount); } } } diff --git a/TimberWinR/Outputs/Stdout.cs b/TimberWinR/Outputs/Stdout.cs index 91da0f0..f437a30 100644 --- a/TimberWinR/Outputs/Stdout.cs +++ b/TimberWinR/Outputs/Stdout.cs @@ -14,10 +14,12 @@ namespace TimberWinR.Outputs private readonly int _interval; private readonly object _locker = new object(); private readonly List _jsonQueue; + private long _sentMessages; public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken) : base(cancelToken) { + _sentMessages = 0; _manager = manager; _interval = eo.Interval; _jsonQueue = new List(); @@ -26,6 +28,16 @@ namespace TimberWinR.Outputs elsThread.Start(); } + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("stdout", + new JObject( + new JProperty("sent_messages", _sentMessages)))); + + return json; + } + // // Pull off messages from the Queue, batch them up and send them all across // @@ -37,7 +49,7 @@ namespace TimberWinR.Outputs lock (_locker) { messages = _jsonQueue.Take(1).ToArray(); - _jsonQueue.RemoveRange(0, messages.Length); + _jsonQueue.RemoveRange(0, messages.Length); } if (messages.Length > 0) @@ -47,6 +59,7 @@ namespace TimberWinR.Outputs foreach (JObject obj in messages) { Console.WriteLine(obj.ToString()); + _sentMessages++; } } catch (Exception ex) diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index 5eff87e..38eeda7 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -67,6 +67,7 @@ + @@ -121,9 +122,7 @@ Resources.Designer.cs - - - +