Fixed bug with elasticsearch output to disable ping by default, and parametrize ping timeot

TimberWinR.ServiceHost/Program.cs

@@ -28,9 +28,6 @@ namespace TimberWinR.ServiceHost
         
         private static void Main(string[] args)
         {
-
-           
-
             Arguments arguments = new Arguments();
            
             HostFactory.Run(hostConfigurator =>

TimberWinR.ServiceHost/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.22.0")]
+[assembly: AssemblyVersion("1.3.23.0")]
-[assembly: AssemblyFileVersion("1.3.22.0")]
+[assembly: AssemblyFileVersion("1.3.23.0")]

TimberWinR.TestGenerator/CommandLineOptions.cs

@@ -15,6 +15,9 @@ namespace TimberWinR.TestGenerator
         [Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")]
         public string TimberWinRConfigFile { get; set; }
 
+        [Option("start", HelpText = "Start an instance of TimberWinR")]
+        public bool StartTimberWinR { get; set; }
+
         [Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")]
         public string TestDir { get; set; }
 

TimberWinR.TestGenerator/Program.cs

@@ -4,6 +4,7 @@ using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Linq.Expressions;
+using System.Net;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -55,7 +56,7 @@ namespace TimberWinR.TestGenerator
 
             ramCounter.CategoryName = "Memory";
             ramCounter.CounterName = "% Committed Bytes In Use";
-           
+
             Options = new CommandLineOptions();
 
             if (CommandLine.Parser.Default.ParseArguments(args, Options))
@@ -86,7 +87,8 @@ namespace TimberWinR.TestGenerator
                 var sw = Stopwatch.StartNew();
 
                 // Startup TimberWinR
-                StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
+                if (Options.StartTimberWinR)
+                    StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
 
                 // Run the Generators
                 var arrayOfTasks = RunGenerators(Options);
@@ -114,7 +116,16 @@ namespace TimberWinR.TestGenerator
                 sw.Start();
 
                 // Get all the stats
-                var jsonTimberWinr = ShutdownTimberWinR();
+                JObject jsonTimberWinr;
+
+                if (Options.StartTimberWinR)
+                    jsonTimberWinr = ShutdownTimberWinR();
+                else
+                {
+                    jsonTimberWinr = GetDiagnosticsOutput();
+                    if (jsonTimberWinr == null)
+                        return 3;
+                }
 
                 LogManager.GetCurrentClassLogger().Info("Finished Shutdown: " + sw.Elapsed);
                 sw.Stop();
@@ -131,6 +142,30 @@ namespace TimberWinR.TestGenerator
             return 1;
         }
 
+        public static string GET(string url)
+        {
+            try
+            {
+                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
+                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
+                Stream stream = response.GetResponseStream();
+                StreamReader reader = new StreamReader(stream);
+
+                string data = reader.ReadToEnd();
+
+                reader.Close();
+                stream.Close();
+
+                return data;
+            }
+            catch (Exception e)
+            {
+                LogManager.GetCurrentClassLogger().ErrorException("Error in GET", e);
+            }
+
+            return null;
+        }
+
         private static void CopySourceFile(string fileName, string outputDir)
         {
             FileInfo fi = new FileInfo(fileName);
@@ -199,16 +234,16 @@ namespace TimberWinR.TestGenerator
                 switch (inputProp.Name)
                 {
                     case "udp":
-                        if (VerifyConditions(json, new string[] {"udp"}, inputProp, jresult) != 0)
+                        if (VerifyConditions(json, new string[] { "udp" }, inputProp, jresult) != 0)
                             return 1;
                         break;
                     case "tcp":
-                        if (VerifyConditions(json, new string[] {"tcp"}, inputProp, jresult) != 0)
+                        if (VerifyConditions(json, new string[] { "tcp" }, inputProp, jresult) != 0)
                             return 1;
                         break;
                     case "log":
                     case "taillog":
-                        if (VerifyConditions(json, new string[] {"log", "taillog"}, inputProp, jresult) != 0)
+                        if (VerifyConditions(json, new string[] { "log", "taillog" }, inputProp, jresult) != 0)
                             return 1;
                         break;
                 }
@@ -269,13 +304,32 @@ namespace TimberWinR.TestGenerator
             return 0;
         }
 
+
+        private static JObject GetDiagnosticsOutput()
+        {
+            if (Diagnostics != null)
+                return Diagnostics.DiagnosticsOutput();
+            else
+            {
+                var jsonDiag = GET("http://localhost:5141");
+                if (jsonDiag == null)
+                {
+                    LogManager.GetCurrentClassLogger().Error("TimberWinR diagnostics port not responding.");
+                    return null;
+                }
+                return JObject.Parse(jsonDiag);
+            }        
+        }
+
         // Wait till all output has been transmitted.
         private static void WaitForOutputTransmission()
         {
             bool completed = false;
             do
             {
-                var json = Diagnostics.DiagnosticsOutput();
+                var json = GetDiagnosticsOutput();
+                if (json == null)
+                    return;
 
                 //Console.WriteLine(json.ToString(Formatting.Indented));
 
@@ -337,21 +391,27 @@ namespace TimberWinR.TestGenerator
 
         private static JObject ShutdownTimberWinR()
         {
-            // Cancel any/all other threads
+            if (_timberWinR != null)
-            _cancellationTokenSource.Cancel();
+            {
+                // Cancel any/all other threads
+                _cancellationTokenSource.Cancel();
 
-            _timberWinR.Shutdown();
+                _timberWinR.Shutdown();
-          
-            var json = Diagnostics.DiagnosticsOutput();
 
-            LogManager.GetCurrentClassLogger()
+                var json = Diagnostics.DiagnosticsOutput();
-                .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB", _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
 
-            LogManager.GetCurrentClassLogger().Info(json.ToString());
+                LogManager.GetCurrentClassLogger()
+                    .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB",
+                        _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
 
-            Diagnostics.Shutdown();
+                LogManager.GetCurrentClassLogger().Info(json.ToString());
 
-            return json;
+                Diagnostics.Shutdown();
+
+                return json;
+            }
+
+            return new JObject();
         }
 
         static void StartTimberWinR(string configFile, string logLevel, string logFileDir, bool enableLiveMonitor)
@@ -363,7 +423,7 @@ namespace TimberWinR.TestGenerator
         }
 
         private static void TimberWinROnOnConfigurationProcessed(Configuration configuration)
-        {         
+        {
             if (!string.IsNullOrEmpty(Options.RedisHost) && configuration.RedisOutputs != null && configuration.RedisOutputs.Count() > 0)
             {
                 foreach (var ro in configuration.RedisOutputs)

TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj

@@ -108,7 +108,13 @@
     <Content Include="results3.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="test3-twconfig.json">
+    <Content Include="test4-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results4.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test4.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>

TimberWinR.TestGenerator/results4.json

@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            },
+            {
+                "tcp": {
+                    "test4: message sent count": "[messages] == 1234",
+                    "test5: average cpu": "[avgCpuUsage] <= 30",
+                    "test6: maximum memory": "[maxMemUsage] <= 15"
+                }
+            }
+        ]
+    }
+}

TimberWinR.TestGenerator/test1.json

@@ -1,6 +1,7 @@
 {
     "test": "Test 1",
     "arguments": {
+        "--start":  "",
         "--testFile": "test1.json",
         "--testDir": "test1",
         "--timberWinRConfig": "test1-twconfig.json",

TimberWinR.TestGenerator/test3.json

@@ -1,6 +1,7 @@
 {
     "test": "Test 3",
     "arguments": {
+        "--start":  "",
         "--testFile": "test3.json",
         "--testDir": "test3",
         "--timberWinRConfig": "test3-tw.json",

TimberWinR.TestGenerator/test4.json

@@ -0,0 +1,11 @@
+{
+    "test": "Test 4",
+    "arguments": {
+        "--testFile": "test4.json",                
+        "--testDir": "test4",
+        "--timberWinRConfig": "test4-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",         
+        "--resultsFile": "results4.json"
+    }
+}

TimberWinR/Manager.cs

@@ -66,13 +66,13 @@ namespace TimberWinR
 
         public Manager()
         {
-            LogsFileDatabase.Manager = this;                   
+            LogsFileDatabase.Manager = this;
         }
 
         public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken, bool processConfiguration = true)
         {
-            LogsFileDatabase.Manager = this;           
+            LogsFileDatabase.Manager = this;
-  
+
             StartedOn = DateTime.UtcNow;
             LiveMonitor = liveMonitor;
 
@@ -119,17 +119,9 @@ namespace TimberWinR
 
             try
             {
-                // Is it a directory?
+                var fi = new FileInfo(jsonConfigFile);
-                if (Directory.Exists(jsonConfigFile))
+                if (fi.Exists)
                 {
-                    DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
-                    LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
-                    Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
-                }
-                else
-                {
-                    var fi = new FileInfo(jsonConfigFile);
-
                     LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", fi.FullName);
 
                     if (!fi.Exists)
@@ -138,6 +130,12 @@ namespace TimberWinR
                     LogManager.GetCurrentClassLogger().Info("Initialized, Reading Config: {0}", fi.FullName);
                     Config = Configuration.FromFile(jsonConfigFile);
                 }
+                else if (Directory.Exists(jsonConfigFile))
+                {
+                    DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
+                    LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
+                    Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
+                }
             }
             catch (JsonSerializationException jse)
             {
@@ -145,30 +143,30 @@ namespace TimberWinR
             }
             catch (Exception ex)
             {
-                LogManager.GetCurrentClassLogger().Error(ex);               
+                LogManager.GetCurrentClassLogger().Error(ex);
             }
             LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
             LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);
 
             if (processConfiguration)
             {
-                ProcessConfiguration(cancelToken, Config);               
+                ProcessConfiguration(cancelToken, Config);
             }
         }
 
         public void Start(CancellationToken cancelToken)
         {
-            ProcessConfiguration(cancelToken, Config);          
+            ProcessConfiguration(cancelToken, Config);
         }
 
         public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
         {
-// Read the Configuration file
+            // Read the Configuration file
             if (config != null)
             {
                 if (OnConfigurationProcessed != null)
                     OnConfigurationProcessed(config);
-  
+
                 if (config.RedisOutputs != null)
                 {
                     foreach (var ro in config.RedisOutputs)
@@ -272,7 +270,7 @@ namespace TimberWinR
                             new JObject(
                                 new JProperty("version",
                                     Assembly.GetEntryAssembly().GetName().Version.ToString()),
-                                    //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+                        //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
                                 new JProperty("host", computerName),
                                 new JProperty("output", output.Name),
                                 new JProperty("initialized", DateTime.UtcNow)
@@ -280,7 +278,7 @@ namespace TimberWinR
                     json.Add(new JProperty("type", "Win32-TimberWinR"));
                     json.Add(new JProperty("host", computerName));
                     output.Startup(json);
-                }              
+                }
             }
         }
 

TimberWinR/Outputs/Elasticsearch.cs

@@ -41,7 +41,10 @@ namespace TimberWinR.Outputs
         private long _sentMessages;
         private long _errorCount;
         private readonly int _maxQueueSize;
-        private readonly bool _queueOverflowDiscardOldest;    
+        private readonly bool _queueOverflowDiscardOldest;
+        private readonly bool _disablePing;
+        private readonly int _pingTimeout;
+
         private Parser.ElasticsearchOutputParameters _parameters;
         public bool Stop { get; set; }
       
@@ -61,6 +64,11 @@ namespace TimberWinR.Outputs
             var settings = new ConnectionSettings(pool)
                 .ExposeRawResponse();
 
+            if (_disablePing)
+                settings.DisablePing();
+            else if (_pingTimeout != 0)
+                settings.SetPingTimeout(_pingTimeout);
+
             var client = new ElasticClient(settings);
             return client;
         }
@@ -70,7 +78,7 @@ namespace TimberWinR.Outputs
         {
             _sentMessages = 0;
             _errorCount = 0;
-
+           
             _parameters = parameters;
             _flushSize = parameters.FlushSize;
             _idleFlushTimeSeconds = parameters.IdleFlushTimeInSeconds;
@@ -84,7 +92,8 @@ namespace TimberWinR.Outputs
             _numThreads = parameters.NumThreads;
             _maxQueueSize = parameters.MaxQueueSize;
             _queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
-      
+            _disablePing = !parameters.EnablePing;
+            _pingTimeout = parameters.PingTimeout;
 
             for (int i = 0; i < parameters.NumThreads; i++)
             {
@@ -99,7 +108,7 @@ namespace TimberWinR.Outputs
                     new JObject(
                         new JProperty("host", string.Join(",", _hosts)),
                         new JProperty("errors", _errorCount),
-                        new JProperty("sentMmessageCount", _sentMessages),
+                        new JProperty("messages", _sentMessages),
                         new JProperty("queuedMessageCount", _jsonQueue.Count),
                         new JProperty("port", _port),
                         new JProperty("flushSize", _flushSize),                                       
@@ -123,6 +132,10 @@ namespace TimberWinR.Outputs
             // Force an inital flush
             DateTime lastFlushTime = DateTime.MinValue;
 
+            LogManager.GetCurrentClassLogger()
+                            .Info("{0}: Elasticsarch Output To {1} Ready", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));
+         
+
             using (var syncHandle = new ManualResetEventSlim())
             {               
                 // Execute the query
@@ -188,7 +201,7 @@ namespace TimberWinR.Outputs
                             }
                             GC.Collect();
                             if (!Stop)
-                            {
+                            {                               
                                 syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                             }
                         }
@@ -203,6 +216,10 @@ namespace TimberWinR.Outputs
                     }
                 }
             }
+
+            LogManager.GetCurrentClassLogger()
+                          .Info("{0}: Elasticsarch Output To {1} Terminated", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));         
+
         }
 
         //

TimberWinR/Parser.cs

@@ -519,7 +519,11 @@ namespace TimberWinR.Parser
         [JsonProperty(PropertyName = "max_queue_size")]
         public int MaxQueueSize { get; set; }
         [JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
-        public bool QueueOverflowDiscardOldest { get; set; }
+        public bool QueueOverflowDiscardOldest { get; set; }        
+        [JsonProperty(PropertyName = "enable_ping")]
+        public bool EnablePing { get; set; }      
+        [JsonProperty(PropertyName = "ping_timeout")]
+        public int PingTimeout { get; set; }
 
         public ElasticsearchOutputParameters()
         {
@@ -534,6 +538,8 @@ namespace TimberWinR.Parser
             Interval = 1000;
             QueueOverflowDiscardOldest = true;
             MaxQueueSize = 50000;
+            EnablePing = false;
+            PingTimeout = 0;
         }
 
         public string GetIndexName(JObject json)

TimberWinR/ReleaseNotes.md

@@ -3,6 +3,10 @@
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
 
 Version / Date
+### 1.3.23.0 - 2015-04-23
+1. Fixed bug with parsing a single json config file, rather than reading
+   JSON files from a directory.
+2. Diabled elasticsearch outputter ping by default and parameterized the ping capability.
 
 ### 1.3.22.0 - 2015-04-14
 1. Fixed minor bug with TailFiles and service re-starts not picking up

TimberWinR/mdocs/ElasticsearchOutput.md

@@ -16,6 +16,8 @@ The following parameters are allowed when configuring the Redis output.
 | *port*                          | integer  | Elasticsearch port number                                   | This port must be open  | 9200  |
 | *queue_overflow_discard_oldest* | bool  | If true, discard oldest messages when max_queue_size reached otherwise discard newest |  | true |
 | *threads*                       | [string]    | Number of Threads                         | Number of worker threads processing messages | 1 |
+| *enable_ping* | bool  | If true, pings the server to test for keep alive |  | false |
+| *ping_timeout* | integer  | Default ping timeout when enable_ping is true | milliseconds | 200 |
 
 ### Index parameter
 If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need.