diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs
index 94ec34b..9b25c48 100644
--- a/TimberWinR.ServiceHost/Program.cs
+++ b/TimberWinR.ServiceHost/Program.cs
@@ -28,9 +28,6 @@ namespace TimberWinR.ServiceHost
private static void Main(string[] args)
{
-
-
-
Arguments arguments = new Arguments();
HostFactory.Run(hostConfigurator =>
diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
index 6162eaa..91f7b14 100644
--- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
+++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.22.0")]
-[assembly: AssemblyFileVersion("1.3.22.0")]
+[assembly: AssemblyVersion("1.3.23.0")]
+[assembly: AssemblyFileVersion("1.3.23.0")]
diff --git a/TimberWinR.TestGenerator/CommandLineOptions.cs b/TimberWinR.TestGenerator/CommandLineOptions.cs
index 81fc217..37ba465 100644
--- a/TimberWinR.TestGenerator/CommandLineOptions.cs
+++ b/TimberWinR.TestGenerator/CommandLineOptions.cs
@@ -15,6 +15,9 @@ namespace TimberWinR.TestGenerator
[Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")]
public string TimberWinRConfigFile { get; set; }
+ [Option("start", HelpText = "Start an instance of TimberWinR")]
+ public bool StartTimberWinR { get; set; }
+
[Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")]
public string TestDir { get; set; }
diff --git a/TimberWinR.TestGenerator/Program.cs b/TimberWinR.TestGenerator/Program.cs
index f6aaa6c..b94c401 100644
--- a/TimberWinR.TestGenerator/Program.cs
+++ b/TimberWinR.TestGenerator/Program.cs
@@ -4,6 +4,7 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
+using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -55,7 +56,7 @@ namespace TimberWinR.TestGenerator
ramCounter.CategoryName = "Memory";
ramCounter.CounterName = "% Committed Bytes In Use";
-
+
Options = new CommandLineOptions();
if (CommandLine.Parser.Default.ParseArguments(args, Options))
@@ -86,7 +87,8 @@ namespace TimberWinR.TestGenerator
var sw = Stopwatch.StartNew();
// Startup TimberWinR
- StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
+ if (Options.StartTimberWinR)
+ StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
// Run the Generators
var arrayOfTasks = RunGenerators(Options);
@@ -114,7 +116,16 @@ namespace TimberWinR.TestGenerator
sw.Start();
// Get all the stats
- var jsonTimberWinr = ShutdownTimberWinR();
+ JObject jsonTimberWinr;
+
+ if (Options.StartTimberWinR)
+ jsonTimberWinr = ShutdownTimberWinR();
+ else
+ {
+ jsonTimberWinr = GetDiagnosticsOutput();
+ if (jsonTimberWinr == null)
+ return 3;
+ }
LogManager.GetCurrentClassLogger().Info("Finished Shutdown: " + sw.Elapsed);
sw.Stop();
@@ -131,6 +142,30 @@ namespace TimberWinR.TestGenerator
return 1;
}
+ public static string GET(string url)
+ {
+ try
+ {
+ HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
+ HttpWebResponse response = (HttpWebResponse)request.GetResponse();
+ Stream stream = response.GetResponseStream();
+ StreamReader reader = new StreamReader(stream);
+
+ string data = reader.ReadToEnd();
+
+ reader.Close();
+ stream.Close();
+
+ return data;
+ }
+ catch (Exception e)
+ {
+ LogManager.GetCurrentClassLogger().ErrorException("Error in GET", e);
+ }
+
+ return null;
+ }
+
private static void CopySourceFile(string fileName, string outputDir)
{
FileInfo fi = new FileInfo(fileName);
@@ -199,16 +234,16 @@ namespace TimberWinR.TestGenerator
switch (inputProp.Name)
{
case "udp":
- if (VerifyConditions(json, new string[] {"udp"}, inputProp, jresult) != 0)
+ if (VerifyConditions(json, new string[] { "udp" }, inputProp, jresult) != 0)
return 1;
break;
case "tcp":
- if (VerifyConditions(json, new string[] {"tcp"}, inputProp, jresult) != 0)
+ if (VerifyConditions(json, new string[] { "tcp" }, inputProp, jresult) != 0)
return 1;
break;
case "log":
case "taillog":
- if (VerifyConditions(json, new string[] {"log", "taillog"}, inputProp, jresult) != 0)
+ if (VerifyConditions(json, new string[] { "log", "taillog" }, inputProp, jresult) != 0)
return 1;
break;
}
@@ -269,13 +304,32 @@ namespace TimberWinR.TestGenerator
return 0;
}
+
+ private static JObject GetDiagnosticsOutput()
+ {
+ if (Diagnostics != null)
+ return Diagnostics.DiagnosticsOutput();
+ else
+ {
+ var jsonDiag = GET("http://localhost:5141");
+ if (jsonDiag == null)
+ {
+ LogManager.GetCurrentClassLogger().Error("TimberWinR diagnostics port not responding.");
+ return null;
+ }
+ return JObject.Parse(jsonDiag);
+ }
+ }
+
// Wait till all output has been transmitted.
private static void WaitForOutputTransmission()
{
bool completed = false;
do
{
- var json = Diagnostics.DiagnosticsOutput();
+ var json = GetDiagnosticsOutput();
+ if (json == null)
+ return;
//Console.WriteLine(json.ToString(Formatting.Indented));
@@ -337,21 +391,27 @@ namespace TimberWinR.TestGenerator
private static JObject ShutdownTimberWinR()
{
- // Cancel any/all other threads
- _cancellationTokenSource.Cancel();
+ if (_timberWinR != null)
+ {
+ // Cancel any/all other threads
+ _cancellationTokenSource.Cancel();
- _timberWinR.Shutdown();
-
- var json = Diagnostics.DiagnosticsOutput();
+ _timberWinR.Shutdown();
- LogManager.GetCurrentClassLogger()
- .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB", _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
+ var json = Diagnostics.DiagnosticsOutput();
- LogManager.GetCurrentClassLogger().Info(json.ToString());
+ LogManager.GetCurrentClassLogger()
+ .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB",
+ _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
- Diagnostics.Shutdown();
+ LogManager.GetCurrentClassLogger().Info(json.ToString());
- return json;
+ Diagnostics.Shutdown();
+
+ return json;
+ }
+
+ return new JObject();
}
static void StartTimberWinR(string configFile, string logLevel, string logFileDir, bool enableLiveMonitor)
@@ -363,7 +423,7 @@ namespace TimberWinR.TestGenerator
}
private static void TimberWinROnOnConfigurationProcessed(Configuration configuration)
- {
+ {
if (!string.IsNullOrEmpty(Options.RedisHost) && configuration.RedisOutputs != null && configuration.RedisOutputs.Count() > 0)
{
foreach (var ro in configuration.RedisOutputs)
diff --git a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
index 5d08e3d..f4d9f23 100644
--- a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
+++ b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
@@ -108,7 +108,13 @@
PreserveNewest
-
+
+ PreserveNewest
+
+
+ PreserveNewest
+
+
PreserveNewest
diff --git a/TimberWinR.TestGenerator/results4.json b/TimberWinR.TestGenerator/results4.json
new file mode 100644
index 0000000..6e91d13
--- /dev/null
+++ b/TimberWinR.TestGenerator/results4.json
@@ -0,0 +1,20 @@
+{
+ "Results": {
+ "Inputs": [
+ {
+ "taillog": {
+ "test1: message sent count": "[messages] == 7404",
+ "test2: average cpu": "[avgCpuUsage] <= 30",
+ "test3: maximum memory": "[maxMemUsage] <= 15"
+ }
+ },
+ {
+ "tcp": {
+ "test4: message sent count": "[messages] == 1234",
+ "test5: average cpu": "[avgCpuUsage] <= 30",
+ "test6: maximum memory": "[maxMemUsage] <= 15"
+ }
+ }
+ ]
+ }
+}
diff --git a/TimberWinR.TestGenerator/test1.json b/TimberWinR.TestGenerator/test1.json
index aa3e4cf..1420bbf 100644
--- a/TimberWinR.TestGenerator/test1.json
+++ b/TimberWinR.TestGenerator/test1.json
@@ -1,6 +1,7 @@
{
"test": "Test 1",
"arguments": {
+ "--start": "",
"--testFile": "test1.json",
"--testDir": "test1",
"--timberWinRConfig": "test1-twconfig.json",
diff --git a/TimberWinR.TestGenerator/test3.json b/TimberWinR.TestGenerator/test3.json
index 28d5faf..fe0d496 100644
--- a/TimberWinR.TestGenerator/test3.json
+++ b/TimberWinR.TestGenerator/test3.json
@@ -1,6 +1,7 @@
{
"test": "Test 3",
"arguments": {
+ "--start": "",
"--testFile": "test3.json",
"--testDir": "test3",
"--timberWinRConfig": "test3-tw.json",
diff --git a/TimberWinR.TestGenerator/test3-twconfig.json b/TimberWinR.TestGenerator/test4-tw.json
similarity index 100%
rename from TimberWinR.TestGenerator/test3-twconfig.json
rename to TimberWinR.TestGenerator/test4-tw.json
diff --git a/TimberWinR.TestGenerator/test4.json b/TimberWinR.TestGenerator/test4.json
new file mode 100644
index 0000000..a4a943a
--- /dev/null
+++ b/TimberWinR.TestGenerator/test4.json
@@ -0,0 +1,11 @@
+{
+ "test": "Test 4",
+ "arguments": {
+ "--testFile": "test4.json",
+ "--testDir": "test4",
+ "--timberWinRConfig": "test4-tw.json",
+ "--numMessages": 1234,
+ "--logLevel": "debug",
+ "--resultsFile": "results4.json"
+ }
+}
diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs
index ee15bdb..cdb68e4 100644
--- a/TimberWinR/Manager.cs
+++ b/TimberWinR/Manager.cs
@@ -66,13 +66,13 @@ namespace TimberWinR
public Manager()
{
- LogsFileDatabase.Manager = this;
+ LogsFileDatabase.Manager = this;
}
public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken, bool processConfiguration = true)
{
- LogsFileDatabase.Manager = this;
-
+ LogsFileDatabase.Manager = this;
+
StartedOn = DateTime.UtcNow;
LiveMonitor = liveMonitor;
@@ -119,17 +119,9 @@ namespace TimberWinR
try
{
- // Is it a directory?
- if (Directory.Exists(jsonConfigFile))
+ var fi = new FileInfo(jsonConfigFile);
+ if (fi.Exists)
{
- DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
- LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
- Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
- }
- else
- {
- var fi = new FileInfo(jsonConfigFile);
-
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", fi.FullName);
if (!fi.Exists)
@@ -138,6 +130,12 @@ namespace TimberWinR
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Config: {0}", fi.FullName);
Config = Configuration.FromFile(jsonConfigFile);
}
+ else if (Directory.Exists(jsonConfigFile))
+ {
+ DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
+ LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
+ Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
+ }
}
catch (JsonSerializationException jse)
{
@@ -145,30 +143,30 @@ namespace TimberWinR
}
catch (Exception ex)
{
- LogManager.GetCurrentClassLogger().Error(ex);
+ LogManager.GetCurrentClassLogger().Error(ex);
}
LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);
if (processConfiguration)
{
- ProcessConfiguration(cancelToken, Config);
+ ProcessConfiguration(cancelToken, Config);
}
}
public void Start(CancellationToken cancelToken)
{
- ProcessConfiguration(cancelToken, Config);
+ ProcessConfiguration(cancelToken, Config);
}
public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
{
-// Read the Configuration file
+ // Read the Configuration file
if (config != null)
{
if (OnConfigurationProcessed != null)
OnConfigurationProcessed(config);
-
+
if (config.RedisOutputs != null)
{
foreach (var ro in config.RedisOutputs)
@@ -272,7 +270,7 @@ namespace TimberWinR
new JObject(
new JProperty("version",
Assembly.GetEntryAssembly().GetName().Version.ToString()),
- //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+ //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
new JProperty("host", computerName),
new JProperty("output", output.Name),
new JProperty("initialized", DateTime.UtcNow)
@@ -280,7 +278,7 @@ namespace TimberWinR
json.Add(new JProperty("type", "Win32-TimberWinR"));
json.Add(new JProperty("host", computerName));
output.Startup(json);
- }
+ }
}
}
diff --git a/TimberWinR/Outputs/Elasticsearch.cs b/TimberWinR/Outputs/Elasticsearch.cs
index 2b21880..4f71cea 100644
--- a/TimberWinR/Outputs/Elasticsearch.cs
+++ b/TimberWinR/Outputs/Elasticsearch.cs
@@ -41,7 +41,10 @@ namespace TimberWinR.Outputs
private long _sentMessages;
private long _errorCount;
private readonly int _maxQueueSize;
- private readonly bool _queueOverflowDiscardOldest;
+ private readonly bool _queueOverflowDiscardOldest;
+ private readonly bool _disablePing;
+ private readonly int _pingTimeout;
+
private Parser.ElasticsearchOutputParameters _parameters;
public bool Stop { get; set; }
@@ -61,6 +64,11 @@ namespace TimberWinR.Outputs
var settings = new ConnectionSettings(pool)
.ExposeRawResponse();
+ if (_disablePing)
+ settings.DisablePing();
+ else if (_pingTimeout != 0)
+ settings.SetPingTimeout(_pingTimeout);
+
var client = new ElasticClient(settings);
return client;
}
@@ -70,7 +78,7 @@ namespace TimberWinR.Outputs
{
_sentMessages = 0;
_errorCount = 0;
-
+
_parameters = parameters;
_flushSize = parameters.FlushSize;
_idleFlushTimeSeconds = parameters.IdleFlushTimeInSeconds;
@@ -84,7 +92,8 @@ namespace TimberWinR.Outputs
_numThreads = parameters.NumThreads;
_maxQueueSize = parameters.MaxQueueSize;
_queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
-
+ _disablePing = !parameters.EnablePing;
+ _pingTimeout = parameters.PingTimeout;
for (int i = 0; i < parameters.NumThreads; i++)
{
@@ -99,7 +108,7 @@ namespace TimberWinR.Outputs
new JObject(
new JProperty("host", string.Join(",", _hosts)),
new JProperty("errors", _errorCount),
- new JProperty("sentMmessageCount", _sentMessages),
+ new JProperty("messages", _sentMessages),
new JProperty("queuedMessageCount", _jsonQueue.Count),
new JProperty("port", _port),
new JProperty("flushSize", _flushSize),
@@ -123,6 +132,10 @@ namespace TimberWinR.Outputs
// Force an inital flush
DateTime lastFlushTime = DateTime.MinValue;
+ LogManager.GetCurrentClassLogger()
+ .Info("{0}: Elasticsarch Output To {1} Ready", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));
+
+
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
@@ -188,7 +201,7 @@ namespace TimberWinR.Outputs
}
GC.Collect();
if (!Stop)
- {
+ {
syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
}
}
@@ -203,6 +216,10 @@ namespace TimberWinR.Outputs
}
}
}
+
+ LogManager.GetCurrentClassLogger()
+ .Info("{0}: Elasticsarch Output To {1} Terminated", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));
+
}
//
diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs
index 535d5b0..b7eebe9 100644
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -519,7 +519,11 @@ namespace TimberWinR.Parser
[JsonProperty(PropertyName = "max_queue_size")]
public int MaxQueueSize { get; set; }
[JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
- public bool QueueOverflowDiscardOldest { get; set; }
+ public bool QueueOverflowDiscardOldest { get; set; }
+ [JsonProperty(PropertyName = "enable_ping")]
+ public bool EnablePing { get; set; }
+ [JsonProperty(PropertyName = "ping_timeout")]
+ public int PingTimeout { get; set; }
public ElasticsearchOutputParameters()
{
@@ -534,6 +538,8 @@ namespace TimberWinR.Parser
Interval = 1000;
QueueOverflowDiscardOldest = true;
MaxQueueSize = 50000;
+ EnablePing = false;
+ PingTimeout = 0;
}
public string GetIndexName(JObject json)
diff --git a/TimberWinR/ReleaseNotes.md b/TimberWinR/ReleaseNotes.md
index a3bdc7f..4957f6a 100644
--- a/TimberWinR/ReleaseNotes.md
+++ b/TimberWinR/ReleaseNotes.md
@@ -3,6 +3,10 @@
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version / Date
+### 1.3.23.0 - 2015-04-23
+1. Fixed bug with parsing a single json config file, rather than reading
+ JSON files from a directory.
+2. Diabled elasticsearch outputter ping by default and parameterized the ping capability.
### 1.3.22.0 - 2015-04-14
1. Fixed minor bug with TailFiles and service re-starts not picking up
diff --git a/TimberWinR/mdocs/ElasticsearchOutput.md b/TimberWinR/mdocs/ElasticsearchOutput.md
index 9e4ac35..7edab04 100644
--- a/TimberWinR/mdocs/ElasticsearchOutput.md
+++ b/TimberWinR/mdocs/ElasticsearchOutput.md
@@ -16,6 +16,8 @@ The following parameters are allowed when configuring the Redis output.
| *port* | integer | Elasticsearch port number | This port must be open | 9200 |
| *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true |
| *threads* | [string] | Number of Threads | Number of worker threads processing messages | 1 |
+| *enable_ping* | bool | If true, pings the server to test for keep alive | | false |
+| *ping_timeout* | integer | Default ping timeout when enable_ping is true | milliseconds | 200 |
### Index parameter
If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need.