TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed bug with elasticsearch output to disable ping by default, and parametrize ping timeot

Browse Source
This commit is contained in:
Eric Fontana
2015-04-23 08:35:37 -04:00
parent 1f3aaf90fd
commit fd67c271b5
15 changed files with 176 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
TimberWinR.ServiceHost/Program.cs
View File

@@ -28,9 +28,6 @@ namespace TimberWinR.ServiceHost
private static void Main(string[] args)
{
Arguments arguments = new Arguments();
HostFactory.Run(hostConfigurator =>

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.22.0")]
[assembly: AssemblyFileVersion("1.3.22.0")]
[assembly: AssemblyVersion("1.3.23.0")]
[assembly: AssemblyFileVersion("1.3.23.0")]

3
TimberWinR.TestGenerator/CommandLineOptions.cs
View File

@@ -15,6 +15,9 @@ namespace TimberWinR.TestGenerator
[Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")]
public string TimberWinRConfigFile { get; set; }
[Option("start", HelpText = "Start an instance of TimberWinR")]
public bool StartTimberWinR { get; set; }
[Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")]
public string TestDir { get; set; }

72
TimberWinR.TestGenerator/Program.cs
View File

@@ -4,6 +4,7 @@ using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Linq.Expressions;
using System.Net;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
@@ -86,6 +87,7 @@ namespace TimberWinR.TestGenerator
var sw = Stopwatch.StartNew();
// Startup TimberWinR
if (Options.StartTimberWinR)
StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
// Run the Generators
@@ -114,7 +116,16 @@ namespace TimberWinR.TestGenerator
sw.Start();
// Get all the stats
var jsonTimberWinr = ShutdownTimberWinR();
JObject jsonTimberWinr;
if (Options.StartTimberWinR)
jsonTimberWinr = ShutdownTimberWinR();
else
{
jsonTimberWinr = GetDiagnosticsOutput();
if (jsonTimberWinr == null)
return 3;
}
LogManager.GetCurrentClassLogger().Info("Finished Shutdown: " + sw.Elapsed);
sw.Stop();
@@ -131,6 +142,30 @@ namespace TimberWinR.TestGenerator
return 1;
}
public static string GET(string url)
{
try
{
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
HttpWebResponse response = (HttpWebResponse)request.GetResponse();
Stream stream = response.GetResponseStream();
StreamReader reader = new StreamReader(stream);
string data = reader.ReadToEnd();
reader.Close();
stream.Close();
return data;
}
catch (Exception e)
{
LogManager.GetCurrentClassLogger().ErrorException("Error in GET", e);
}
return null;
}
private static void CopySourceFile(string fileName, string outputDir)
{
FileInfo fi = new FileInfo(fileName);
@@ -199,16 +234,16 @@ namespace TimberWinR.TestGenerator
switch (inputProp.Name)
{
case "udp":
if (VerifyConditions(json, new string[] {"udp"}, inputProp, jresult) != 0)
if (VerifyConditions(json, new string[] { "udp" }, inputProp, jresult) != 0)
return 1;
break;
case "tcp":
if (VerifyConditions(json, new string[] {"tcp"}, inputProp, jresult) != 0)
if (VerifyConditions(json, new string[] { "tcp" }, inputProp, jresult) != 0)
return 1;
break;
case "log":
case "taillog":
if (VerifyConditions(json, new string[] {"log", "taillog"}, inputProp, jresult) != 0)
if (VerifyConditions(json, new string[] { "log", "taillog" }, inputProp, jresult) != 0)
return 1;
break;
}
@@ -269,13 +304,32 @@ namespace TimberWinR.TestGenerator
return 0;
}
private static JObject GetDiagnosticsOutput()
{
if (Diagnostics != null)
return Diagnostics.DiagnosticsOutput();
else
{
var jsonDiag = GET("http://localhost:5141");
if (jsonDiag == null)
{
LogManager.GetCurrentClassLogger().Error("TimberWinR diagnostics port not responding.");
return null;
}
return JObject.Parse(jsonDiag);
}
}
// Wait till all output has been transmitted.
private static void WaitForOutputTransmission()
{
bool completed = false;
do
{
var json = Diagnostics.DiagnosticsOutput();
var json = GetDiagnosticsOutput();
if (json == null)
return;
//Console.WriteLine(json.ToString(Formatting.Indented));
@@ -336,6 +390,8 @@ namespace TimberWinR.TestGenerator
}
private static JObject ShutdownTimberWinR()
{
if (_timberWinR != null)
{
// Cancel any/all other threads
_cancellationTokenSource.Cancel();
@@ -345,7 +401,8 @@ namespace TimberWinR.TestGenerator
var json = Diagnostics.DiagnosticsOutput();
LogManager.GetCurrentClassLogger()
.Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB", _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
.Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB",
_avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
LogManager.GetCurrentClassLogger().Info(json.ToString());
@@ -354,6 +411,9 @@ namespace TimberWinR.TestGenerator
return json;
}
return new JObject();
}
static void StartTimberWinR(string configFile, string logLevel, string logFileDir, bool enableLiveMonitor)
{
_timberWinR = new TimberWinR.Manager(configFile, logLevel, logFileDir, enableLiveMonitor, _cancellationTokenSource.Token, false);

8
TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
View File

@@ -108,7 +108,13 @@
<Content Include="results3.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="test3-twconfig.json">
<Content Include="test4-tw.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="results4.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="test4.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>

20
TimberWinR.TestGenerator/results4.json Normal file
View File

@@ -0,0 +1,20 @@
{
"Results": {
"Inputs": [
{
"taillog": {
"test1: message sent count": "[messages] == 7404",
"test2: average cpu": "[avgCpuUsage] <= 30",
"test3: maximum memory": "[maxMemUsage] <= 15"
}
},
{
"tcp": {
"test4: message sent count": "[messages] == 1234",
"test5: average cpu": "[avgCpuUsage] <= 30",
"test6: maximum memory": "[maxMemUsage] <= 15"
}
}
]
}
}

1
TimberWinR.TestGenerator/test1.json
View File

@@ -1,6 +1,7 @@
{
"test": "Test 1",
"arguments": {
"--start": "",
"--testFile": "test1.json",
"--testDir": "test1",
"--timberWinRConfig": "test1-twconfig.json",

1
TimberWinR.TestGenerator/test3.json
View File

@@ -1,6 +1,7 @@
{
"test": "Test 3",
"arguments": {
"--start": "",
"--testFile": "test3.json",
"--testDir": "test3",
"--timberWinRConfig": "test3-tw.json",

0
TimberWinR.TestGenerator/test3-twconfig.json → TimberWinR.TestGenerator/test4-tw.json
View File

11
TimberWinR.TestGenerator/test4.json Normal file
View File

@@ -0,0 +1,11 @@
{
"test": "Test 4",
"arguments": {
"--testFile": "test4.json",
"--testDir": "test4",
"--timberWinRConfig": "test4-tw.json",
"--numMessages": 1234,
"--logLevel": "debug",
"--resultsFile": "results4.json"
}
}

20
TimberWinR/Manager.cs
View File

@@ -118,18 +118,10 @@ namespace TimberWinR
.Info("Database Filename: {0}", LogsFileDatabase.Instance.DatabaseFileName);
try
{
// Is it a directory?
if (Directory.Exists(jsonConfigFile))
{
DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
}
else
{
var fi = new FileInfo(jsonConfigFile);
if (fi.Exists)
{
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", fi.FullName);
if (!fi.Exists)
@@ -138,6 +130,12 @@ namespace TimberWinR
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Config: {0}", fi.FullName);
Config = Configuration.FromFile(jsonConfigFile);
}
else if (Directory.Exists(jsonConfigFile))
{
DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
}
}
catch (JsonSerializationException jse)
{
@@ -163,7 +161,7 @@ namespace TimberWinR
public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
{
// Read the Configuration file
// Read the Configuration file
if (config != null)
{
if (OnConfigurationProcessed != null)

21
TimberWinR/Outputs/Elasticsearch.cs
View File

@@ -42,6 +42,9 @@ namespace TimberWinR.Outputs
private long _errorCount;
private readonly int _maxQueueSize;
private readonly bool _queueOverflowDiscardOldest;
private readonly bool _disablePing;
private readonly int _pingTimeout;
private Parser.ElasticsearchOutputParameters _parameters;
public bool Stop { get; set; }
@@ -61,6 +64,11 @@ namespace TimberWinR.Outputs
var settings = new ConnectionSettings(pool)
.ExposeRawResponse();
if (_disablePing)
settings.DisablePing();
else if (_pingTimeout != 0)
settings.SetPingTimeout(_pingTimeout);
var client = new ElasticClient(settings);
return client;
}
@@ -84,7 +92,8 @@ namespace TimberWinR.Outputs
_numThreads = parameters.NumThreads;
_maxQueueSize = parameters.MaxQueueSize;
_queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
_disablePing = !parameters.EnablePing;
_pingTimeout = parameters.PingTimeout;
for (int i = 0; i < parameters.NumThreads; i++)
{
@@ -99,7 +108,7 @@ namespace TimberWinR.Outputs
new JObject(
new JProperty("host", string.Join(",", _hosts)),
new JProperty("errors", _errorCount),
new JProperty("sentMmessageCount", _sentMessages),
new JProperty("messages", _sentMessages),
new JProperty("queuedMessageCount", _jsonQueue.Count),
new JProperty("port", _port),
new JProperty("flushSize", _flushSize),
@@ -123,6 +132,10 @@ namespace TimberWinR.Outputs
// Force an inital flush
DateTime lastFlushTime = DateTime.MinValue;
LogManager.GetCurrentClassLogger()
.Info("{0}: Elasticsarch Output To {1} Ready", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
@@ -203,6 +216,10 @@ namespace TimberWinR.Outputs
}
}
}
LogManager.GetCurrentClassLogger()
.Info("{0}: Elasticsarch Output To {1} Terminated", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));
}
//

6
TimberWinR/Parser.cs
View File

@@ -520,6 +520,10 @@ namespace TimberWinR.Parser
public int MaxQueueSize { get; set; }
[JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
public bool QueueOverflowDiscardOldest { get; set; }
[JsonProperty(PropertyName = "enable_ping")]
public bool EnablePing { get; set; }
[JsonProperty(PropertyName = "ping_timeout")]
public int PingTimeout { get; set; }
public ElasticsearchOutputParameters()
{
@@ -534,6 +538,8 @@ namespace TimberWinR.Parser
Interval = 1000;
QueueOverflowDiscardOldest = true;
MaxQueueSize = 50000;
EnablePing = false;
PingTimeout = 0;
}
public string GetIndexName(JObject json)

4
TimberWinR/ReleaseNotes.md
View File

@@ -3,6 +3,10 @@
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version / Date
### 1.3.23.0 - 2015-04-23
1. Fixed bug with parsing a single json config file, rather than reading
JSON files from a directory.
2. Diabled elasticsearch outputter ping by default and parameterized the ping capability.
### 1.3.22.0 - 2015-04-14
1. Fixed minor bug with TailFiles and service re-starts not picking up

2
TimberWinR/mdocs/ElasticsearchOutput.md
View File

@@ -16,6 +16,8 @@ The following parameters are allowed when configuring the Redis output.
| *port* | integer | Elasticsearch port number | This port must be open | 9200 |
| *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true |
| *threads* | [string] | Number of Threads | Number of worker threads processing messages | 1 |
| *enable_ping* | bool | If true, pings the server to test for keep alive | | false |
| *ping_timeout* | integer | Default ping timeout when enable_ping is true | milliseconds | 200 |
### Index parameter
If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need.