TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

redis last output time and refactored diagnostics

Browse Source
This commit is contained in:
Greg Lutz
2015-06-19 11:33:56 -04:00
parent 7cf1230d7a
commit 8694ede85d
8 changed files with 94 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

61
TimberWinR/Diagnostics/Diagnostics.cs
View File

@@ -8,7 +8,7 @@ using System.Threading;
using System.Net;
using System.Net.Sockets;
using System.IO;
using System.Linq.Expressions;
using Newtonsoft.Json.Linq;
using NLog;
@@ -52,29 +52,48 @@ namespace TimberWinR.Diagnostics
public JObject DiagnosticsOutput()
{
JObject json = new JObject(
new JProperty("timberwinr",
new JObject(
new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
new JProperty("messages", Manager.NumMessages),
new JProperty("startedon", Manager.StartedOn),
new JProperty("configfile", Manager.JsonConfig),
new JProperty("logdir", Manager.LogfileDir),
new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
new JProperty("inputs",
new JArray(
from i in Manager.Listeners
select new JObject(i.ToJson()))),
new JProperty("filters",
new JArray(
from f in Manager.Config.Filters
select new JObject(f.ToJson()))),
new JProperty("outputs",
new JArray(
from o in Manager.Outputs
select new JObject(o.ToJson()))))));
new JProperty("timberwinr",
new JObject(
new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
new JProperty("messages", Manager.NumMessages),
new JProperty("startedon", Manager.StartedOn),
new JProperty("configfile", Manager.JsonConfig),
new JProperty("logdir", Manager.LogfileDir),
new JProperty("logginglevel", LogManager.GlobalThreshold.ToString())
)));
AddDiagnosis(json);
return json;
}
protected void AddDiagnosis(JObject wrapper)
{
wrapper.Add("inputs", GetDiagnosisByType("inputs", Manager.Listeners.ToList<IDiagnosable>()));
wrapper.Add("filters", GetDiagnosisByType("inputs", Manager.Config.Filters.ToList<IDiagnosable>()));
wrapper.Add("outputs", GetDiagnosisByType("inputs", Manager.Outputs.ToList<IDiagnosable>()));
}
protected JObject GetDiagnosisByType(String type, List<IDiagnosable> diags)
{
JObject category = new JObject();
foreach(IDiagnosable diag in diags)
{
JArray array = GetTypeArray(diag.GetType().Name.ToString(), category);
array.Add(diag.ToJson());
}
return category;
}
protected JArray GetTypeArray(String name, JObject category)
{
JArray ret = (JArray)category.GetValue(name);
if (ret == null)
{
ret = new JArray();
category.Add(new JProperty(name, ret));
}
return ret;
}
private void DiagnosticCallback(IAsyncResult result)
{
if (web == null)

13
TimberWinR/Diagnostics/IDiagnosable.cs Normal file
View File

@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using Newtonsoft.Json.Linq;
namespace TimberWinR.Diagnostics
{
public interface IDiagnosable
{
JObject ToJson();
}
}

6
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -145,6 +145,12 @@ namespace TimberWinR.Inputs
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
try
{
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (Exception) { }
}
}
}

3
TimberWinR/Inputs/InputListener.cs
View File

@@ -8,10 +8,11 @@ using System.Linq;
using System.Text;
using System.Threading;
using NLog;
using TimberWinR.Diagnostics;
namespace TimberWinR.Inputs
{
public abstract class InputListener
public abstract class InputListener: IDiagnosable
{
public CancellationToken CancelToken { get; set; }
public event Action<JObject> OnMessageRecieved;

31
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -8,6 +8,7 @@ using TimberWinR.Parser;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using EventLogInputFormat = Interop.MSUtil.COMEventLogInputContextClassClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
using System.IO;
namespace TimberWinR.Inputs
{
@@ -97,12 +98,13 @@ namespace TimberWinR.Inputs
// Execute the query
if (!CancelToken.IsCancellationRequested)
{
var oLogQuery = new LogQuery();
try
{
var oLogQuery = new LogQuery();
{
var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
var rsfiles = oLogQuery.Execute(qfiles, iFmt);
for (; !rsfiles.atEnd(); rsfiles.moveNext())
{
var record = rsfiles.getRecord();
@@ -113,7 +115,7 @@ namespace TimberWinR.Inputs
logName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
var lrn = (Int64) qr.getValueEx("MaxRecordNumber");
logFileMaxRecords[logName] = lrn;
}
}
@@ -137,12 +139,13 @@ namespace TimberWinR.Inputs
object v = record.getValue(field.Name);
if (field.Name == "Data")
v = ToPrintable(v.ToString());
if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && field.DataType == typeof (DateTime))
if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") &&
field.DataType == typeof (DateTime))
v = ((DateTime) v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
}
var lrn = (Int64)record.getValueEx("RecordNumber");
var lrn = (Int64) record.getValueEx("RecordNumber");
logFileMaxRecords[fileName] = lrn;
ProcessJson(json);
@@ -163,6 +166,24 @@ namespace TimberWinR.Inputs
{
LogManager.GetCurrentClassLogger().Error(ex);
}
finally
{
try
{
oLogQuery = null;
// Sleep
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (OperationCanceledException)
{
}
catch (Exception ex1)
{
LogManager.GetCurrentClassLogger().Warn(ex1);
}
}
}
}
Finished();

3
TimberWinR/Outputs/OutputSender.cs
View File

@@ -4,11 +4,12 @@ using System.Linq;
using System.Text;
using System.Threading;
using Newtonsoft.Json.Linq;
using TimberWinR.Diagnostics;
using TimberWinR.Inputs;
namespace TimberWinR.Outputs
{
public abstract class OutputSender
public abstract class OutputSender : IDiagnosable
{
public CancellationToken CancelToken { get; private set; }
private List<InputListener> _inputs;

3
TimberWinR/Outputs/Redis.cs
View File

@@ -132,6 +132,7 @@ namespace TimberWinR.Outputs
private long _errorCount;
private long _redisDepth;
private DateTime? _lastErrorTimeUTC;
private DateTime? _lastSentTimeUTC;
private readonly int _maxQueueSize;
private readonly bool _queueOverflowDiscardOldest;
private BatchCounter _batchCounter;
@@ -180,6 +181,7 @@ namespace TimberWinR.Outputs
new JProperty("host", string.Join(",", _redisHosts)),
new JProperty("errors", _errorCount),
new JProperty("lastErrorTimeUTC", _lastErrorTimeUTC),
new JProperty("lastSentTimeUTC", _lastSentTimeUTC),
new JProperty("redisQueueDepth", _redisDepth),
new JProperty("sentMessageCount", _sentMessages),
new JProperty("queuedMessageCount", _jsonQueue.Count),
@@ -348,6 +350,7 @@ namespace TimberWinR.Outputs
Interlocked.Add(ref _sentMessages, messages.Length);
client.EndPipe();
sentSuccessfully = true;
_lastSentTimeUTC = DateTime.UtcNow;
if (messages.Length > 0)
_manager.IncrementMessageCount(messages.Length);
}

3
TimberWinR/Parser.cs
View File

@@ -15,6 +15,7 @@ using NLog;
using NLog.Config;
using TimberWinR.Outputs;
using System.CodeDom.Compiler;
using TimberWinR.Diagnostics;
namespace TimberWinR.Parser
{
@@ -26,7 +27,7 @@ namespace TimberWinR.Parser
}
public abstract class LogstashFilter : IValidateSchema
public abstract class LogstashFilter : IValidateSchema, IDiagnosable
{
public abstract bool Apply(JObject json);