TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed shutdown slowness

Browse Source
This commit is contained in:
Eric Fontana
2014-12-19 13:29:28 -05:00
parent 1eaa5a4657
commit 0da192167a
9 changed files with 419 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.16.0")]
[assembly: AssemblyFileVersion("1.3.16.0")]
[assembly: AssemblyVersion("1.3.17.0")]
[assembly: AssemblyFileVersion("1.3.17.0")]

30
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -18,7 +18,7 @@ namespace TimberWinR.Inputs
private readonly int _pollingIntervalInSeconds;
private readonly Parser.IISW3CLog _arguments;
private long _receivedMessages;
public bool Stop { get; set; }
private IisW3CRowReader rowReader;
public IISW3CInputListener(Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
@@ -38,6 +38,7 @@ namespace TimberWinR.Inputs
public override void Shutdown()
{
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
base.Shutdown();
}
@@ -80,8 +81,13 @@ namespace TimberWinR.Inputs
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!CancelToken.IsCancellationRequested)
while (!Stop)
{
// Execute the query
if (!CancelToken.IsCancellationRequested)
{
try
{
@@ -95,10 +101,11 @@ namespace TimberWinR.Inputs
string fileName = record.getValue("LogFilename") as string;
if (!logFileMaxRecords.ContainsKey(fileName))
{
var qcount = string.Format("SELECT max(LogRow) as MaxRecordNumber FROM {0}", fileName);
var qcount = string.Format("SELECT max(LogRow) as MaxRecordNumber FROM {0}",
fileName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
var lrn = (Int64) qr.getValueEx("MaxRecordNumber");
logFileMaxRecords[fileName] = lrn;
}
}
@@ -106,7 +113,8 @@ namespace TimberWinR.Inputs
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName, lastRecordNumber);
var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName,
lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
rowReader.ReadColumnMap(rs);
@@ -118,7 +126,7 @@ namespace TimberWinR.Inputs
var json = rowReader.ReadToJson(record);
ProcessJson(json);
_receivedMessages++;
var lrn = (Int64)record.getValueEx("LogRow");
var lrn = (Int64) record.getValueEx("LogRow");
logFileMaxRecords[fileName] = lrn;
record = null;
json = null;
@@ -127,13 +135,19 @@ namespace TimberWinR.Inputs
rs.close();
GC.Collect();
}
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
}
}
Finished();

5
TimberWinR/Inputs/InputListener.cs
View File

@@ -58,14 +58,17 @@ namespace TimberWinR.Inputs
public void Finished()
{
LogManager.GetCurrentClassLogger().Info("Signaling Event Shutdown {0}", InputType);
FinishedEvent.Set();
LogManager.GetCurrentClassLogger().Info("Finished Shutdown {0}", InputType);
LogManager.GetCurrentClassLogger().Info("Finished signaling Shutdown {0}", InputType);
}
public virtual void Shutdown()
{
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
FinishedEvent.WaitOne();
LogManager.GetCurrentClassLogger().Info("Finished Wait For {0}", InputType);
try
{
if (File.Exists(CheckpointFileName))

9
TimberWinR/Inputs/LogsListener.cs
View File

@@ -31,14 +31,14 @@ namespace TimberWinR.Inputs
private Dictionary<string, DateTime> _logFileCreationTimes;
private Dictionary<string, DateTime> _logFileSampleTimes;
private Dictionary<string, long> _logFileSizes;
private CancellationToken _cancelToken;
public bool Stop { get; set; }
public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-FileLog")
{
Stop = false;
_cancelToken = cancelToken;
_logFileMaxRecords = new Dictionary<string, Int64>();
_logFileCreationTimes = new Dictionary<string, DateTime>();
_logFileSampleTimes = new Dictionary<string, DateTime>();
@@ -108,7 +108,7 @@ namespace TimberWinR.Inputs
while (!Stop)
{
var oLogQuery = new LogQuery();
if (!_cancelToken.IsCancellationRequested)
if (!CancelToken.IsCancellationRequested)
{
try
{
@@ -216,7 +216,8 @@ namespace TimberWinR.Inputs
GC.Collect();
}
// Sleep
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), _cancelToken);
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (FileNotFoundException fnfex)
{

27
TimberWinR/Inputs/W3CInputListener.cs
View File

@@ -25,6 +25,7 @@ namespace TimberWinR.Inputs
private readonly int _pollingIntervalInSeconds;
private readonly TimberWinR.Parser.W3CLog _arguments;
private long _receivedMessages;
public bool Stop { get; set; }
public W3CInputListener(TimberWinR.Parser.W3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
: base(cancelToken, "Win32-W3CLog")
@@ -41,6 +42,7 @@ namespace TimberWinR.Inputs
public override void Shutdown()
{
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
base.Shutdown();
}
@@ -78,9 +80,13 @@ namespace TimberWinR.Inputs
};
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!CancelToken.IsCancellationRequested)
while (!Stop)
{
// Execute the query
if (!CancelToken.IsCancellationRequested)
{
try
{
@@ -94,7 +100,8 @@ namespace TimberWinR.Inputs
string fileName = record.getValue("LogFilename") as string;
if (!logFileMaxRecords.ContainsKey(fileName))
{
var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}", fileName);
var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}",
fileName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
@@ -106,7 +113,9 @@ namespace TimberWinR.Inputs
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName, lastRecordNumber);
var query = string.Format(
"SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName,
lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
var colMap = new Dictionary<string, int>();
for (int col = 0; col < rs.getColumnCount(); col++)
@@ -141,13 +150,19 @@ namespace TimberWinR.Inputs
// Close the recordset
rs.close();
}
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
}
}
Finished();

44
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -27,6 +27,7 @@ namespace TimberWinR.Inputs
private TimberWinR.Parser.WindowsEvent _arguments;
private long _receivedMessages;
private List<Thread> _tasks { get; set; }
public bool Stop { get; set; }
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-Eventlog")
@@ -46,11 +47,8 @@ namespace TimberWinR.Inputs
public override void Shutdown()
{
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
foreach(Thread t in _tasks)
{
t.Abort();
}
base.Shutdown();
}
@@ -99,13 +97,16 @@ namespace TimberWinR.Inputs
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!CancelToken.IsCancellationRequested)
while (!Stop)
{
// Execute the query
if (!CancelToken.IsCancellationRequested)
{
try
{
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
oLogQuery = new LogQuery();
var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
@@ -116,7 +117,8 @@ namespace TimberWinR.Inputs
string logName = record.getValue("EventLog") as string;
if (!logFileMaxRecords.ContainsKey(logName))
{
var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName);
var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}",
logName);
var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
@@ -128,7 +130,8 @@ namespace TimberWinR.Inputs
foreach (string fileName in logFileMaxRecords.Keys.ToList())
{
var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber);
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location,
lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt);
// Browse the recordset
@@ -159,36 +162,21 @@ namespace TimberWinR.Inputs
rs = null;
GC.Collect();
}
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (System.Threading.ThreadAbortException tex)
catch (OperationCanceledException oce)
{
Thread.ResetAbort();
break;
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
try
{
Thread.CurrentThread.Priority = ThreadPriority.Normal;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
catch (System.Threading.ThreadAbortException tex)
{
Thread.ResetAbort();
break;
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
}
Finished();
}
}
}
}

31
TimberWinR/Outputs/Elasticsearch.cs
View File

@@ -28,6 +28,7 @@ namespace TimberWinR.Outputs
private long _sentMessages;
private long _errorCount;
private Parser.ElasticsearchOutput eo;
public bool Stop { get; set; }
public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken)
: base(cancelToken, "Elasticsearch")
@@ -76,7 +77,14 @@ namespace TimberWinR.Outputs
//
private void ElasticsearchSender()
{
while (!CancelToken.IsCancellationRequested)
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{
JObject[] messages;
lock (_locker)
@@ -106,7 +114,9 @@ namespace TimberWinR.Outputs
{
var typeName = this.eo.GetTypeName(json);
var indexName = this.eo.GetIndexName(json);
var req = new RestRequest(string.Format("/{0}/{1}/", indexName, typeName), Method.POST);
var req =
new RestRequest(string.Format("/{0}/{1}/", indexName, typeName),
Method.POST);
req.AddParameter("text/json", json.ToString(), ParameterType.RequestBody);
@@ -154,10 +164,21 @@ namespace TimberWinR.Outputs
}
}
GC.Collect();
System.Threading.Thread.Sleep(_interval);
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_interval), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception)
{
throw;
}
}
}
}
}
private RestClient getClient()
{
if (_hostIndex >= _host.Length)
@@ -168,7 +189,7 @@ namespace TimberWinR.Outputs
{
try
{
string url = string.Format("{0}://{1}:{2}", _protocol.Replace(":",""), _host[_hostIndex], _port);
string url = string.Format("{0}://{1}:{2}", _protocol.Replace(":", ""), _host[_hostIndex], _port);
var client = new RestClient(url);
client.Timeout = _timeout;

26
TimberWinR/Outputs/Redis.cs
View File

@@ -38,6 +38,8 @@ namespace TimberWinR.Outputs
private int _maxQueueSize;
private bool _queueOverflowDiscardOldest;
public bool Stop { get; set; }
/// <summary>
/// Get the next client
/// </summary>
@@ -185,7 +187,14 @@ namespace TimberWinR.Outputs
//
private void RedisSender()
{
while (!CancelToken.IsCancellationRequested)
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{
string[] messages;
lock (_locker)
@@ -245,7 +254,20 @@ namespace TimberWinR.Outputs
}
}
GC.Collect();
System.Threading.Thread.Sleep(_interval);
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_interval), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception ex)
{
throw;
}
}
}
}
}
}

23
TimberWinR/Outputs/Stdout.cs
View File

@@ -15,6 +15,7 @@ namespace TimberWinR.Outputs
private readonly object _locker = new object();
private readonly List<JObject> _jsonQueue;
private long _sentMessages;
public bool Stop { get; set; }
public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken)
: base(cancelToken, "Stdout")
@@ -43,7 +44,14 @@ namespace TimberWinR.Outputs
//
private void StdoutSender()
{
while (!CancelToken.IsCancellationRequested)
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{
JObject[] messages;
lock (_locker)
@@ -67,7 +75,18 @@ namespace TimberWinR.Outputs
LogManager.GetCurrentClassLogger().Error(ex);
}
}
System.Threading.Thread.Sleep(_interval);
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_interval), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception)
{
}
}
}
}
}