TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed shutdown slowness

Browse Source
This commit is contained in:
Eric Fontana
2014-12-19 13:29:28 -05:00
parent 1eaa5a4657
commit 0da192167a
9 changed files with 419 additions and 336 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.16.0")] [assembly: AssemblyVersion("1.3.17.0")]
[assembly: AssemblyFileVersion("1.3.16.0")] [assembly: AssemblyFileVersion("1.3.17.0")]

26
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -18,7 +18,7 @@ namespace TimberWinR.Inputs
private readonly int _pollingIntervalInSeconds; private readonly int _pollingIntervalInSeconds;
private readonly Parser.IISW3CLog _arguments; private readonly Parser.IISW3CLog _arguments;
private long _receivedMessages; private long _receivedMessages;
public bool Stop { get; set; }
private IisW3CRowReader rowReader; private IisW3CRowReader rowReader;
public IISW3CInputListener(Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5) public IISW3CInputListener(Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
@@ -38,6 +38,7 @@ namespace TimberWinR.Inputs
public override void Shutdown() public override void Shutdown()
{ {
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType); LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
base.Shutdown(); base.Shutdown();
} }
@@ -80,8 +81,13 @@ namespace TimberWinR.Inputs
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>(); Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query // Execute the query
while (!CancelToken.IsCancellationRequested) while (!Stop)
{
// Execute the query
if (!CancelToken.IsCancellationRequested)
{ {
try try
{ {
@@ -95,7 +101,8 @@ namespace TimberWinR.Inputs
string fileName = record.getValue("LogFilename") as string; string fileName = record.getValue("LogFilename") as string;
if (!logFileMaxRecords.ContainsKey(fileName)) if (!logFileMaxRecords.ContainsKey(fileName))
{ {
var qcount = string.Format("SELECT max(LogRow) as MaxRecordNumber FROM {0}", fileName); var qcount = string.Format("SELECT max(LogRow) as MaxRecordNumber FROM {0}",
fileName);
var rcount = oLogQuery.Execute(qcount, iFmt); var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord(); var qr = rcount.getRecord();
var lrn = (Int64) qr.getValueEx("MaxRecordNumber"); var lrn = (Int64) qr.getValueEx("MaxRecordNumber");
@@ -106,7 +113,8 @@ namespace TimberWinR.Inputs
foreach (string fileName in logFileMaxRecords.Keys.ToList()) foreach (string fileName in logFileMaxRecords.Keys.ToList())
{ {
var lastRecordNumber = logFileMaxRecords[fileName]; var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName, lastRecordNumber); var query = string.Format("SELECT * FROM '{0}' Where LogRow > {1}", fileName,
lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt); var rs = oLogQuery.Execute(query, iFmt);
rowReader.ReadColumnMap(rs); rowReader.ReadColumnMap(rs);
@@ -127,13 +135,19 @@ namespace TimberWinR.Inputs
rs.close(); rs.close();
GC.Collect(); GC.Collect();
} }
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
} }
catch (Exception ex) catch (Exception ex)
{ {
LogManager.GetCurrentClassLogger().Error(ex); LogManager.GetCurrentClassLogger().Error(ex);
} }
}
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); }
} }
Finished(); Finished();

5
TimberWinR/Inputs/InputListener.cs
View File

@@ -58,14 +58,17 @@ namespace TimberWinR.Inputs
public void Finished() public void Finished()
{ {
LogManager.GetCurrentClassLogger().Info("Signaling Event Shutdown {0}", InputType);
FinishedEvent.Set(); FinishedEvent.Set();
LogManager.GetCurrentClassLogger().Info("Finished Shutdown {0}", InputType); LogManager.GetCurrentClassLogger().Info("Finished signaling Shutdown {0}", InputType);
} }
public virtual void Shutdown() public virtual void Shutdown()
{ {
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType); LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
FinishedEvent.WaitOne(); FinishedEvent.WaitOne();
LogManager.GetCurrentClassLogger().Info("Finished Wait For {0}", InputType);
try try
{ {
if (File.Exists(CheckpointFileName)) if (File.Exists(CheckpointFileName))

9
TimberWinR/Inputs/LogsListener.cs
View File

@@ -31,14 +31,14 @@ namespace TimberWinR.Inputs
private Dictionary<string, DateTime> _logFileCreationTimes; private Dictionary<string, DateTime> _logFileCreationTimes;
private Dictionary<string, DateTime> _logFileSampleTimes; private Dictionary<string, DateTime> _logFileSampleTimes;
private Dictionary<string, long> _logFileSizes; private Dictionary<string, long> _logFileSizes;
private CancellationToken _cancelToken;
public bool Stop { get; set; } public bool Stop { get; set; }
public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken) public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-FileLog") : base(cancelToken, "Win32-FileLog")
{ {
Stop = false; Stop = false;
_cancelToken = cancelToken;
_logFileMaxRecords = new Dictionary<string, Int64>(); _logFileMaxRecords = new Dictionary<string, Int64>();
_logFileCreationTimes = new Dictionary<string, DateTime>(); _logFileCreationTimes = new Dictionary<string, DateTime>();
_logFileSampleTimes = new Dictionary<string, DateTime>(); _logFileSampleTimes = new Dictionary<string, DateTime>();
@@ -108,7 +108,7 @@ namespace TimberWinR.Inputs
while (!Stop) while (!Stop)
{ {
var oLogQuery = new LogQuery(); var oLogQuery = new LogQuery();
if (!_cancelToken.IsCancellationRequested) if (!CancelToken.IsCancellationRequested)
{ {
try try
{ {
@@ -216,7 +216,8 @@ namespace TimberWinR.Inputs
GC.Collect(); GC.Collect();
} }
// Sleep // Sleep
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), _cancelToken); if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
} }
catch (FileNotFoundException fnfex) catch (FileNotFoundException fnfex)
{ {

27
TimberWinR/Inputs/W3CInputListener.cs
View File

@@ -25,6 +25,7 @@ namespace TimberWinR.Inputs
private readonly int _pollingIntervalInSeconds; private readonly int _pollingIntervalInSeconds;
private readonly TimberWinR.Parser.W3CLog _arguments; private readonly TimberWinR.Parser.W3CLog _arguments;
private long _receivedMessages; private long _receivedMessages;
public bool Stop { get; set; }
public W3CInputListener(TimberWinR.Parser.W3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5) public W3CInputListener(TimberWinR.Parser.W3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
: base(cancelToken, "Win32-W3CLog") : base(cancelToken, "Win32-W3CLog")
@@ -41,6 +42,7 @@ namespace TimberWinR.Inputs
public override void Shutdown() public override void Shutdown()
{ {
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType); LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
base.Shutdown(); base.Shutdown();
} }
@@ -78,9 +80,13 @@ namespace TimberWinR.Inputs
}; };
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>(); Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query // Execute the query
while (!CancelToken.IsCancellationRequested) while (!Stop)
{
// Execute the query
if (!CancelToken.IsCancellationRequested)
{ {
try try
{ {
@@ -94,7 +100,8 @@ namespace TimberWinR.Inputs
string fileName = record.getValue("LogFilename") as string; string fileName = record.getValue("LogFilename") as string;
if (!logFileMaxRecords.ContainsKey(fileName)) if (!logFileMaxRecords.ContainsKey(fileName))
{ {
var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}", fileName); var qcount = string.Format("SELECT max(RowNumber) as MaxRecordNumber FROM {0}",
fileName);
var rcount = oLogQuery.Execute(qcount, iFmt); var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord(); var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
@@ -106,7 +113,9 @@ namespace TimberWinR.Inputs
foreach (string fileName in logFileMaxRecords.Keys.ToList()) foreach (string fileName in logFileMaxRecords.Keys.ToList())
{ {
var lastRecordNumber = logFileMaxRecords[fileName]; var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName, lastRecordNumber); var query = string.Format(
"SELECT * FROM '{0}' Where RowNumber > {1} order by RowNumber", fileName,
lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt); var rs = oLogQuery.Execute(query, iFmt);
var colMap = new Dictionary<string, int>(); var colMap = new Dictionary<string, int>();
for (int col = 0; col < rs.getColumnCount(); col++) for (int col = 0; col < rs.getColumnCount(); col++)
@@ -141,13 +150,19 @@ namespace TimberWinR.Inputs
// Close the recordset // Close the recordset
rs.close(); rs.close();
} }
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
} }
catch (Exception ex) catch (Exception ex)
{ {
LogManager.GetCurrentClassLogger().Error(ex); LogManager.GetCurrentClassLogger().Error(ex);
} }
}
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000); }
} }
Finished(); Finished();

44
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -27,6 +27,7 @@ namespace TimberWinR.Inputs
private TimberWinR.Parser.WindowsEvent _arguments; private TimberWinR.Parser.WindowsEvent _arguments;
private long _receivedMessages; private long _receivedMessages;
private List<Thread> _tasks { get; set; } private List<Thread> _tasks { get; set; }
public bool Stop { get; set; }
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken) public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-Eventlog") : base(cancelToken, "Win32-Eventlog")
@@ -46,11 +47,8 @@ namespace TimberWinR.Inputs
public override void Shutdown() public override void Shutdown()
{ {
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType); LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
foreach(Thread t in _tasks)
{
t.Abort();
}
base.Shutdown(); base.Shutdown();
} }
@@ -99,13 +97,16 @@ namespace TimberWinR.Inputs
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>(); Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query // Execute the query
while (!CancelToken.IsCancellationRequested) while (!Stop)
{
// Execute the query
if (!CancelToken.IsCancellationRequested)
{ {
try try
{ {
Thread.CurrentThread.Priority = ThreadPriority.BelowNormal;
oLogQuery = new LogQuery(); oLogQuery = new LogQuery();
var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location); var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
@@ -116,7 +117,8 @@ namespace TimberWinR.Inputs
string logName = record.getValue("EventLog") as string; string logName = record.getValue("EventLog") as string;
if (!logFileMaxRecords.ContainsKey(logName)) if (!logFileMaxRecords.ContainsKey(logName))
{ {
var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}", logName); var qcount = string.Format("SELECT max(RecordNumber) as MaxRecordNumber FROM {0}",
logName);
var rcount = oLogQuery.Execute(qcount, iFmt); var rcount = oLogQuery.Execute(qcount, iFmt);
var qr = rcount.getRecord(); var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber"); var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
@@ -128,7 +130,8 @@ namespace TimberWinR.Inputs
foreach (string fileName in logFileMaxRecords.Keys.ToList()) foreach (string fileName in logFileMaxRecords.Keys.ToList())
{ {
var lastRecordNumber = logFileMaxRecords[fileName]; var lastRecordNumber = logFileMaxRecords[fileName];
var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location, lastRecordNumber); var query = string.Format("SELECT * FROM {0} where RecordNumber > {1}", location,
lastRecordNumber);
var rs = oLogQuery.Execute(query, iFmt); var rs = oLogQuery.Execute(query, iFmt);
// Browse the recordset // Browse the recordset
@@ -159,36 +162,21 @@ namespace TimberWinR.Inputs
rs = null; rs = null;
GC.Collect(); GC.Collect();
} }
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
} }
catch (System.Threading.ThreadAbortException tex) catch (OperationCanceledException oce)
{ {
Thread.ResetAbort();
break; break;
} }
catch (Exception ex) catch (Exception ex)
{ {
LogManager.GetCurrentClassLogger().Error(ex); LogManager.GetCurrentClassLogger().Error(ex);
} }
try
{
Thread.CurrentThread.Priority = ThreadPriority.Normal;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
} }
catch (System.Threading.ThreadAbortException tex)
{
Thread.ResetAbort();
break;
} }
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
}
Finished(); Finished();
} }
} }
} }
}

29
TimberWinR/Outputs/Elasticsearch.cs
View File

@@ -28,6 +28,7 @@ namespace TimberWinR.Outputs
private long _sentMessages; private long _sentMessages;
private long _errorCount; private long _errorCount;
private Parser.ElasticsearchOutput eo; private Parser.ElasticsearchOutput eo;
public bool Stop { get; set; }
public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken) public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken)
: base(cancelToken, "Elasticsearch") : base(cancelToken, "Elasticsearch")
@@ -76,7 +77,14 @@ namespace TimberWinR.Outputs
// //
private void ElasticsearchSender() private void ElasticsearchSender()
{ {
while (!CancelToken.IsCancellationRequested) using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{ {
JObject[] messages; JObject[] messages;
lock (_locker) lock (_locker)
@@ -106,7 +114,9 @@ namespace TimberWinR.Outputs
{ {
var typeName = this.eo.GetTypeName(json); var typeName = this.eo.GetTypeName(json);
var indexName = this.eo.GetIndexName(json); var indexName = this.eo.GetIndexName(json);
var req = new RestRequest(string.Format("/{0}/{1}/", indexName, typeName), Method.POST); var req =
new RestRequest(string.Format("/{0}/{1}/", indexName, typeName),
Method.POST);
req.AddParameter("text/json", json.ToString(), ParameterType.RequestBody); req.AddParameter("text/json", json.ToString(), ParameterType.RequestBody);
@@ -154,10 +164,21 @@ namespace TimberWinR.Outputs
} }
} }
GC.Collect(); GC.Collect();
System.Threading.Thread.Sleep(_interval); if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_interval), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception)
{
throw;
}
}
}
} }
} }
private RestClient getClient() private RestClient getClient()
{ {
if (_hostIndex >= _host.Length) if (_hostIndex >= _host.Length)

26
TimberWinR/Outputs/Redis.cs
View File

@@ -38,6 +38,8 @@ namespace TimberWinR.Outputs
private int _maxQueueSize; private int _maxQueueSize;
private bool _queueOverflowDiscardOldest; private bool _queueOverflowDiscardOldest;
public bool Stop { get; set; }
/// <summary> /// <summary>
/// Get the next client /// Get the next client
/// </summary> /// </summary>
@@ -185,7 +187,14 @@ namespace TimberWinR.Outputs
// //
private void RedisSender() private void RedisSender()
{ {
while (!CancelToken.IsCancellationRequested) using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{ {
string[] messages; string[] messages;
lock (_locker) lock (_locker)
@@ -245,7 +254,20 @@ namespace TimberWinR.Outputs
} }
} }
GC.Collect(); GC.Collect();
System.Threading.Thread.Sleep(_interval); if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_interval), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception ex)
{
throw;
}
}
}
} }
} }
} }

23
TimberWinR/Outputs/Stdout.cs
View File

@@ -15,6 +15,7 @@ namespace TimberWinR.Outputs
private readonly object _locker = new object(); private readonly object _locker = new object();
private readonly List<JObject> _jsonQueue; private readonly List<JObject> _jsonQueue;
private long _sentMessages; private long _sentMessages;
public bool Stop { get; set; }
public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken) public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken)
: base(cancelToken, "Stdout") : base(cancelToken, "Stdout")
@@ -43,7 +44,14 @@ namespace TimberWinR.Outputs
// //
private void StdoutSender() private void StdoutSender()
{ {
while (!CancelToken.IsCancellationRequested) using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{ {
JObject[] messages; JObject[] messages;
lock (_locker) lock (_locker)
@@ -67,7 +75,18 @@ namespace TimberWinR.Outputs
LogManager.GetCurrentClassLogger().Error(ex); LogManager.GetCurrentClassLogger().Error(ex);
} }
} }
System.Threading.Thread.Sleep(_interval); if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_interval), CancelToken);
}
catch (OperationCanceledException oce)
{
break;
}
catch (Exception)
{
}
}
}
} }
} }