TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fixed high CPU usage problem for non-existent log files.

Browse Source
This commit is contained in:
Eric Fontana
2015-02-27 07:37:09 -05:00
parent e5237e8e59
commit 57b29a5425
9 changed files with 183 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

25
TimberWinR.ServiceHost/Program.cs
View File

@@ -38,7 +38,8 @@ namespace TimberWinR.ServiceHost
serviceConfigurator.WhenStarted(myService => myService.Start());
serviceConfigurator.WhenStopped(myService => myService.Stop());
});
hostConfigurator.AddCommandLineDefinition("liveMonitor", c => arguments.LiveMonitor = bool.Parse(c.ToString()));
hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c);
hostConfigurator.AddCommandLineDefinition("logLevel", c => arguments.LogLevel = c);
hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c);
@@ -60,6 +61,7 @@ namespace TimberWinR.ServiceHost
AddServiceParameter("-configFile", arguments.ConfigFile);
AddServiceParameter("-logLevel", arguments.LogLevel);
AddServiceParameter("-logDir", arguments.LogfileDir);
AddServiceParameter("-liveMonitor", arguments.LiveMonitor);
if (arguments.DiagnosticPort > 0)
AddServiceParameter("-diagnosticPort", arguments.DiagnosticPort);
}
@@ -68,8 +70,7 @@ namespace TimberWinR.ServiceHost
}
private static void AddServiceParameter(string paramName, string value)
{
{
string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0} ", paramName)))
@@ -80,8 +81,7 @@ namespace TimberWinR.ServiceHost
}
private static void AddServiceParameter(string paramName, int value)
{
{
string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
@@ -91,6 +91,16 @@ namespace TimberWinR.ServiceHost
}
}
private static void AddServiceParameter(string paramName, bool value)
{
string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
{
currentValue += string.Format(" {0} \"{1}\"", paramName, value.ToString());
Registry.SetValue(KeyPath, KeyName, currentValue);
}
}
}
internal class Arguments
@@ -99,9 +109,10 @@ namespace TimberWinR.ServiceHost
public string LogLevel { get; set; }
public string LogfileDir { get; set; }
public int DiagnosticPort { get; set; }
public bool LiveMonitor { get; set; }
public Arguments()
{
LiveMonitor = false;
DiagnosticPort = 5141;
ConfigFile = "default.json";
LogLevel = "Info";
@@ -147,7 +158,7 @@ namespace TimberWinR.ServiceHost
/// </summary>
private void RunService()
{
_manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _cancellationToken);
_manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _args.LiveMonitor, _cancellationToken);
if (_args.DiagnosticPort > 0)
_diags = new Diagnostics.Diagnostics(_manager, _cancellationToken, _args.DiagnosticPort);
}

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.20.0")]
[assembly: AssemblyFileVersion("1.3.20.0")]
[assembly: AssemblyVersion("1.3.19.0")]
[assembly: AssemblyFileVersion("1.3.19.0")]

1
TimberWinR.sln
View File

@@ -21,6 +21,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
chocolateyInstall.ps1.template = chocolateyInstall.ps1.template
chocolateyUninstall.ps1.guid = chocolateyUninstall.ps1.guid
chocolateyUninstall.ps1.template = chocolateyUninstall.ps1.template
chocolateyUninstall.ps1.template.orig = chocolateyUninstall.ps1.template.orig
LICENSE.txt = LICENSE.txt
Performance1.psess = Performance1.psess
README.md = README.md

95
TimberWinR/Configuration.cs
View File

@@ -4,6 +4,8 @@ using System.Data.Odbc;
using System.Diagnostics;
using System.Linq;
using System.Reflection;
using System.Threading;
using System.Threading.Tasks;
using System.Text;
using System.Xml;
using System.Xml.Linq;
@@ -16,6 +18,7 @@ using Newtonsoft.Json.Linq;
using TimberWinR.Inputs;
using TimberWinR.Filters;
using NLog;
using TimberWinR.Parser;
using Topshelf.Configurators;
@@ -25,7 +28,12 @@ using WindowsEvent = TimberWinR.Parser.WindowsEvent;
namespace TimberWinR
{
public class Configuration
{
{
private CancellationToken _cancelToken;
private bool _stopService;
private FileSystemWatcher _dirWatcher;
private Manager _manager;
private List<WindowsEvent> _events = new List<WindowsEvent>();
public IEnumerable<WindowsEvent> Events
{
@@ -102,11 +110,88 @@ namespace TimberWinR
get { return _filters; }
}
private void MonitorDirectory(string directoryToWatch, CancellationToken cancelToken, Manager manager)
{
_manager = manager;
_cancelToken = cancelToken;
if (_dirWatcher == null)
{
_dirWatcher = new FileSystemWatcher();
_dirWatcher.Path = directoryToWatch;
_dirWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName;
// Only watch json files.
_dirWatcher.Filter = "*.json";
_dirWatcher.Created += DirWatcherOnCreated;
_dirWatcher.Changed += DirWatcherOnChanged;
_dirWatcher.Renamed += DirWatcherOnRenamed;
_dirWatcher.EnableRaisingEvents = true;
}
}
public static Configuration FromDirectory(string jsonDirectory)
private void DirWatcherOnRenamed(object sender, RenamedEventArgs e)
{
// The Renamed file could be a different name from .json
FileInfo fi = new FileInfo(e.FullPath);
if (fi.Extension == ".json")
{
LogManager.GetCurrentClassLogger().Info("File: OnRenamed " + e.FullPath + " " + e.ChangeType);
ProcessNewJson(e.FullPath);
}
}
private void DirWatcherOnCreated(object sender, FileSystemEventArgs e)
{
FileInfo fi = new FileInfo(e.FullPath);
if (fi.Extension == ".json")
{
LogManager.GetCurrentClassLogger().Info("File: OnCreated " + e.FullPath + " " + e.ChangeType);
ProcessNewJson(e.FullPath);
}
}
private void DirWatcherOnChanged(object sender, FileSystemEventArgs e)
{
FileInfo fi = new FileInfo(e.FullPath);
if (fi.Extension == ".json")
{
// Specify what is done when a file is changed, created, or deleted.
LogManager.GetCurrentClassLogger()
.Info("File: OnChanged " + e.ChangeType.ToString() + " " + e.FullPath + " " + e.ChangeType);
ProcessNewJson(e.FullPath);
}
}
private void ProcessNewJson(string fileName)
{
try
{
Configuration c = new Configuration();
var config = Configuration.FromFile(fileName, c);
_manager.ProcessConfiguration(_cancelToken, config);
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
}
private void ShutdownDirectoryMonitor()
{
_stopService = true;
_dirWatcher.EnableRaisingEvents = false;
LogManager.GetCurrentClassLogger().Info("Stopping Directory Monitor");
}
private void DirectoryWatcher(string directoryToWatch)
{
LogManager.GetCurrentClassLogger().Info("Starting Directory Monitor {0}", directoryToWatch);
}
public static Configuration FromDirectory(string jsonDirectory, CancellationToken cancelToken, Manager manager)
{
Configuration c = null;
foreach (string jsonConfFile in Directory.GetFiles(jsonDirectory, "*.json"))
{
if (!string.IsNullOrEmpty(jsonConfFile))
@@ -115,6 +200,10 @@ namespace TimberWinR
}
}
// Startup Directory Monitor
if (manager.LiveMonitor)
c.MonitorDirectory(jsonDirectory, cancelToken, manager);
return c;
}

14
TimberWinR/Inputs/LogsListener.cs
View File

@@ -317,19 +317,16 @@ namespace TimberWinR.Inputs
rs.close();
rs = null;
GC.Collect();
}
// Sleep
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
}
catch (FileNotFoundException fnfex)
{
string fn = fnfex.FileName;
if (!_fnfmap.ContainsKey(fn))
LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
else
_fnfmap[fn] = fn;
LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
_fnfmap[fn] = fn;
}
catch (OperationCanceledException oce)
{
@@ -342,6 +339,9 @@ namespace TimberWinR.Inputs
finally
{
oLogQuery = null;
// Sleep
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
}
}

20
TimberWinR/Inputs/TailFileListener.cs
View File

@@ -227,8 +227,8 @@ namespace TimberWinR.Inputs
else
{
ProcessJson(json);
Interlocked.Increment(ref _receivedMessages);
}
Interlocked.Increment(ref _receivedMessages);
}
lineOffset += line.Length;
// Console.WriteLine("File: {0}:{1}: {2}", fileName, reader.BaseStream.Position, line);
}
@@ -240,6 +240,8 @@ namespace TimberWinR.Inputs
// threads.
private void TailFileWatcher(string fileToWatch)
{
Dictionary<string, string> _fnfmap = new Dictionary<string, string>();
using (var syncHandle = new ManualResetEventSlim())
{
// Execute the query
@@ -284,16 +286,15 @@ namespace TimberWinR.Inputs
}
LogsFileDatabase.Update(dbe);
}
// LogManager.GetCurrentClassLogger().Info("Finished Scan...Sleeping");
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
}
catch (FileNotFoundException fnfex)
{
string fn = fnfex.FileName;
LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
string fn = fnfex.FileName;
if (!_fnfmap.ContainsKey(fn))
LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
_fnfmap[fn] = fn;
}
catch (OperationCanceledException oce)
{
@@ -305,7 +306,8 @@ namespace TimberWinR.Inputs
}
finally
{
if (!Stop)
syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
}
}
}

70
TimberWinR/Manager.cs
View File

@@ -26,6 +26,8 @@ namespace TimberWinR
public List<TcpInputListener> Tcps { get; set; }
public List<TcpInputListener> Udps { get; set; }
public List<InputListener> Listeners { get; set; }
public bool LiveMonitor { get; set; }
public DateTime StartedOn { get; set; }
public string JsonConfig { get; set; }
public string LogfileDir { get; set; }
@@ -65,11 +67,12 @@ namespace TimberWinR
LogsFileDatabase.Manager = this;
}
public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken)
public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken)
{
LogsFileDatabase.Manager = this;
StartedOn = DateTime.UtcNow;
LiveMonitor = liveMonitor;
var vfi = new FileInfo(jsonConfigFile);
@@ -110,7 +113,6 @@ namespace TimberWinR
LogManager.GetCurrentClassLogger()
.Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
try
{
// Is it a directory?
@@ -118,7 +120,7 @@ namespace TimberWinR
{
DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
Config = Configuration.FromDirectory(jsonConfigFile);
Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
}
else
{
@@ -144,36 +146,40 @@ namespace TimberWinR
LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);
// Read the Configuration file
if (Config != null)
ProcessConfiguration(cancelToken, Config);
}
public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
{
// Read the Configuration file
if (config != null)
{
if (Config.RedisOutputs != null)
if (config.RedisOutputs != null)
{
foreach (var ro in Config.RedisOutputs)
foreach (var ro in config.RedisOutputs)
{
var redis = new RedisOutput(this, ro, cancelToken);
Outputs.Add(redis);
}
}
if (Config.ElasticsearchOutputs != null)
if (config.ElasticsearchOutputs != null)
{
foreach (var ro in Config.ElasticsearchOutputs)
foreach (var ro in config.ElasticsearchOutputs)
{
var els = new ElasticsearchOutput(this, ro, cancelToken);
Outputs.Add(els);
}
}
if (Config.StdoutOutputs != null)
if (config.StdoutOutputs != null)
{
foreach (var ro in Config.StdoutOutputs)
foreach (var ro in config.StdoutOutputs)
{
var stdout = new StdoutOutput(this, ro, cancelToken);
Outputs.Add(stdout);
}
}
foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C)
foreach (Parser.IISW3CLog iisw3cConfig in config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
Listeners.Add(elistner);
@@ -181,7 +187,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (Parser.W3CLog iisw3cConfig in Config.W3C)
foreach (Parser.W3CLog iisw3cConfig in config.W3C)
{
var elistner = new W3CInputListener(iisw3cConfig, cancelToken);
Listeners.Add(elistner);
@@ -189,7 +195,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (Parser.WindowsEvent eventConfig in Config.Events)
foreach (Parser.WindowsEvent eventConfig in config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);
Listeners.Add(elistner);
@@ -197,7 +203,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (var logConfig in Config.Logs)
foreach (var logConfig in config.Logs)
{
var elistner = new LogsListener(logConfig, cancelToken);
Listeners.Add(elistner);
@@ -205,7 +211,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (var logConfig in Config.TailFiles)
foreach (var logConfig in config.TailFiles)
{
var elistner = new TailFileListener(logConfig, cancelToken);
Listeners.Add(elistner);
@@ -213,7 +219,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (var tcp in Config.Tcps)
foreach (var tcp in config.Tcps)
{
var elistner = new TcpInputListener(cancelToken, tcp.Port);
Listeners.Add(elistner);
@@ -221,7 +227,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (var udp in Config.Udps)
foreach (var udp in config.Udps)
{
var elistner = new UdpInputListener(cancelToken, udp.Port);
Listeners.Add(elistner);
@@ -229,7 +235,7 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (var stdin in Config.Stdins)
foreach (var stdin in config.Stdins)
{
var elistner = new StdinListener(stdin, cancelToken);
Listeners.Add(elistner);
@@ -238,28 +244,28 @@ namespace TimberWinR
}
var computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
foreach (var output in Outputs)
{
var name = Assembly.GetExecutingAssembly().GetName();
JObject json = new JObject(
new JProperty("TimberWinR",
new JObject(
new JProperty("version", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
new JProperty("host", computerName),
new JProperty("output", output.Name),
new JProperty("initialized", DateTime.UtcNow)
)));
new JProperty("TimberWinR",
new JObject(
new JProperty("version",
GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
new JProperty("host", computerName),
new JProperty("output", output.Name),
new JProperty("initialized", DateTime.UtcNow)
)));
json.Add(new JProperty("type", "Win32-TimberWinR"));
json.Add(new JProperty("host", computerName));
output.Startup(json);
}
}
}

12
TimberWinR/ReleaseNotes.md
View File

@@ -3,14 +3,14 @@
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version History
### 1.3.20.0 - 01/29/2015
1. Added new TailFiles input type which uses a native implementation (more-efficient) than using LogParser's Log
2. Updated Udp input listner to use UTF8 Encoding rather than ASCII
3. Reduced noisy complaint about missing log files for Logs listener
### 1.3.19.0 - 01/12/2015
### 1.3.19.0 - 02/26/2015
1. Added support for Multiline codecs for Stdin and Logs listeners, closes issue [#23](https://github.com/Cimpress-MCP/TimberWinR/issues/23)
2. Added new TailFiles input type which uses a native implementation (more-efficient) than using LogParser's Log
3. Updated Udp input listner to use UTF8 Encoding rather than ASCII
4. Reduced noisy complaint about missing log files for Logs listener
5. Fixed bug when tailing non-existent log files which resulted in high cpu-usage.
6. Added feature to watch the configuration directory
### 1.3.18.0 - 12/22/2014

8
chocolateyUninstall.ps1.template.orig Normal file
View File

@@ -0,0 +1,8 @@
$packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
$installerType = 'msi' #only one of these: exe, msi, msu
$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
$silentArgs = '${PROJECTGUID} /quiet'
$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes