1 Commits

Author SHA1 Message Date
Eric Fontana
736d0bd68a Fixed documentation to match desired new default of 200 2015-04-09 12:50:24 -04:00
17 changed files with 99 additions and 194 deletions

1
.gitignore vendored
View File

@@ -155,4 +155,3 @@ $RECYCLE.BIN/
# Mac desktop service store files
.DS_Store
packages
*.msi

View File

@@ -167,7 +167,7 @@ following options:
TimberWinR.ServiceHost.exe -configFile:myconfig.json -logLevel:Debug
```
## Automatic Installation via Chocolatey (embedded)
## Automatic Installation via Chocolatey
[TimbeWinR Chocolatey](https://chocolatey.org/packages/TimberWinR)

View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.21.0")]
[assembly: AssemblyFileVersion("1.3.21.0")]
[assembly: AssemblyVersion("1.3.20.0")]
[assembly: AssemblyFileVersion("1.3.20.0")]

View File

@@ -199,18 +199,11 @@ namespace TimberWinR.TestGenerator
switch (inputProp.Name)
{
case "udp":
if (VerifyConditions(json, new string[] {"udp"}, inputProp, jresult) != 0)
return 1;
break;
case "tcp":
if (VerifyConditions(json, new string[] {"tcp"}, inputProp, jresult) != 0)
return 1;
break;
return VerifyConditions(json, new string[] { "udp" }, inputProp, jresult);
case "log":
case "taillog":
if (VerifyConditions(json, new string[] {"log", "taillog"}, inputProp, jresult) != 0)
return 1;
break;
return VerifyConditions(json, new string[] { "log", "taillog" }, inputProp, jresult);
}
}
@@ -363,7 +356,9 @@ namespace TimberWinR.TestGenerator
}
private static void TimberWinROnOnConfigurationProcessed(Configuration configuration)
{
{
Console.WriteLine("Processed Config: {0}", configuration.GetHashCode());
if (!string.IsNullOrEmpty(Options.RedisHost) && configuration.RedisOutputs != null && configuration.RedisOutputs.Count() > 0)
{
foreach (var ro in configuration.RedisOutputs)

View File

@@ -99,15 +99,6 @@
<Content Include="results2.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="test3.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="test3-tw.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="results3.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

View File

@@ -1,20 +0,0 @@
{
"Results": {
"Inputs": [
{
"taillog": {
"test1: message sent count": "[messages] == 7404",
"test2: average cpu": "[avgCpuUsage] <= 30",
"test3: maximum memory": "[maxMemUsage] <= 15"
}
},
{
"tcp": {
"test4: message sent count": "[messages] == 1234",
"test5: average cpu": "[avgCpuUsage] <= 30",
"test6: maximum memory": "[maxMemUsage] <= 15"
}
}
]
}
}

View File

@@ -1,45 +0,0 @@
{
"TimberWinR": {
"Inputs": {
"Tcp": [
{
"_comment": "Output from NLog",
"port": 5140
}
],
"Logs": [
{
"interval": 5,
"logSource": "log files",
"location": "*.jlog",
"recurse": -1
}
]
},
"Filters": [
{
"grok": {
"condition": "\"[EventTypeName]\" == \"Information Event\"",
"match": [
"Text",
""
],
"drop": "true"
}
}
],
"Outputs": {
"Redis": [
{
"_comment": "Change the host to your Redis instance",
"port": 6379,
"batch_count": 500,
"threads": 2,
"host": [
"tstlexiceapp006.vistaprint.svc"
]
}
]
}
}
}

View File

@@ -1,14 +0,0 @@
{
"test": "Test 3",
"arguments": {
"--testFile": "test3.json",
"--testDir": "test3",
"--timberWinRConfig": "test3-tw.json",
"--numMessages": 1234,
"--logLevel": "debug",
"--tcp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
"--resultsFile": "results3.json"
}
}

View File

@@ -16,9 +16,7 @@ namespace TimberWinR.Inputs
private Thread _listenThreadV4;
private Thread _listenThreadV6;
private readonly int _port;
private long _receivedMessages;
private long _errorCount;
public override JObject ToJson()
{
@@ -26,9 +24,9 @@ namespace TimberWinR.Inputs
new JProperty("tcp",
new JObject(
new JProperty("port", _port),
new JProperty("errors", _errorCount),
new JProperty("messages", _receivedMessages)
)));
return json;
}
@@ -70,6 +68,7 @@ namespace TimberWinR.Inputs
listener.Start();
while (!CancelToken.IsCancellationRequested)
{
try
@@ -80,7 +79,7 @@ namespace TimberWinR.Inputs
// Wait for a client, spin up a thread.
var clientThread = new Thread(new ParameterizedThreadStart(HandleNewClient));
clientThread.Start(client);
}
}
catch (SocketException ex)
{
if (ex.SocketErrorCode == SocketError.Interrupted)
@@ -110,7 +109,7 @@ namespace TimberWinR.Inputs
{
JObject json = JObject.Load(reader);
ProcessJson(json);
Interlocked.Increment(ref _receivedMessages);
_receivedMessages++;
}
catch (Exception ex)
{
@@ -119,16 +118,12 @@ namespace TimberWinR.Inputs
ProcessJson(jex1);
LogManager.GetCurrentClassLogger().Warn(ex);
Interlocked.Increment(ref _errorCount);
}
}
}
}
}
catch(OperationCanceledException)
{
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);

View File

@@ -13,8 +13,8 @@ namespace TimberWinR.Inputs
{
public class UdpInputListener : InputListener
{
private UdpClient _udpListenerV4;
private readonly Thread _listenThreadV4;
private UdpClient _udpListenerV6;
private readonly Thread _listenThreadV6;
private readonly int _port;
private long _receivedMessages;
@@ -36,11 +36,13 @@ namespace TimberWinR.Inputs
public UdpInputListener(CancellationToken cancelToken, int port = 5140)
: base(cancelToken, "Win32-Udp")
{
_port = port;
_port = port;
_receivedMessages = 0;
_listenThreadV4 = new Thread(StartListener);
_listenThreadV4.Start();
_listenThreadV6 = new Thread(StartListener);
_listenThreadV6.Start();
}
@@ -49,21 +51,31 @@ namespace TimberWinR.Inputs
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
// close UDP listeners, which will end the listener threads
_udpListenerV4.Close();
_udpListenerV6.Close();
// wait for completion of the threads
_listenThreadV4.Join();
_listenThreadV6.Join();
base.Shutdown();
}
private void StartListener()
{
var groupV4 = new IPEndPoint(IPAddress.Any, _port);
var groupV6 = new IPEndPoint(IPAddress.IPv6Any, _port);
// Create the socket as IPv6
var dualModeSocket = new Socket(AddressFamily.InterNetworkV6, SocketType.Dgram, ProtocolType.Udp);
//
// Now, disable the IPV6only flag to make it compatable with both ipv4 and ipv6
// See: http://blogs.msdn.com/b/malarch/archive/2005/11/18/494769.aspx
//
dualModeSocket.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, 0);
dualModeSocket.Bind(groupV6);
_udpListenerV4 = new UdpClient(_port);
LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", groupV4);
_udpListenerV6 = new UdpClient();
_udpListenerV6.Client = dualModeSocket;
LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", groupV6);
string lastMessage = "";
try
@@ -72,18 +84,13 @@ namespace TimberWinR.Inputs
{
try
{
byte[] bytes = _udpListenerV4.Receive(ref groupV4);
byte[] bytes = _udpListenerV6.Receive(ref groupV6);
var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
lastMessage = data;
var json = JObject.Parse(data);
ProcessJson(json);
Interlocked.Increment(ref _receivedMessages);
}
catch(ArgumentException aex)
{
LogManager.GetCurrentClassLogger().Error(aex);
break;
}
catch(SocketException)
{
break;
@@ -100,7 +107,7 @@ namespace TimberWinR.Inputs
Interlocked.Increment(ref _parsedErrors);
}
}
_udpListenerV4.Close();
_udpListenerV6.Close();
}
catch (Exception ex)
{

View File

@@ -1,10 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Runtime.InteropServices;
using System.Security.AccessControl;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
using Interop.MSUtil;
using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using NLog;
using TimberWinR.Parser;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using EventLogInputFormat = Interop.MSUtil.COMEventLogInputContextClassClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
@@ -16,13 +23,13 @@ namespace TimberWinR.Inputs
/// </summary>
public class WindowsEvtInputListener : InputListener
{
private readonly int _pollingIntervalInSeconds = 1;
private readonly WindowsEvent _arguments;
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Parser.WindowsEvent _arguments;
private long _receivedMessages;
private readonly List<Thread> _tasks;
private List<Thread> _tasks { get; set; }
public bool Stop { get; set; }
public WindowsEvtInputListener(WindowsEvent arguments, CancellationToken cancelToken)
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-Eventlog")
{
_arguments = arguments;
@@ -31,7 +38,8 @@ namespace TimberWinR.Inputs
foreach (string eventHive in _arguments.Source.Split(','))
{
var thread = new Thread(EventWatcher) {Name = "Win32-Eventlog-" + eventHive};
string hive = eventHive.Trim();
var thread = new Thread(new ParameterizedThreadStart(EventWatcher));
_tasks.Add(thread);
thread.Start(eventHive);
}
@@ -40,11 +48,7 @@ namespace TimberWinR.Inputs
public override void Shutdown()
{
Stop = true;
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
foreach (var thread in _tasks)
{
thread.Join();
}
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
base.Shutdown();
}
@@ -72,6 +76,8 @@ namespace TimberWinR.Inputs
{
string location = ploc.ToString();
LogQuery oLogQuery = new LogQuery();
LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready");
// Instantiate the Event Log Input Format object
@@ -87,7 +93,9 @@ namespace TimberWinR.Inputs
resolveSIDs = _arguments.ResolveSIDS
};
var logFileMaxRecords = new Dictionary<string, Int64>();
oLogQuery = null;
Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
using (var syncHandle = new ManualResetEventSlim())
{
@@ -99,7 +107,7 @@ namespace TimberWinR.Inputs
{
try
{
var oLogQuery = new LogQuery();
oLogQuery = new LogQuery();
var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
var rsfiles = oLogQuery.Execute(qfiles, iFmt);
@@ -137,19 +145,21 @@ namespace TimberWinR.Inputs
object v = record.getValue(field.Name);
if (field.Name == "Data")
v = ToPrintable(v.ToString());
if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && field.DataType == typeof (DateTime))
v = ((DateTime) v).ToUniversalTime();
json.Add(new JProperty(field.Name, v));
}
var lrn = (Int64)record.getValueEx("RecordNumber");
logFileMaxRecords[fileName] = lrn;
record = null;
ProcessJson(json);
_receivedMessages++;
json = null;
}
// Close the recordset
rs.close();
rs = null;
GC.Collect();
}
if (!Stop)

View File

@@ -150,13 +150,12 @@ namespace TimberWinR.Outputs
{
try
{
RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port);
RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port);
client.SendTimeout = _timeout;
return client;
}
catch (Exception ex)
catch (Exception)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
finally
{

View File

@@ -3,12 +3,8 @@
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version / Date
### 1.3.21.0 - 2015-04-13
1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets
and hosts file. If we want to add this back, I will add a Udpv6 input.
### 1.3.20.0 - 2015-04-03
### 1.4.0.0 - 04/03/2015
1. A re-factoring of Logs and TailLogs to be more efficient and detect log rolling correctly,
this requires http://support.microsoft.com/en-us/kb/172190 which will be detected and
set by TimberWinR, however, requires a reboot.
@@ -16,12 +12,11 @@ Version / Date
3. Created TimberWinR.TestGenerator for complete testing of TimberWinR
4. Fixed ipv4/ipv6 thread-safe issue with UdpInputListener which might lead to corrupted input data.
### 1.3.19.1 - 2015-03-03
### 1.3.19.1 - 03/03/2015
1. Added new Redis parameter _max\_batch\_count_ which increases the _batch\_count_ dynamically over time
to handle input flooding. Default is _batch\_count_ * 10
### 1.3.19.0 - 2015-02-26
### 1.3.19.0 - 02/26/2015
1. Added support for Multiline codecs for Stdin and Logs listeners, closes issue [#23](https://github.com/Cimpress-MCP/TimberWinR/issues/23)
2. Added new TailFiles input type which uses a native implementation (more-efficient) than using LogParser's Log
@@ -30,60 +25,61 @@ Version / Date
5. Fixed bug when tailing non-existent log files which resulted in high cpu-usage.
6. Added feature to watch the configuration directory
### 1.3.18.0 - 2014-12-22
### 1.3.18.0 - 12/22/2014
1. Fixed bug introduced in 1.3.17.0 which changed the meaning of the delay for Elasticsearch, Redis and Stdout
intervals to be interpreted as seconds instead of milliseconds. 1.3.17.0 should not be used.
2. Removed ability for installer to downgrade which was leading to leaving previous versions laying around (i.e. reverts 1.3.13.0 change)
### 1.3.17.0 - 2014-12-19
### 1.3.17.0 - 12/19/2014
1. Continued work improving shutdown time by using syncHandle.Wait instead of Thread.Sleep
### 1.3.16.0 - 2014-12-19
### 1.3.16.0 - 12/19/2014
1. Added logSource property to the Log input to facility the steering of log messages to different indices.
### 1.3.15.0 - 2014-12-12
### 1.3.15.0 - 12/12/2014
1. Fixed bug whereby if the Udp or Tcp inputs receive an impropery formatted Json it caused the thread to terminate, and ignore
future messages.
### 1.3.14.0 - 2014-12-11
### 1.3.14.0 - 12/11/2014
1. Fixed bug with the Grok filter to match properly the value of the Text field against non-blank entries.
### 1.3.13.0 - 2014-12-02
### 1.3.13.0 - 12/02/2014
1. Fixed MSI installer to allow downgrades.
### 1.3.12.0 - 2014-11-25
### 1.3.12.0 - 11/25/2014
1. Fixed all remaining memory leaks due to the COM Weak Surrogate which requires an explicit GC.Collect
### 1.3.11.0 - 2014-11-21
### 1.3.11.0 - 11/21/2014
1. Re-worked WindowsEvent listener to enable shutting down in a quicker fashion.
### 1.3.10.0 - 2014-11-18
### 1.3.10.0 - 11/18/2014
1. Refactored Conditions handler to use non-leaking evaluator.
### 1.3.9.0 - 2014-11-11
### 1.3.9.0 - 11/11/2014
1. Merged in pull request #9
2. Updated chocolately uninstall to preserve GUID
### 1.3.8.0 - 2014-11-06
### 1.3.8.0 - 11/06/2014
1. Added interval parameter to WindowsEvent input listener
2. Increased default value for interval to 60 seconds for polling WindowsEvents
### 1.3.7.0 - 2014-10-21
### 1.3.7.0 - 10/21/2014
1. Added additional information for diagnostics port
2. Completed minor handling of Log rolling detection
### 1.3.6.0 - 2014-10-16
### 1.3.6.0 - 10/16/2014
1. Handle rolling of logs whereby the logfile remains the same, but the content resets back to 0 bytes.

View File

@@ -98,12 +98,10 @@
</CreateProperty>
</Target>
<PropertyGroup>
<PreBuildEvent>mkdir $(SolutionDir)tools
cmd.exe /c copy $(SolutionDir)chocolateyUninstall.ps1.template.orig $(SolutionDir)chocolateyUninstall.ps1.template</PreBuildEvent>
<PostBuildEvent>$(SolutionDir)\TimberWinR.ExtractID\$(OutDir)\TimberWinR.ExtractID.exe $(TargetDir) $(SolutionDir)chocolateyUninstall.ps1.guid $(SolutionDir)chocolateyUninstall.ps1.template</PostBuildEvent>
</PropertyGroup>
<PropertyGroup>
<PostBuildEvent>$(SolutionDir)\TimberWinR.ExtractID\$(OutDir)\TimberWinR.ExtractID.exe $(TargetDir) $(SolutionDir)chocolateyUninstall.ps1.guid $(SolutionDir)chocolateyUninstall.ps1.template
cmd.exe /c copy "$(TargetDir)%2a.msi" "$(SolutionDir)tools"</PostBuildEvent>
<PreBuildEvent>cmd.exe /c copy $(SolutionDir)chocolateyUninstall.ps1.template.orig $(SolutionDir)chocolateyUninstall.ps1.template</PreBuildEvent>
</PropertyGroup>
<!--
To modify your build process, add your task inside one of the targets below and uncomment it.

View File

@@ -1,14 +1,8 @@
$packageName = 'TimberWinR-${version}'
$fileType = 'msi'
$installerType = 'msi'
$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi'
$silentArgs = '/quiet'
$scriptPath = $(Split-Path $MyInvocation.MyCommand.Path)
$fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}'
try {
Install-ChocolateyInstallPackage $packageName $fileType $silentArgs $fileFullPath
} catch {
Write-ChocolateyFailure $packageName $($_.Exception.Message)
throw
}
$validExitCodes = @(0)
Install-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" "$url64" -validExitCodes $validExitCodes

View File

@@ -1,9 +1,8 @@
$packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
$installerType = 'msi' #only one of these: exe, msi, msu
$scriptPath = $(Split-Path $MyInvocation.MyCommand.Path)
$fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}'
$silentArgs = '${PROJECTGUID} /quiet'
$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
$silentArgs = '{593EF0C4-54E0-40D5-A3E3-922CD1C25B9E} /quiet'
$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes

View File

@@ -1,7 +1,8 @@
$packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
$installerType = 'msi' #only one of these: exe, msi, msu
$scriptPath = $(Split-Path $MyInvocation.MyCommand.Path)
$fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}'
$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
$silentArgs = '${PROJECTGUID} /quiet'
$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes