TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Incorporated review changes

Browse Source
This commit is contained in:
Eric Fontana
2015-04-30 13:12:44 -04:00
parent 58ae2db05c
commit 8fe3b1815c
12 changed files with 255 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
README.md
View File

@@ -107,6 +107,7 @@ represented as a JSON Property or Array.
1. [Redis](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md)
2. [Elasticsearch](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md)
3. [Stdout](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md)
4. [File](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/FileOutput.md)
## Sample Configuration
TimberWinR reads a JSON configuration file, an example file is shown here:

20
TimberWinR.TestGenerator/UdpTestGenerator.cs
View File

@@ -1,4 +1,6 @@
using System.Threading;
using System.Security.Cryptography;
using System.Threading;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using NLog.Config;
@@ -52,13 +54,27 @@ namespace TimberWinR.TestGenerator
{"Executable", "VP.Common.SvcFrm.Services.Host, Version=29.7.0.0, Culture=neutral, PublicKeyToken=null"},
{"RenderedMessage", "Responding to RequestSchedule message from 10.1.230.36 with Ack because: PRJ byte array is null."},
{"Team", "Manufacturing Software"},
{"RecordNumber", i},
{"Host", hostName},
{"UtcTimestamp", DateTime.UtcNow.ToString("o")},
{"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"},
{"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
{"Index", "logstash"}
};
byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString());
string hashedString = "";
foreach(var key in o)
{
hashedString += key.ToString();
}
var source = ASCIIEncoding.ASCII.GetBytes(hashedString);
var md5 = new MD5CryptoServiceProvider().ComputeHash(source);
var hash = string.Concat(md5.Select(x => x.ToString("X2")));
o["md5"] = hash;
byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None));
IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
s.SendTo(sendbuf, ep);

2
TimberWinR.TestGenerator/results5.json
View File

@@ -3,7 +3,7 @@
"Inputs": [
{
"udp": {
"test1: message sent count": "[messages] == 10000",
"test1: message sent count": "[messages] == 80000",
"test2: average cpu": "[avgCpuUsage] <= 30",
"test3: maximum memory": "[maxMemUsage] <= 30"
}

12
TimberWinR.TestGenerator/test5-twconfig.json
View File

@@ -25,16 +25,10 @@
]
},
"Outputs": {
"Redis": [
"File": [
{
"_comment": "Change the host to your Redis instance",
"port": 6379,
"batch_count": 500,
"interval": 1000,
"threads": 4,
"host": [
"tstlexiceapp006.mycompany.svc"
]
"format": "indented",
"file_name": "test5_output.txt"
}
]
}

4
TimberWinR.TestGenerator/test5.json
View File

@@ -5,11 +5,11 @@
"--testFile": "test5.json",
"--testDir": "test5",
"--timberWinRConfig": "test5-twconfig.json",
"--numMessages": 20000,
"--numMessages": 80000,
"--logLevel": "debug",
"--udp-host": "localhost",
"--udp": "5140",
"--udp-rate": 5,
"--udp-rate": 1,
"--resultsFile": "results5.json"
}
}

9
TimberWinR/Configuration.cs
View File

@@ -58,6 +58,12 @@ namespace TimberWinR
get { return _stdoutOutputs; }
}
private List<FileOutputParameters> _fileOutputs = new List<FileOutputParameters>();
public IEnumerable<FileOutputParameters> FileOutputs
{
get { return _fileOutputs; }
}
private List<TcpParameters> _tcps = new List<TcpParameters>();
public IEnumerable<TcpParameters> Tcps
{
@@ -265,6 +271,8 @@ namespace TimberWinR
c._elasticsearchOutputs.AddRange(x.TimberWinR.Outputs.Elasticsearch.ToList());
if (x.TimberWinR.Outputs.Stdout != null)
c._stdoutOutputs.AddRange(x.TimberWinR.Outputs.Stdout.ToList());
if (x.TimberWinR.Outputs.File != null)
c._fileOutputs.AddRange(x.TimberWinR.Outputs.File.ToList());
}
if (x.TimberWinR.Filters != null)
@@ -302,6 +310,7 @@ namespace TimberWinR
_redisOutputs = new List<RedisOutputParameters>();
_elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
_stdoutOutputs = new List<StdoutOutputParameters>();
_fileOutputs = new List<FileOutputParameters>();
_tcps = new List<TcpParameters>();
_udps = new List<UdpParameters>();
}

9
TimberWinR/Manager.cs
View File

@@ -190,6 +190,15 @@ namespace TimberWinR
}
}
if (config.FileOutputs != null)
{
foreach (var ro in config.FileOutputs)
{
var output = new FileOutput(this, ro, cancelToken);
Outputs.Add(output);
}
}
foreach (Parser.IISW3CLogParameters iisw3cConfig in config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);

140
TimberWinR/Outputs/File.cs Normal file
View File

@@ -0,0 +1,140 @@
using System.IO;
using System.Text;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
namespace TimberWinR.Outputs
{
public class FileOutput : OutputSender
{
private TimberWinR.Manager _manager;
private readonly int _interval;
private readonly object _locker = new object();
private readonly List<JObject> _jsonQueue;
private long _sentMessages;
private Parser.FileOutputParameters _arguments;
public bool Stop { get; set; }
public FileOutput(TimberWinR.Manager manager, Parser.FileOutputParameters arguments, CancellationToken cancelToken)
: base(cancelToken, "File")
{
_arguments = arguments;
_sentMessages = 0;
_manager = manager;
_interval = arguments.Interval;
_jsonQueue = new List<JObject>();
var elsThread = new Task(FileSender, cancelToken);
elsThread.Start();
}
public override JObject ToJson()
{
JObject json = new JObject(
new JProperty("file-output",
new JObject(
new JProperty("queuedMessageCount", _jsonQueue.Count),
new JProperty("sentMessageCount", _sentMessages))));
return json;
}
//
// Pull off messages from the Queue, batch them up and send them all across
//
private void FileSender()
{
using (var syncHandle = new ManualResetEventSlim())
{
var fi = new FileInfo(_arguments.FileName);
if (File.Exists(_arguments.FileName))
File.Delete(_arguments.FileName);
LogManager.GetCurrentClassLogger().Info("File Output Sending To: {0}", fi.FullName);
using (StreamWriter sw = File.AppendText(_arguments.FileName))
{
// Execute the query
while (!Stop)
{
if (!CancelToken.IsCancellationRequested)
{
try
{
JObject[] messages;
lock (_locker)
{
messages = _jsonQueue.Take(_jsonQueue.Count).ToArray();
_jsonQueue.RemoveRange(0, messages.Length);
}
if (messages.Length > 0)
{
try
{
foreach (JObject obj in messages)
{
sw.WriteLine(obj.ToString(_arguments.ToFormat()));
_sentMessages++;
}
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
}
if (!Stop)
syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception)
{
}
}
}
}
}
}
protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage)
{
if (_manager.Config.Filters != null)
{
if (ApplyFilters(jsonMessage))
return;
}
var message = jsonMessage.ToString();
lock (_locker)
{
_jsonQueue.Add(jsonMessage);
}
}
private bool ApplyFilters(JObject json)
{
bool drop = false;
foreach (var filter in _manager.Config.Filters)
{
if (!filter.Apply(json))
drop = true;
}
return drop;
}
}
}

40
TimberWinR/Parser.cs
View File

@@ -679,6 +679,43 @@ namespace TimberWinR.Parser
}
}
public class FileOutputParameters
{
public enum FormatKind
{
none, indented
};
[JsonProperty(PropertyName = "interval")]
public int Interval { get; set; }
[JsonProperty(PropertyName = "file_name")]
public string FileName { get; set; }
[JsonProperty(PropertyName = "format")]
public FormatKind Format { get; set; }
public FileOutputParameters()
{
Format = FormatKind.none;
Interval = 1000;
FileName = "timberwinr.out";
}
public Newtonsoft.Json.Formatting ToFormat()
{
switch (Format)
{
case FormatKind.indented:
return Newtonsoft.Json.Formatting.Indented;
case FormatKind.none:
default:
return Newtonsoft.Json.Formatting.None;
}
}
}
public class OutputTargets
{
[JsonProperty("Redis")]
@@ -689,6 +726,9 @@ namespace TimberWinR.Parser
[JsonProperty("Stdout")]
public StdoutOutputParameters[] Stdout { get; set; }
[JsonProperty("File")]
public FileOutputParameters[] File { get; set; }
}
public class InputSources

1
TimberWinR/ReleaseNotes.md
View File

@@ -8,6 +8,7 @@ Version / Date
2. Fixed potential non-thread safe when renaming properties
3. Added add_field, rename support to Udp/Tcp Input Listeners
4. Fixed issue with multiple renames (was previously only renaming the first one)
5. Added File outputter for testing.
### 1.3.24.0 - 2015-04-29
1. Fixed potential bug in TailFiles when tailing log files which are partially flushed

2
TimberWinR/TimberWinR.csproj
View File

@@ -115,6 +115,7 @@
<Compile Include="Outputs\Elasticsearch.cs" />
<Compile Include="Outputs\OutputSender.cs" />
<Compile Include="Outputs\Redis.cs" />
<Compile Include="Outputs\File.cs" />
<Compile Include="Outputs\Stdout.cs" />
<Compile Include="Parser.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
@@ -141,6 +142,7 @@
<None Include="mdocs\Filters.md" />
<None Include="mdocs\GeoIPFilter.md" />
<None Include="mdocs\Generator.md" />
<None Include="mdocs\FileOutput.md" />
<None Include="mdocs\TailFiles.md" />
<None Include="mdocs\UdpInput.md" />
<None Include="mdocs\W3CInput.md" />

27
TimberWinR/mdocs/FileOutput.md Normal file
View File

@@ -0,0 +1,27 @@
# Output: File
The File output passes on data into a text file.
## Parameters
The following parameters are allowed when configuring the File output.
| Parameter | Type | Description | Details | Default |
| :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- |
| *interval* | integer | Interval in milliseconds to sleep before appending data | Interval | 1000 |
| *file_name* | string | Name of the file to be created | | timberwinr.out |
Example Input:
```json
{
"TimberWinR": {
"Outputs": {
"File": [
{
"file_name": "foo.out",
"interval": 1000
}
]
}
}
}
```