Compare commits
1 Commits
rel-1.3.26
...
add_rename
| Author | SHA1 | Date | |
|---|---|---|---|
|
7156d9c6f8 |
@@ -2,6 +2,6 @@
|
||||
<packages>
|
||||
<package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
|
||||
<package id="NUnit.Runners" version="2.6.4" />
|
||||
<package id="RapidRegex.Core" version="1.0.0.4" targetFramework="net40" />
|
||||
<package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
|
||||
<package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
|
||||
</packages>
|
||||
@@ -16,7 +16,7 @@ TimberWinR uses a configuration file to control how the logs are collected, filt
|
||||
These are broken down into:
|
||||
1. Inputs (Collect data from different sources)
|
||||
2. Filters (Are applied to all Inputs)
|
||||
3. Outputs (e.g. Redis, Elasticsearch, Stdout, StatsD)
|
||||
3. Outputs (Redis, Elasticsearch or Stdout)
|
||||
|
||||
### Support ###
|
||||
Please use the TimberWinR Google Group for discussion and support:
|
||||
@@ -108,7 +108,6 @@ represented as a JSON Property or Array.
|
||||
2. [Elasticsearch](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md)
|
||||
3. [Stdout](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md)
|
||||
4. [File](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/FileOutput.md)
|
||||
5. [StatsD](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StatsD.md)
|
||||
|
||||
## Sample Configuration
|
||||
TimberWinR reads a JSON configuration file, an example file is shown here:
|
||||
|
||||
@@ -131,8 +131,6 @@ namespace TimberWinR.ServiceHost
|
||||
private readonly Arguments _args;
|
||||
private TimberWinR.Diagnostics.Diagnostics _diags;
|
||||
private TimberWinR.Manager _manager;
|
||||
public bool StartingUp { get; set; }
|
||||
public bool Started { get; set; }
|
||||
|
||||
public TimberWinRService(Arguments args)
|
||||
{
|
||||
@@ -149,8 +147,6 @@ namespace TimberWinR.ServiceHost
|
||||
|
||||
public void Stop()
|
||||
{
|
||||
WaitForStartupToComplete();
|
||||
|
||||
_cancellationTokenSource.Cancel();
|
||||
if (_diags != null)
|
||||
_diags.Shutdown();
|
||||
@@ -159,31 +155,14 @@ namespace TimberWinR.ServiceHost
|
||||
_manager.Shutdown();
|
||||
}
|
||||
|
||||
// If you bounce the service too quickly, the shutdown can occur
|
||||
// before the service has started, which results in a hang, this blocks until
|
||||
// all thread have properly started (waiting up to 10 seconds max)
|
||||
private void WaitForStartupToComplete()
|
||||
{
|
||||
int tries = 100; // 10 seconds max
|
||||
if (StartingUp)
|
||||
{
|
||||
while (!Started && tries-- >= 0)
|
||||
{
|
||||
Thread.Sleep(100);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// The Main body of the Service Worker Thread
|
||||
/// </summary>
|
||||
private void RunService()
|
||||
{
|
||||
StartingUp = true;
|
||||
_manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _args.LiveMonitor, _cancellationToken);
|
||||
if (_args.DiagnosticPort > 0)
|
||||
_diags = new Diagnostics.Diagnostics(_manager, _cancellationToken, _args.DiagnosticPort);
|
||||
Started = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
|
||||
// You can specify all the values or you can default the Build and Revision Numbers
|
||||
// by using the '*' as shown below:
|
||||
// [assembly: AssemblyVersion("1.0.*")]
|
||||
[assembly: AssemblyVersion("1.3.26.0")]
|
||||
[assembly: AssemblyFileVersion("1.3.26.0")]
|
||||
[assembly: AssemblyVersion("1.3.25.0")]
|
||||
[assembly: AssemblyFileVersion("1.3.25.0")]
|
||||
|
||||
@@ -38,16 +38,9 @@
|
||||
<ApplicationIcon>timberwinr.ico</ApplicationIcon>
|
||||
</PropertyGroup>
|
||||
<ItemGroup>
|
||||
<Reference Include="RapidRegex.Core, Version=1.0.0.4, Culture=neutral, processorArchitecture=MSIL">
|
||||
<Reference Include="RapidRegex.Core, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\RapidRegex.Core.1.0.0.4\lib\net40\RapidRegex.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="StatsdClient, Version=1.0.0.19, Culture=neutral, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="StatsdClient.Configuration">
|
||||
<HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.Configuration.dll</HintPath>
|
||||
<HintPath>..\packages\RapidRegex.Core.1.0.0.2\lib\net40\RapidRegex.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.Configuration.Install" />
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<packages>
|
||||
<package id="RapidRegex.Core" version="1.0.0.4" targetFramework="net40" />
|
||||
<package id="StatsdClient" version="1.0.0.19" targetFramework="net40" />
|
||||
<package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
|
||||
<package id="Topshelf" version="3.1.4" targetFramework="net40" />
|
||||
</packages>
|
||||
@@ -27,9 +27,6 @@ namespace TimberWinR.TestGenerator
|
||||
[Option("resultsFile", HelpText = "Expected results Results json file")]
|
||||
public string ExpectedResultsFile { get; set; }
|
||||
|
||||
[Option("totalMessages", DefaultValue = 0, HelpText = "The total number of messages to send to the output(s)")]
|
||||
public int TotalMessages { get; set; }
|
||||
|
||||
[Option('n', "numMessages", DefaultValue = 1000, HelpText = "The number of messages to send to the output(s)")]
|
||||
public int NumMessages { get; set; }
|
||||
|
||||
|
||||
@@ -350,7 +350,7 @@ namespace TimberWinR.TestGenerator
|
||||
var mbc = outputToken["queuedMessageCount"].Value<int>();
|
||||
var smc = outputToken["sentMessageCount"].Value<int>();
|
||||
|
||||
//LogManager.GetCurrentClassLogger().Info("Output: {2} Queued: {0}, Sent: {1}", mbc, smc, outputToken.ToString());
|
||||
// LogManager.GetCurrentClassLogger().Info("Queued: {0}, Sent: {1}", mbc, smc);
|
||||
|
||||
completed = mbc == 0 && smc >= _totalMessagesToSend;
|
||||
}
|
||||
@@ -504,8 +504,6 @@ namespace TimberWinR.TestGenerator
|
||||
|
||||
static Task[] RunGenerators(CommandLineOptions options)
|
||||
{
|
||||
_totalMessagesToSend = options.TotalMessages;
|
||||
|
||||
_monitorTask = Task.Factory.StartNew(() =>
|
||||
{
|
||||
using (var syncHandle = new ManualResetEventSlim())
|
||||
|
||||
@@ -126,18 +126,15 @@
|
||||
<Content Include="results5.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="test7-tw.json">
|
||||
<Content Include="test6-tw.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="test7.json">
|
||||
<Content Include="test6.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<Content Include="results7.json">
|
||||
<Content Include="results6.json">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</Content>
|
||||
<None Include="sample-apache.log">
|
||||
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
|
||||
</None>
|
||||
</ItemGroup>
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">
|
||||
|
||||
@@ -59,13 +59,16 @@ namespace TimberWinR.TestGenerator
|
||||
{"UtcTimestamp", DateTime.UtcNow.ToString("o")},
|
||||
{"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"},
|
||||
{"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
|
||||
{"Index", "logstash"}
|
||||
{"Index", "logstash"},
|
||||
{"HashedFields", "Application,Executable,RenderedMessage,Team,RecordNumber,Host,Message,Index"}
|
||||
};
|
||||
|
||||
string hashedString = "";
|
||||
foreach(var key in o)
|
||||
if (o["HashedFields"] != null)
|
||||
{
|
||||
hashedString += key.ToString();
|
||||
foreach (var key in o["HashedFields"].ToString().Split(new char[] {','}))
|
||||
{
|
||||
hashedString += string.Format("{0}:{1}", key, o[key].ToString());
|
||||
}
|
||||
|
||||
var source = ASCIIEncoding.ASCII.GetBytes(hashedString);
|
||||
@@ -73,6 +76,7 @@ namespace TimberWinR.TestGenerator
|
||||
var hash = string.Concat(md5.Select(x => x.ToString("X2")));
|
||||
|
||||
o["md5"] = hash;
|
||||
}
|
||||
|
||||
byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None));
|
||||
IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -38,10 +38,11 @@
|
||||
{
|
||||
"_comment": "Change the host to your Redis instance",
|
||||
"port": 6379,
|
||||
"max_queue_size": 20000,
|
||||
"batch_count": 500,
|
||||
"threads": 1,
|
||||
"host": [
|
||||
"tstlexiceapp006.mycompany.svc"
|
||||
"tstlexiceapp006.vistaprint.svc"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,12 +1,11 @@
|
||||
{
|
||||
"test": "Test 2",
|
||||
"arguments": {
|
||||
"--start": "",
|
||||
"--testFile": "test2.json",
|
||||
"--testDir": "test2",
|
||||
"--timberWinRConfig": "test2-tw.json",
|
||||
"--numMessages": 1234,
|
||||
"--logLevel": "debug",
|
||||
"--logLevel": "trace",
|
||||
"--udp": "5140",
|
||||
"--jroll": ["r1.jlog", "r2.jlog"],
|
||||
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
|
||||
|
||||
@@ -39,9 +39,10 @@
|
||||
"_comment": "Change the host to your Redis instance",
|
||||
"port": 6379,
|
||||
"batch_count": 500,
|
||||
"max_queue_size": 200,
|
||||
"threads": 2,
|
||||
"host": [
|
||||
"tstlexiceapp006.mycompany.svc"
|
||||
"tstlexiceapp006.vistaprint.svc"
|
||||
]
|
||||
}
|
||||
]
|
||||
|
||||
@@ -1,11 +1,14 @@
|
||||
{
|
||||
"test": "Test 4",
|
||||
"arguments": {
|
||||
"--start": "",
|
||||
"--testFile": "test4.json",
|
||||
"--testDir": "test4",
|
||||
"--timberWinRConfig": "test4-tw.json",
|
||||
"--numMessages": 1234,
|
||||
"--logLevel": "debug",
|
||||
"--jroll": ["r1.jlog", "r2.jlog"],
|
||||
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
|
||||
"--resultsFile": "results4.json"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"--testFile": "test5.json",
|
||||
"--testDir": "test5",
|
||||
"--timberWinRConfig": "test5-twconfig.json",
|
||||
"--numMessages": 80000,
|
||||
"--numMessages": 1000000,
|
||||
"--logLevel": "debug",
|
||||
"--udp-host": "localhost",
|
||||
"--udp": "5140",
|
||||
|
||||
51
TimberWinR.TestGenerator/test6-tw.json
Normal file
51
TimberWinR.TestGenerator/test6-tw.json
Normal file
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"TimberWinR": {
|
||||
"Inputs": {
|
||||
"Tcp": [
|
||||
{
|
||||
"_comment": "Output from NLog",
|
||||
"port": 5140
|
||||
}
|
||||
],
|
||||
"TailFiles": [
|
||||
{
|
||||
"interval": 5,
|
||||
"logSource": "log files",
|
||||
"location": "*.jlog",
|
||||
"recurse": -1
|
||||
}
|
||||
]
|
||||
},
|
||||
"Filters": [
|
||||
{
|
||||
"grok": {
|
||||
"condition": "\"[EventTypeName]\" == \"Information Event\"",
|
||||
"match": [
|
||||
"Text",
|
||||
""
|
||||
],
|
||||
"drop": "true"
|
||||
},
|
||||
"json": {
|
||||
"type": "Win32-TailFile",
|
||||
"source": "Text",
|
||||
"promote": "Text"
|
||||
}
|
||||
}
|
||||
],
|
||||
"Outputs": {
|
||||
"Redis": [
|
||||
{
|
||||
"_comment": "Change the host to your Redis instance",
|
||||
"port": 6379,
|
||||
"max_queue_size": 20000,
|
||||
"batch_count": 500,
|
||||
"threads": 1,
|
||||
"host": [
|
||||
"tstlexiceapp006.vistaprint.svc"
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
15
TimberWinR.TestGenerator/test6.json
Normal file
15
TimberWinR.TestGenerator/test6.json
Normal file
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"test": "Test 6",
|
||||
"arguments": {
|
||||
"--start": "",
|
||||
"--testFile": "test6.json",
|
||||
"--testDir": "test6",
|
||||
"--timberWinRConfig": "test6-tw.json",
|
||||
"--numMessages": 10000,
|
||||
"--logLevel": "debug",
|
||||
"--tcp-host": "localhost",
|
||||
"--tcp-rate": 1,
|
||||
"--tcp": "5140",
|
||||
"--resultsFile": "results1.json"
|
||||
}
|
||||
}
|
||||
@@ -1,37 +0,0 @@
|
||||
{
|
||||
"TimberWinR": {
|
||||
"Inputs": {
|
||||
"TailFiles": [
|
||||
{
|
||||
"interval": 5,
|
||||
"logSource": "apache log files",
|
||||
"location": "..\\sample-apache.log",
|
||||
"recurse": -1
|
||||
}
|
||||
]
|
||||
},
|
||||
"Filters": [
|
||||
{
|
||||
"grok": {
|
||||
"type": "Win32-TailLog",
|
||||
"match": [
|
||||
"Text",
|
||||
"%{COMBINEDAPACHELOG}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"Outputs": {
|
||||
"StatsD": [
|
||||
{
|
||||
"type": "Win32-TailLog",
|
||||
"namespace": "timberwinrtest",
|
||||
"port": 8125,
|
||||
"host": "devlexicesnu003.mycompany.svc",
|
||||
"increment": ["apache.response.%{response}"],
|
||||
"count": ["apache.bytes", "%{bytes}"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,13 +0,0 @@
|
||||
{
|
||||
"test": "Test 7",
|
||||
"arguments": {
|
||||
"--totalMessages": 2223,
|
||||
"--start": "",
|
||||
"--testFile": "test7.json",
|
||||
"--testDir": "test7",
|
||||
"--timberWinRConfig": "test7-tw.json",
|
||||
"--numMessages": 1234,
|
||||
"--logLevel": "debug",
|
||||
"--resultsFile": "results7.json"
|
||||
}
|
||||
}
|
||||
@@ -367,6 +367,55 @@ namespace TimberWinR.UnitTests
|
||||
Assert.AreEqual(json["tags"][0].ToString(), "tag2");
|
||||
}
|
||||
|
||||
[Test]
|
||||
public void TestRename()
|
||||
{
|
||||
JObject json = new JObject
|
||||
{
|
||||
{"LogFilename", @"C:\\Logs1\\test1.log"},
|
||||
{"Index", 7},
|
||||
{"Text", null},
|
||||
{"tags", new JArray
|
||||
{
|
||||
"tag1",
|
||||
"tag2"
|
||||
}
|
||||
},
|
||||
{"type", "Win32-FileLog"},
|
||||
{"ComputerName", "dev.mycompany.net"}
|
||||
};
|
||||
|
||||
string grokJson = @"{
|
||||
""TimberWinR"":{
|
||||
""Filters"":[
|
||||
{
|
||||
""mutate"":{
|
||||
""condition"": ""\""[type]\"" == \""Win32-FileLog\"""",
|
||||
""match"":[
|
||||
""Text"",
|
||||
""""
|
||||
],
|
||||
""rename"":[
|
||||
""type"", ""newtype""
|
||||
]
|
||||
}
|
||||
}]
|
||||
}
|
||||
}";
|
||||
|
||||
Configuration c = Configuration.FromString(grokJson);
|
||||
|
||||
Mutate grok = c.Filters.First() as Mutate;
|
||||
|
||||
Assert.IsTrue(grok.Apply(json));
|
||||
|
||||
Assert.IsNull(json["type"]);
|
||||
Assert.AreEqual("Win32-FileLog", json["newtype"].ToString());
|
||||
Assert.IsTrue(json["tags"].Children().Count() == 2);
|
||||
|
||||
}
|
||||
|
||||
|
||||
[Test]
|
||||
public void TestMatchCount()
|
||||
{
|
||||
@@ -476,5 +525,8 @@ namespace TimberWinR.UnitTests
|
||||
Assert.IsTrue(grok.AddTag.Length >= 1);
|
||||
}
|
||||
|
||||
|
||||
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
@@ -39,12 +39,6 @@ namespace TimberWinR
|
||||
get { return _events; }
|
||||
}
|
||||
|
||||
private List<StatsDOutputParameters> _statsdOutputs = new List<StatsDOutputParameters>();
|
||||
public IEnumerable<StatsDOutputParameters> StatsDOutputs
|
||||
{
|
||||
get { return _statsdOutputs; }
|
||||
}
|
||||
|
||||
private List<RedisOutputParameters> _redisOutputs = new List<RedisOutputParameters>();
|
||||
public IEnumerable<RedisOutputParameters> RedisOutputs
|
||||
{
|
||||
@@ -271,8 +265,6 @@ namespace TimberWinR
|
||||
|
||||
if (x.TimberWinR.Outputs != null)
|
||||
{
|
||||
if (x.TimberWinR.Outputs.StatsD != null)
|
||||
c._statsdOutputs.AddRange(x.TimberWinR.Outputs.StatsD.ToList());
|
||||
if (x.TimberWinR.Outputs.Redis != null)
|
||||
c._redisOutputs.AddRange(x.TimberWinR.Outputs.Redis.ToList());
|
||||
if (x.TimberWinR.Outputs.Elasticsearch != null)
|
||||
@@ -315,7 +307,6 @@ namespace TimberWinR
|
||||
_events = new List<WindowsEvent>();
|
||||
_iisw3clogs = new List<IISW3CLogParameters>();
|
||||
_logs = new List<LogParameters>();
|
||||
_statsdOutputs = new List<StatsDOutputParameters>();
|
||||
_redisOutputs = new List<RedisOutputParameters>();
|
||||
_elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
|
||||
_stdoutOutputs = new List<StdoutOutputParameters>();
|
||||
|
||||
@@ -45,6 +45,7 @@ namespace TimberWinR.Parser
|
||||
new JProperty("condition", Condition),
|
||||
new JProperty("addfields", AddField),
|
||||
new JProperty("addtags", AddTag),
|
||||
new JProperty("rename", Rename),
|
||||
new JProperty("drop", DropIfMatch),
|
||||
new JProperty("type", Type),
|
||||
new JProperty("removefields", RemoveField),
|
||||
@@ -85,6 +86,7 @@ namespace TimberWinR.Parser
|
||||
AddTags(json);
|
||||
RemoveFields(json);
|
||||
RemoveTags(json);
|
||||
DoRenames(json);
|
||||
}
|
||||
|
||||
return true;
|
||||
@@ -125,6 +127,19 @@ namespace TimberWinR.Parser
|
||||
return false; // Not specified is failure
|
||||
}
|
||||
|
||||
private void DoRenames(Newtonsoft.Json.Linq.JObject json)
|
||||
{
|
||||
if (Rename != null && Rename.Length > 0)
|
||||
{
|
||||
for (int i = 0; i < Rename.Length; i += 2)
|
||||
{
|
||||
string oldName = ExpandField(Rename[i], json);
|
||||
string newName = ExpandField(Rename[i + 1], json);
|
||||
RenameProperty(json, oldName, newName);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void AddFields(Newtonsoft.Json.Linq.JObject json)
|
||||
{
|
||||
if (AddField != null && AddField.Length > 0)
|
||||
|
||||
@@ -30,6 +30,7 @@ namespace TimberWinR.Parser
|
||||
new JProperty("promote", Source),
|
||||
new JProperty("target", Target),
|
||||
new JProperty("type", Type),
|
||||
new JProperty("rename", Rename),
|
||||
new JProperty("addfields", AddField),
|
||||
new JProperty("addtags", AddTag),
|
||||
new JProperty("removefields", RemoveField),
|
||||
|
||||
@@ -165,15 +165,6 @@ namespace TimberWinR
|
||||
if (OnConfigurationProcessed != null)
|
||||
OnConfigurationProcessed(config);
|
||||
|
||||
if (config.StatsDOutputs != null)
|
||||
{
|
||||
foreach (var ro in config.StatsDOutputs)
|
||||
{
|
||||
var output = new StatsDOutput(this, ro, cancelToken);
|
||||
Outputs.Add(output);
|
||||
}
|
||||
}
|
||||
|
||||
if (config.RedisOutputs != null)
|
||||
{
|
||||
foreach (var ro in config.RedisOutputs)
|
||||
|
||||
@@ -1,368 +0,0 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Diagnostics.Eventing.Reader;
|
||||
using System.Linq;
|
||||
using System.Linq.Expressions;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using System.Threading;
|
||||
using CSRedis;
|
||||
using Nest;
|
||||
using Newtonsoft.Json;
|
||||
using Newtonsoft.Json.Linq;
|
||||
using NLog;
|
||||
using System.Threading.Tasks;
|
||||
using RapidRegex.Core;
|
||||
using System.Text.RegularExpressions;
|
||||
using System.Globalization;
|
||||
using StatsdClient;
|
||||
using TimberWinR.Parser;
|
||||
|
||||
namespace TimberWinR.Outputs
|
||||
{
|
||||
public class StatsDOutput : OutputSender
|
||||
{
|
||||
public int QueueDepth
|
||||
{
|
||||
get { return _jsonQueue.Count; }
|
||||
}
|
||||
|
||||
public long SentMessages
|
||||
{
|
||||
get { return _sentMessages; }
|
||||
}
|
||||
|
||||
|
||||
private readonly int _port;
|
||||
public string _host { get; set; }
|
||||
|
||||
private readonly int _interval;
|
||||
private readonly object _locker = new object();
|
||||
private readonly List<JObject> _jsonQueue;
|
||||
private TimberWinR.Manager _manager;
|
||||
private long _sentMessages;
|
||||
private long _errorCount;
|
||||
private readonly int _maxQueueSize;
|
||||
private readonly bool _queueOverflowDiscardOldest;
|
||||
private readonly int _flushSize;
|
||||
private readonly int _idleFlushTimeSeconds;
|
||||
private readonly int _numThreads;
|
||||
private Parser.StatsDOutputParameters _params;
|
||||
|
||||
public bool Stop { get; set; }
|
||||
|
||||
public override JObject ToJson()
|
||||
{
|
||||
var json = new JObject(
|
||||
new JProperty("statsd",
|
||||
new JObject(
|
||||
new JProperty("errors", _errorCount),
|
||||
new JProperty("sentMessageCount", _sentMessages),
|
||||
new JProperty("queuedMessageCount", _jsonQueue.Count),
|
||||
new JProperty("port", _port),
|
||||
new JProperty("threads", _numThreads),
|
||||
new JProperty("flushSize", _flushSize),
|
||||
new JProperty("idleFlushTime", _idleFlushTimeSeconds),
|
||||
new JProperty("maxQueueSize", _maxQueueSize),
|
||||
new JProperty("overflowDiscardOldest", _queueOverflowDiscardOldest),
|
||||
new JProperty("interval", _interval),
|
||||
new JProperty("host", _host)
|
||||
)));
|
||||
|
||||
return json;
|
||||
}
|
||||
|
||||
public StatsDOutput(TimberWinR.Manager manager, Parser.StatsDOutputParameters parameters, CancellationToken cancelToken)
|
||||
: base(cancelToken, "StatsD")
|
||||
{
|
||||
_params = parameters;
|
||||
_manager = manager;
|
||||
_port = parameters.Port;
|
||||
_host = parameters.Host;
|
||||
_interval = parameters.Interval;
|
||||
_flushSize = parameters.FlushSize;
|
||||
_idleFlushTimeSeconds = parameters.IdleFlushTimeInSeconds;
|
||||
_maxQueueSize = parameters.MaxQueueSize;
|
||||
_queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
|
||||
_numThreads = parameters.NumThreads;
|
||||
_jsonQueue = new List<JObject>();
|
||||
|
||||
var metricsConfig = new MetricsConfig
|
||||
{
|
||||
StatsdServerName = _host,
|
||||
Prefix = parameters.Namespace,
|
||||
};
|
||||
|
||||
StatsdClient.Metrics.Configure(metricsConfig);
|
||||
|
||||
for (int i = 0; i < _numThreads; i++)
|
||||
{
|
||||
Task.Factory.StartNew(StatsDSender, cancelToken, TaskCreationOptions.LongRunning, TaskScheduler.Current);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public override string ToString()
|
||||
{
|
||||
return string.Format("StatsD Host: {0} Port: {1}", _host, _port);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Forward on Json message to Redis Logstash queue
|
||||
/// </summary>
|
||||
/// <param name="jsonMessage"></param>
|
||||
protected override void MessageReceivedHandler(JObject jsonMessage)
|
||||
{
|
||||
if (_manager.Config.Filters != null)
|
||||
{
|
||||
if (ApplyFilters(jsonMessage))
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
var message = jsonMessage.ToString();
|
||||
LogManager.GetCurrentClassLogger().Trace(message);
|
||||
|
||||
lock (_locker)
|
||||
{
|
||||
if (_jsonQueue.Count >= _maxQueueSize)
|
||||
{
|
||||
// If we've exceeded our queue size, and we're supposed to throw out the oldest objects first,
|
||||
// then remove as many as necessary to get us under our limit
|
||||
if (_queueOverflowDiscardOldest)
|
||||
{
|
||||
LogManager.GetCurrentClassLogger()
|
||||
.Warn("Overflow discarding oldest {0} messages", _jsonQueue.Count - _maxQueueSize + 1);
|
||||
|
||||
_jsonQueue.RemoveRange(0, (_jsonQueue.Count - _maxQueueSize) + 1);
|
||||
}
|
||||
// Otherwise we're in a "discard newest" mode, and this is the newest message, so just ignore it
|
||||
else
|
||||
{
|
||||
LogManager.GetCurrentClassLogger()
|
||||
.Warn("Overflow discarding newest message: {0}", message);
|
||||
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
_jsonQueue.Add(jsonMessage);
|
||||
}
|
||||
}
|
||||
|
||||
private bool ApplyFilters(JObject json)
|
||||
{
|
||||
bool drop = false;
|
||||
foreach (var filter in _manager.Config.Filters)
|
||||
{
|
||||
if (!filter.Apply(json))
|
||||
{
|
||||
LogManager.GetCurrentClassLogger().Debug("{0}: Dropping: {1}", Thread.CurrentThread.ManagedThreadId, json.ToString());
|
||||
drop = true;
|
||||
}
|
||||
}
|
||||
// Check for matching type (if defined).
|
||||
if (!drop && !string.IsNullOrEmpty(_params.InputType) && json["type"] != null)
|
||||
{
|
||||
string msgType = json["type"].ToString();
|
||||
if (!string.IsNullOrEmpty(msgType) && msgType != _params.InputType)
|
||||
return true;
|
||||
}
|
||||
|
||||
return drop;
|
||||
}
|
||||
|
||||
// Places messages back into the queue (for a future attempt)
|
||||
private void interlockedInsert(List<JObject> messages)
|
||||
{
|
||||
lock (_locker)
|
||||
{
|
||||
Interlocked.Increment(ref _errorCount);
|
||||
_jsonQueue.InsertRange(0, messages);
|
||||
if (_jsonQueue.Count > _maxQueueSize)
|
||||
{
|
||||
LogManager.GetCurrentClassLogger().Warn("Exceeded maximum queue depth");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Pull off messages from the Queue, batch them up and send them all across
|
||||
//
|
||||
private void StatsDSender()
|
||||
{
|
||||
DateTime lastFlushTime = DateTime.MinValue;
|
||||
|
||||
using (var syncHandle = new ManualResetEventSlim())
|
||||
{
|
||||
// Execute the query
|
||||
while (!Stop)
|
||||
{
|
||||
if (!CancelToken.IsCancellationRequested)
|
||||
{
|
||||
try
|
||||
{
|
||||
int messageCount = 0;
|
||||
List<JObject> messages = new List<JObject>();
|
||||
|
||||
// Lets get whats in the queue
|
||||
lock (_locker)
|
||||
{
|
||||
messageCount = _jsonQueue.Count;
|
||||
|
||||
// Time to flush?
|
||||
if (messageCount >= _flushSize || (DateTime.UtcNow - lastFlushTime).Seconds >= _idleFlushTimeSeconds)
|
||||
{
|
||||
messages = _jsonQueue.Take(messageCount).ToList();
|
||||
_jsonQueue.RemoveRange(0, messageCount);
|
||||
if (messages.Count > 0)
|
||||
_manager.IncrementMessageCount(messages.Count);
|
||||
}
|
||||
}
|
||||
|
||||
TransmitStats(messages);
|
||||
|
||||
if (!Stop)
|
||||
syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (ThreadAbortException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Interlocked.Increment(ref _errorCount);
|
||||
LogManager.GetCurrentClassLogger().Error(ex);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
protected string ExpandField(string fieldName, JObject json)
|
||||
{
|
||||
foreach (var token in json.Children())
|
||||
{
|
||||
string replaceString = "%{" + token.Path + "}";
|
||||
fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
|
||||
}
|
||||
return fieldName;
|
||||
}
|
||||
|
||||
private string BuildMetricPath(string metric, JObject json)
|
||||
{
|
||||
return string.Format("{0}.{1}", ExpandField(_params.Sender, json), ExpandField(metric, json));
|
||||
}
|
||||
|
||||
private void TransmitStats(List<JObject> messages)
|
||||
{
|
||||
// We've got some to send.
|
||||
if (messages.Count > 0)
|
||||
{
|
||||
do
|
||||
{
|
||||
try
|
||||
{
|
||||
int numMessages = messages.Count;
|
||||
foreach (var m in messages)
|
||||
{
|
||||
SendMetrics(m);
|
||||
}
|
||||
messages.RemoveRange(0, numMessages);
|
||||
Interlocked.Add(ref _sentMessages, numMessages);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
LogManager.GetCurrentClassLogger().Error(ex);
|
||||
|
||||
interlockedInsert(messages); // Put the messages back into the queue
|
||||
break;
|
||||
}
|
||||
} while (messages.Count > 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Process all the metrics for this json
|
||||
private void SendMetrics(JObject m)
|
||||
{
|
||||
if (_params.Gauges != null && _params.Gauges.Length > 0)
|
||||
DoGauges(m);
|
||||
if (_params.Counts != null && _params.Counts.Length > 0)
|
||||
DoCounts(m);
|
||||
if (_params.Timings != null && _params.Timings.Length > 0)
|
||||
DoTimings(m);
|
||||
if (_params.Increments != null && _params.Increments.Length > 0)
|
||||
DoIncrements(m);
|
||||
if (_params.Decrements != null && _params.Decrements.Length > 0)
|
||||
DoDecrements(m);
|
||||
}
|
||||
|
||||
// Process the Gauges
|
||||
private void DoGauges(JObject json)
|
||||
{
|
||||
for (int i=0; i<_params.Gauges.Length; i += 2)
|
||||
{
|
||||
string metricPath = BuildMetricPath(_params.Gauges[i], json);
|
||||
string gaugeName = ExpandField(_params.Gauges[i + 1], json);
|
||||
int value;
|
||||
if (int.TryParse(gaugeName, out value))
|
||||
{
|
||||
Metrics.Gauge(metricPath, value);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process the Gauges
|
||||
private void DoTimings(JObject json)
|
||||
{
|
||||
for (int i = 0; i < _params.Timings.Length; i += 2)
|
||||
{
|
||||
string metricPath = BuildMetricPath(_params.Timings[i], json);
|
||||
string timingName = ExpandField(_params.Timings[i + 1], json);
|
||||
int value;
|
||||
if (int.TryParse(timingName, out value))
|
||||
{
|
||||
Metrics.Timer(metricPath, value, _params.SampleRate);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Process the Counts
|
||||
private void DoCounts(JObject json)
|
||||
{
|
||||
for (int i = 0; i < _params.Counts.Length; i += 2)
|
||||
{
|
||||
string metricPath = BuildMetricPath(_params.Counts[i], json);
|
||||
string countName = ExpandField(_params.Counts[i + 1], json);
|
||||
int value;
|
||||
if (int.TryParse(countName, out value))
|
||||
{
|
||||
Metrics.Counter(metricPath, value, _params.SampleRate);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Process the Increments
|
||||
private void DoIncrements(JObject json)
|
||||
{
|
||||
foreach (var metric in _params.Increments)
|
||||
{
|
||||
string metricPath = BuildMetricPath(metric, json);
|
||||
Metrics.Counter(metricPath, 1,_params.SampleRate);
|
||||
}
|
||||
}
|
||||
|
||||
// Process the Increments
|
||||
private void DoDecrements(JObject json)
|
||||
{
|
||||
foreach (var metric in _params.Increments)
|
||||
{
|
||||
string metricPath = BuildMetricPath(metric, json);
|
||||
Metrics.Counter(metricPath, -1, _params.SampleRate);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -34,7 +34,6 @@ namespace TimberWinR.Outputs
|
||||
JObject json = new JObject(
|
||||
new JProperty("stdout",
|
||||
new JObject(
|
||||
new JProperty("queuedMessageCount", _jsonQueue.Count),
|
||||
new JProperty("sentMessageCount", _sentMessages))));
|
||||
|
||||
return json;
|
||||
@@ -68,7 +67,7 @@ namespace TimberWinR.Outputs
|
||||
foreach (JObject obj in messages)
|
||||
{
|
||||
Console.WriteLine(obj.ToString());
|
||||
Interlocked.Increment(ref _sentMessages);
|
||||
_sentMessages++;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
|
||||
@@ -543,88 +543,6 @@ namespace TimberWinR.Parser
|
||||
}
|
||||
}
|
||||
|
||||
public class StatsDOutputParameters : IValidateSchema
|
||||
{
|
||||
public class StatsDGaugeHashException : Exception
|
||||
{
|
||||
public StatsDGaugeHashException()
|
||||
: base("StatsD output 'gauge' must be an array of pairs.")
|
||||
{
|
||||
}
|
||||
}
|
||||
|
||||
public class StatsDCountHashException : Exception
|
||||
{
|
||||
public StatsDCountHashException()
|
||||
: base("StatsD output 'count' must be an array of pairs.")
|
||||
{
|
||||
}
|
||||
}
|
||||
[JsonProperty(PropertyName = "type")]
|
||||
public string InputType { get; set; }
|
||||
[JsonProperty(PropertyName = "sender")]
|
||||
public string Sender { get; set; }
|
||||
[JsonProperty(PropertyName = "namespace")]
|
||||
public string Namespace { get; set; }
|
||||
[JsonProperty(PropertyName = "host")]
|
||||
public string Host { get; set; }
|
||||
[JsonProperty(PropertyName = "port")]
|
||||
public int Port { get; set; }
|
||||
[JsonProperty(PropertyName = "interval")]
|
||||
public int Interval { get; set; }
|
||||
[JsonProperty(PropertyName = "flush_size")]
|
||||
public int FlushSize { get; set; }
|
||||
[JsonProperty(PropertyName = "idle_flush_time")]
|
||||
public int IdleFlushTimeInSeconds { get; set; }
|
||||
[JsonProperty(PropertyName = "max_queue_size")]
|
||||
public int MaxQueueSize { get; set; }
|
||||
[JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
|
||||
public bool QueueOverflowDiscardOldest { get; set; }
|
||||
[JsonProperty(PropertyName = "threads")]
|
||||
public int NumThreads { get; set; }
|
||||
[JsonProperty(PropertyName = "sample_rate")]
|
||||
public double SampleRate { get; set; }
|
||||
[JsonProperty(PropertyName = "increment")] // Array: metric names
|
||||
public string[] Increments { get; set; }
|
||||
[JsonProperty(PropertyName = "decrement")] // Array: metric names
|
||||
public string[] Decrements { get; set; }
|
||||
[JsonProperty(PropertyName = "gauge")] // Hash: metric_name => gauge
|
||||
public string[] Gauges { get; set; }
|
||||
[JsonProperty(PropertyName = "count")] // Hash: metric_name => count
|
||||
public string[] Counts { get; set; }
|
||||
[JsonProperty(PropertyName = "timing")] // Hash: metric_name => count
|
||||
public string[] Timings { get; set; }
|
||||
|
||||
public StatsDOutputParameters()
|
||||
{
|
||||
SampleRate = 1;
|
||||
Port = 8125;
|
||||
Host = "localhost";
|
||||
Interval = 5000;
|
||||
FlushSize = 5000;
|
||||
IdleFlushTimeInSeconds = 10;
|
||||
QueueOverflowDiscardOldest = true;
|
||||
MaxQueueSize = 50000;
|
||||
NumThreads = 1;
|
||||
Namespace = "timberwinr";
|
||||
Sender = System.Environment.MachineName.ToLower() + "." +
|
||||
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
|
||||
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
|
||||
.GetValue("Domain", "")
|
||||
.ToString().ToLower();
|
||||
}
|
||||
|
||||
public void Validate()
|
||||
{
|
||||
if (Gauges != null && Gauges.Length % 2 != 0)
|
||||
throw new StatsDGaugeHashException();
|
||||
|
||||
if (Counts != null && Counts.Length % 2 != 0)
|
||||
throw new StatsDCountHashException();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
public class ElasticsearchOutputParameters
|
||||
{
|
||||
const string IndexDatePattern = "(%\\{(?<format>[^\\}]+)\\})";
|
||||
@@ -750,8 +668,6 @@ namespace TimberWinR.Parser
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
public class StdoutOutputParameters
|
||||
{
|
||||
[JsonProperty(PropertyName = "interval")]
|
||||
@@ -813,9 +729,6 @@ namespace TimberWinR.Parser
|
||||
|
||||
[JsonProperty("File")]
|
||||
public FileOutputParameters[] File { get; set; }
|
||||
|
||||
[JsonProperty("StatsD")]
|
||||
public StatsDOutputParameters[] StatsD { get; set; }
|
||||
}
|
||||
|
||||
public class InputSources
|
||||
@@ -890,6 +803,9 @@ namespace TimberWinR.Parser
|
||||
[JsonProperty("remove_tag")]
|
||||
public string[] RemoveTag { get; set; }
|
||||
|
||||
[JsonProperty("rename")]
|
||||
public string[] Rename { get; set; }
|
||||
|
||||
public override void Validate()
|
||||
{
|
||||
if (Match == null || Match.Length % 2 != 0)
|
||||
|
||||
@@ -3,11 +3,6 @@
|
||||
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
|
||||
|
||||
Version / Date
|
||||
### 1.3.26.0 - 2015-05-15
|
||||
1. Added StatsD outputter
|
||||
2. Fixed shutdown hang if shutdown was received before service was fully started up.
|
||||
3. Closed issue [#36](https://github.com/Cimpress-MCP/TimberWinR/issues/36)
|
||||
|
||||
### 1.3.25.0 - 2015-04-30
|
||||
1. Fixed Issue [#49](https://github.com/Cimpress-MCP/TimberWinR/issues/49)
|
||||
2. Fixed potential non-thread safe when renaming properties
|
||||
|
||||
@@ -60,21 +60,13 @@
|
||||
<Reference Include="NLog">
|
||||
<HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="RapidRegex.Core, Version=1.0.0.4, Culture=neutral, processorArchitecture=MSIL">
|
||||
<SpecificVersion>False</SpecificVersion>
|
||||
<HintPath>..\packages\RapidRegex.Core.1.0.0.4\lib\net40\RapidRegex.Core.dll</HintPath>
|
||||
<Reference Include="RapidRegex.Core">
|
||||
<HintPath>..\packages\RapidRegex.Core.1.0.0.2\lib\net40\RapidRegex.Core.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="RestSharp">
|
||||
<HintPath>..\packages\RestSharp.105.0.0\lib\net4\RestSharp.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="StatsdClient">
|
||||
<HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="StatsdClient.Configuration">
|
||||
<HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.Configuration.dll</HintPath>
|
||||
</Reference>
|
||||
<Reference Include="System" />
|
||||
<Reference Include="System.configuration" />
|
||||
<Reference Include="System.Core" />
|
||||
<Reference Include="System.Linq.Dynamic">
|
||||
<HintPath>..\packages\System.Linq.Dynamic.1.0.4\lib\net40\System.Linq.Dynamic.dll</HintPath>
|
||||
@@ -122,7 +114,6 @@
|
||||
<Compile Include="Manager.cs" />
|
||||
<Compile Include="Outputs\Elasticsearch.cs" />
|
||||
<Compile Include="Outputs\OutputSender.cs" />
|
||||
<Compile Include="Outputs\StatsD.cs" />
|
||||
<Compile Include="Outputs\Redis.cs" />
|
||||
<Compile Include="Outputs\File.cs" />
|
||||
<Compile Include="Outputs\Stdout.cs" />
|
||||
@@ -148,7 +139,6 @@
|
||||
</Content>
|
||||
<None Include="mdocs\Codec.md" />
|
||||
<None Include="mdocs\DateFilter.md" />
|
||||
<None Include="mdocs\StatsD.md" />
|
||||
<None Include="mdocs\Filters.md" />
|
||||
<None Include="mdocs\GeoIPFilter.md" />
|
||||
<None Include="mdocs\Generator.md" />
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
The Elasticsearch output passes on data directly to Elasticsearch.
|
||||
|
||||
## Parameters
|
||||
The following parameters are allowed when configuring the Elasticsearch output.
|
||||
The following parameters are allowed when configuring the Redis output.
|
||||
|
||||
| Parameter | Type | Description | Details | Default |
|
||||
| :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- |
|
||||
|
||||
@@ -33,7 +33,7 @@ The following operations are allowed when mutating a field.
|
||||
| *remove_field* | property:array |If the filter is successful, remove arbitrary fields from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
|
||||
| *remove_tag* | property:array |If the filter is successful, remove arbitrary tags from this event. Field names can be dynamic and include parts of the event using the %{field} syntax.
|
||||
| *rename* | property:array |Rename one or more fields
|
||||
| *type* | property:string |Type to which this filter applies, if empty, applies to all types.
|
||||
| *type* | property:string |Type to which this filter applyes, if empty, applies to all types.
|
||||
|
||||
## Operation Details
|
||||
### match
|
||||
|
||||
@@ -1,94 +0,0 @@
|
||||
# Output: StatsD
|
||||
|
||||
The StatsD output passes on data directly to StatsD. (https://github.com/etsy/statsd)
|
||||
|
||||
## Parameters
|
||||
The following parameters are allowed when configuring the StatsD output.
|
||||
|
||||
| Parameter | Type | Description | Details | Default |
|
||||
| :-------------|:---------|:------------------------------------------------------------| :--------------------------- | :-- |
|
||||
| *count* | string | Array of (metric_name, gauge name) pairs counted | Must come in pairs | |
|
||||
| *decrement* | string | Array of metrics to be decremented | | |
|
||||
| *flush_size* | integer | Maximum number of messages before flushing | | 50000 |
|
||||
| *gauge* | string | Array of (metric_name, gauge name) pairs gauged | Must come in pairs | |
|
||||
| *host* | string | Hostname or IP of StatsD server | localhost | |
|
||||
| *idle_flush_time* | integer | Maximum number of seconds elapsed before triggering a flush | | 10 |
|
||||
| *increment* | string | Array of metrics to be incremented | | |
|
||||
| *interval* | integer | Interval in milliseconds to sleep between sends | Interval | 5000 |
|
||||
| *max_queue_size* | integer | Maximum StatsD queue depth | | 50000 |
|
||||
| *namespace* | string | Namespace for stats | timberwinr | |
|
||||
| *port* | integer | StatsD port number | This port must be open | 8125 |
|
||||
| *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true |
|
||||
| *sample_rate* | integer | StatsD sample rate | | 1 |
|
||||
| *sender* | string | Sender name | FQDN | |
|
||||
| *threads* | string | Number of Threads processing messages | | 1 |
|
||||
| *timing* | string | Array of (metric_name, timing_name) pairs timed | Must come in pairs | |
|
||||
| *type* | string |Type to which this filter applies, if empty, applies to all types.
|
||||
|
||||
### Example Usage
|
||||
Example Input: Tail an apache log file, and record counts for bytes and increments for response codes.
|
||||
|
||||
sample-apache.log (snip)
|
||||
```
|
||||
180.76.5.25 - - [13/May/2015:17:02:26 -0700] "GET /frameset.htm HTTP/1.1" 404 89 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" "www.redlug.com"
|
||||
208.115.113.94 - - [13/May/2015:17:03:55 -0700] "GET /robots.txt HTTP/1.1" 200 37 "-" "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" "redlug.com"
|
||||
208.115.113.94 - - [13/May/2015:17:03:55 -0700] "GET /robots.txt HTTP/1.1" 200 37 "-" "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" "www.redlug.com"
|
||||
```
|
||||
|
||||
Note: [COMBINEDAPACHELOG](https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns) is a standard
|
||||
Grok Pattern.
|
||||
|
||||
TimberWinR configuration
|
||||
|
||||
```json
|
||||
{
|
||||
"TimberWinR": {
|
||||
"Inputs": {
|
||||
"TailFiles": [
|
||||
{
|
||||
"interval": 5,
|
||||
"logSource": "apache log files",
|
||||
"location": "..\\sample-apache.log",
|
||||
"recurse": -1
|
||||
}
|
||||
]
|
||||
},
|
||||
"Filters": [
|
||||
{
|
||||
"grok": {
|
||||
"type": "Win32-TailLog",
|
||||
"match": [
|
||||
"Text",
|
||||
"%{COMBINEDAPACHELOG}"
|
||||
]
|
||||
}
|
||||
}
|
||||
],
|
||||
"Outputs": {
|
||||
"StatsD": [
|
||||
{
|
||||
"type": "Win32-TailLog",
|
||||
"port": 8125,
|
||||
"host": "stats.mycompany.svc",
|
||||
"increment": ["apache.response.%{response}"],
|
||||
"count": ["apache.bytes", "%{bytes}"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
```
|
||||
|
||||
Assuming your FQDN is something like mymachine.mycompany.com, you should see the following in Graphite:
|
||||
|
||||
```
|
||||
stats.counters.timberwinr.mymachine.mycompany.com.apache.bytes.count
|
||||
stats.counters.timberwinr.mymachine.mycompany.com.apache.bytes.rate
|
||||
stats.counters.timberwinr.mymachine.mycompany.com.apache.response.200.count
|
||||
stats.counters.timberwinr.mymachine.mycompany.com.apache.response.200.rate
|
||||
stats.counters.timberwinr.mymachine.mycompany.com.apache.response.404.count
|
||||
stats.counters.timberwinr.mymachine.mycompany.com.apache.response.404.rate
|
||||
...
|
||||
...
|
||||
```
|
||||
@@ -7,9 +7,8 @@
|
||||
<package id="NEST" version="1.3.1" targetFramework="net40" />
|
||||
<package id="Newtonsoft.Json" version="6.0.5" targetFramework="net40" />
|
||||
<package id="NLog" version="3.2.0.0" targetFramework="net40" />
|
||||
<package id="RapidRegex.Core" version="1.0.0.4" targetFramework="net40" />
|
||||
<package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
|
||||
<package id="RestSharp" version="105.0.0" targetFramework="net40" />
|
||||
<package id="StatsdClient" version="1.0.0.19" targetFramework="net40" />
|
||||
<package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
|
||||
<package id="Topshelf" version="3.1.4" targetFramework="net40" />
|
||||
</packages>
|
||||
@@ -54,8 +54,6 @@
|
||||
<File Id="default.json" Source="$(var.TimberWinR.ServiceHost.TargetDir)\default.json" />
|
||||
<File Id="Newtonsoft.Json.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\Newtonsoft.Json.dll" />
|
||||
<File Id="Nlog.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\Nlog.dll" />
|
||||
<File Id="StatsdClient.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\StatsdClient.dll" />
|
||||
<File Id="StatsdClient.Configuration.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\StatsdClient.Configuration.dll" />
|
||||
<File Id="RestSharp.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\RestSharp.dll" />
|
||||
<File Id="RapidRegex.Core.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\RapidRegex.Core.dll" />
|
||||
<File Id="TimberWinR.dll" Source="$(var.TimberWinR.ServiceHost.TargetDir)\TimberWinR.dll" />
|
||||
|
||||
@@ -2,6 +2,6 @@ $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in
|
||||
$installerType = 'msi' #only one of these: exe, msi, msu
|
||||
$scriptPath = $(Split-Path $MyInvocation.MyCommand.Path)
|
||||
$fileFullPath = Join-Path $scriptPath 'TimberWinR-${version}.0.msi'
|
||||
$silentArgs = '{E001D138-669B-4604-88C5-02C756461C15} /quiet'
|
||||
$silentArgs = '{267F6E29-2B1B-4BFF-BE88-AC6DE2B50E06} /quiet'
|
||||
$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
|
||||
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "fileFullPath" -validExitCodes $validExitCodes
|
||||
|
||||
Reference in New Issue
Block a user