TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated doc

Browse Source
This commit is contained in:
Eric Fontana
2014-07-29 09:35:35 -04:00
parent 640e18e8c4
commit 916c7a4e27
10 changed files with 188 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
README.md
View File

@@ -1,4 +1,68 @@
TimberWinR
==========
A Native Windows to Redis Logstash Agent which runs as a service.
A Native Windows to Redis Logstash Agent
## Why have TimberWinR?
TimberWinR is a native .NET implementation utilizing Microsoft's [LogParser](http://technet.microsoft.com/en-us/scriptcenter/dd919274.aspx). This means
no JVM/JRuby is required, and LogParser does all the heavy lifting. TimberWinR collects
the data from LogParser and ships it to Logstash via Redis.
## Configuration
TimberWinR reads a JSON configuration file, an example file is shown here:
{
"TimberWinR": {
"Inputs": {
"WindowsEvents": [
{
"source": "System,Application",
"binaryFormat": "PRINT",
"resolveSIDS": true
}
]
},
"Outputs": {
"Redis": [
{
"host": [
"server1.host.com"
]
}
]
}
}
This configuration collects Events from the Windows Event Logs (System, Application) and forwards them
to Redis.
## what is Markdown?
see [Wikipedia](http://en.wikipedia.org/wiki/Markdown)
> Markdown is a lightweight markup language, originally created by John Gruber and Aaron Swartz allowing people "to write using an easy-to-read, easy-to-write plain text format, then convert it to structurally valid XHTML (or HTML)".
----
## usage
1. Write markdown text in this textarea.
2. Click 'HTML Preview' button.
----
## markdown quick reference
# headers
*emphasis*
**strong**
* list
>block quote
code (4 spaces indent)
[links](http://wikipedia.org)
----
## changelog
* 17-Feb-2013 re-design
----
## thanks
* [markdown-js](https://github.com/evilstreak/markdown-js)

18
TimberWinR.ServiceHost/config.json
View File

@@ -28,9 +28,12 @@
},
"Outputs": {
"Redis": [
{
{
"threads": 1,
"interval": 5000,
"batch_count": 500,
"host": [
"logaggregator.vistaprint.svc"
"tstlexiceapp006.vistaprint.svc"
]
}
]
@@ -42,7 +45,7 @@
"match": [
"Message",
""
],
],
"remove_field": [
"ComputerName"
]
@@ -78,13 +81,15 @@
},
{
"date": {
"condition": "[type] == \"Win32-IISLog\"",
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
],
"target": "UtcTimestamp",
"add_field": [
"UtcTimestamp"
],
"convertToUTC": true
}
},
@@ -99,8 +104,7 @@
"SID", "Username"
]
}
},
}
]
}
}

24
TimberWinR/Filters/DateFilter.cs
View File

@@ -22,6 +22,7 @@ namespace TimberWinR.Parser
if (Matches(json))
{
ApplyFilter(json);
AddFields(json);
}
return true;
@@ -48,9 +49,28 @@ namespace TimberWinR.Parser
}
}
// copy_field "field1" -> "field2"
private void AddFields(Newtonsoft.Json.Linq.JObject json)
{
string srcField = Match[0];
if (AddField != null && AddField.Length > 0)
{
for (int i = 0; i < AddField.Length; i++)
{
string dstField = ExpandField(AddField[i], json);
if (json[srcField] != null)
AddOrModify(json, dstField, json[srcField]);
}
}
}
private bool Matches(Newtonsoft.Json.Linq.JObject json)
{
string field = Match[0];
string field = Match[0];
CultureInfo ci = new CultureInfo(Locale);
JToken token = null;
if (json.TryGetValue(field, out token))
@@ -66,7 +86,7 @@ namespace TimberWinR.Parser
var pattern = resolver.ResolveToRegex(exprArray[i]);
exprArray[i] = pattern;
}
if (DateTime.TryParseExact(text, exprArray, CultureInfo.InvariantCulture,DateTimeStyles.None, out ts))
if (DateTime.TryParseExact(text, exprArray, ci,DateTimeStyles.None, out ts))
AddOrModify(json, ts);
}
return true; // Empty field is no match

4
TimberWinR/Filters/GrokFilter.cs
View File

@@ -40,7 +40,7 @@ namespace TimberWinR.Parser
if (Matches(json))
{
AddFields(json);
AddTags(json);
AddTags(json);
RemoveFields(json);
RemoveTags(json);
return true;
@@ -91,6 +91,8 @@ namespace TimberWinR.Parser
}
}
private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
{
if (RemoveField != null && RemoveField.Length > 0)

2
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -36,7 +36,7 @@ namespace TimberWinR.Inputs
public override void Shutdown()
{
base.Shutdown();
}
}
private void IISW3CWatcher()
{

19
TimberWinR/Inputs/InputListener.cs
View File

@@ -32,6 +32,22 @@ namespace TimberWinR.Inputs
.ToString();
}
protected string ToPrintable(string inputString)
{
string asAscii = Encoding.ASCII.GetString(
Encoding.Convert(
Encoding.UTF8,
Encoding.GetEncoding(
Encoding.ASCII.EncodingName,
new EncoderReplacementFallback(string.Empty),
new DecoderExceptionFallback()
),
Encoding.UTF8.GetBytes(inputString)
)
);
return asAscii;
}
public void Finished()
{
FinishedEvent.Set();
@@ -57,6 +73,9 @@ namespace TimberWinR.Inputs
if (json["host"] == null)
json.Add(new JProperty("host", _computerName));
if (json["@version"] == null)
json.Add(new JProperty("@version", 1));
if (json["@timestamp"] == null)
json.Add(new JProperty("@timestamp", DateTime.UtcNow));
}

2
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -81,6 +81,8 @@ namespace TimberWinR.Inputs
foreach (var field in _arguments.Fields)
{
object v = record.getValue(field.Name);
if (field.Name == "Data")
v = ToPrintable(v.ToString());
json.Add(new JProperty(field.Name, v));
}

4
TimberWinR/Manager.cs
View File

@@ -58,7 +58,7 @@ namespace TimberWinR
{
foreach (var ro in Config.RedisOutputs)
{
var redis = new RedisOutput(this, ro.Host, cancelToken, ro.Index, ro.Port, ro.Timeout);
var redis = new RedisOutput(this, ro, cancelToken);
Outputs.Add(redis);
}
}
@@ -108,7 +108,7 @@ namespace TimberWinR
return new FileTarget
{
ArchiveEvery = FileArchivePeriod.None,
ArchiveAboveSize = 10 * 1024 * 1024,
ArchiveAboveSize = 5 * 1024 * 1024,
MaxArchiveFiles = 5,
BufferSize = 10,
FileName = Path.Combine(logPath, "TimberWinR", "TimberWinR.txt"),

50
TimberWinR/Outputs/Redis.cs
View File

@@ -23,10 +23,12 @@ namespace TimberWinR.Outputs
private readonly int _timeout;
private readonly object _locker = new object();
private readonly List<string> _jsonQueue;
readonly Task _consumerTask;
// readonly Task _consumerTask;
private readonly string[] _redisHosts;
private int _redisHostIndex;
private TimberWinR.Manager _manager;
private readonly int _batchCount;
private readonly int _interval;
/// <summary>
/// Get the next client
@@ -50,9 +52,8 @@ namespace TimberWinR.Outputs
return client;
}
catch (Exception ex)
catch (Exception )
{
}
numTries++;
}
@@ -60,18 +61,24 @@ namespace TimberWinR.Outputs
return null;
}
public RedisOutput(TimberWinR.Manager manager, string[] redisHosts, CancellationToken cancelToken, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000)
public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken) //string[] redisHosts, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000, int batch_count = 10)
: base(cancelToken)
{
_batchCount = ro.BatchCount;
_manager = manager;
_redisHostIndex = 0;
_redisHosts = redisHosts;
_redisHosts = ro.Host;
_jsonQueue = new List<string>();
_port = port;
_timeout = timeout;
_logstashIndexName = logstashIndexName;
_consumerTask = new Task(RedisSender, cancelToken);
_consumerTask.Start();
_port = ro.Port;
_timeout = ro.Timeout;
_logstashIndexName = ro.Index;
_interval = ro.Interval;
for (int i = 0; i < ro.NumThreads; i++)
{
var redisThread = new Task(RedisSender, cancelToken);
redisThread.Start();
}
}
@@ -82,10 +89,10 @@ namespace TimberWinR.Outputs
protected override void MessageReceivedHandler(JObject jsonMessage)
{
if (_manager.Config.Filters != null)
ProcessGroks(jsonMessage);
ApplyFilters(jsonMessage);
var message = jsonMessage.ToString();
LogManager.GetCurrentClassLogger().Info(message);
LogManager.GetCurrentClassLogger().Trace(message);
lock (_locker)
{
@@ -93,11 +100,11 @@ namespace TimberWinR.Outputs
}
}
private void ProcessGroks(JObject json)
private void ApplyFilters(JObject json)
{
foreach (var grok in _manager.Config.Filters)
foreach (var filter in _manager.Config.Filters)
{
grok.Apply(json);
filter.Apply(json);
}
}
@@ -111,8 +118,8 @@ namespace TimberWinR.Outputs
string[] messages;
lock (_locker)
{
messages = _jsonQueue.ToArray();
_jsonQueue.Clear();
messages = _jsonQueue.Take(_batchCount).ToArray();
_jsonQueue.RemoveRange(0, messages.Length);
}
if (messages.Length > 0)
@@ -128,15 +135,18 @@ namespace TimberWinR.Outputs
if (client != null)
{
client.StartPipe();
LogManager.GetCurrentClassLogger()
.Info("Sending {0} Messages to {1}", messages.Length, client.Host);
foreach (string jsonMessage in messages)
{
try
{
{
client.RPush(_logstashIndexName, jsonMessage);
}
catch (SocketException)
catch (SocketException ex)
{
LogManager.GetCurrentClassLogger().Warn(ex);
}
}
client.EndPipe();
@@ -156,7 +166,7 @@ namespace TimberWinR.Outputs
}
}
}
System.Threading.Thread.Sleep(1000);
System.Threading.Thread.Sleep(_interval);
}
}
}

34
TimberWinR/Parser.cs
View File

@@ -121,6 +121,15 @@ namespace TimberWinR.Parser
json[fieldName] = fieldValue;
}
protected void AddOrModify(JObject json, string fieldName, JToken token)
{
if (json[fieldName] == null)
json.Add(fieldName, token);
else
json[fieldName] = token;
}
protected string ExpandField(string fieldName, JObject json)
{
foreach (var token in json.Children())
@@ -228,6 +237,8 @@ namespace TimberWinR.Parser
StringsSep = "|";
FormatMsg = true;
FullText = true;
BinaryFormat = FormatKinds.ASC;
Fields = new List<Field>();
Fields.Add(new Field("EventLog", "string"));
Fields.Add(new Field("RecordNumber", "int"));
@@ -374,6 +385,12 @@ namespace TimberWinR.Parser
public int Port { get; set; }
[JsonProperty(PropertyName = "timeout")]
public int Timeout { get; set; }
[JsonProperty(PropertyName = "batch_count")]
public int BatchCount { get; set; }
[JsonProperty(PropertyName = "threads")]
public int NumThreads { get; set; }
[JsonProperty(PropertyName = "interval")]
public int Interval { get; set; }
public RedisOutput()
{
@@ -381,6 +398,9 @@ namespace TimberWinR.Parser
Index = "logstash";
Host = new string[] {"localhost"};
Timeout = 10000;
BatchCount = 10;
NumThreads = 1;
Interval = 5000;
}
}
@@ -435,7 +455,7 @@ namespace TimberWinR.Parser
public string[] AddTag { get; set; }
[JsonProperty("add_field")]
public string[] AddField { get; set; }
public string[] AddField { get; set; }
[JsonProperty("remove_field")]
public string[] RemoveField { get; set; }
@@ -474,6 +494,9 @@ namespace TimberWinR.Parser
[JsonProperty("condition")]
public string Condition { get; set; }
[JsonProperty("locale")]
public string Locale { get; set; }
[JsonProperty("match")]
public string[] Match { get; set; }
@@ -486,6 +509,9 @@ namespace TimberWinR.Parser
[JsonProperty("pattern")]
public string[] Patterns { get; set; }
[JsonProperty("add_field")]
public string[] AddField { get; set; }
public override void Validate()
{
if (Match == null || Match.Length < 2)
@@ -494,6 +520,12 @@ namespace TimberWinR.Parser
if (string.IsNullOrEmpty(Target))
throw new DateFilterTargetException();
}
public DateFilter()
{
Target = "timestamp";
Locale = "en-US";
}
}
public partial class Mutate : LogstashFilter