TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Multiline codec closed issue #23

Browse Source
This commit is contained in:
Eric Fontana
2015-01-12 12:35:20 -05:00
parent 884efffb25
commit dec51efccd
17 changed files with 623 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.18.0")]
[assembly: AssemblyFileVersion("1.3.18.0")]
[assembly: AssemblyVersion("1.3.19.0")]
[assembly: AssemblyFileVersion("1.3.19.0")]

13
TimberWinR.UnitTests/Multiline1.txt Normal file
View File

@@ -0,0 +1,13 @@
multiline1 \
ml1_1 \
ml1_2 \
ml1_2
singleline1
singleline2
multiline2 \
ml2_1 \
ml2_2
multiline3 \
ml3_1 \
ml3_2
singleline3

19
TimberWinR.UnitTests/Multiline2.txt Normal file
View File

@@ -0,0 +1,19 @@
2015-01-07 13:14:26,572 TEST DEBUG [THREAD : 25] - Sending message to TServer - tcp://10.1111.11.111:1111
'RequestAttachUserData' ('30')
message attributes:
AttributeConnID [long] = 00890
AttributeReferenceID [int] = 88
AttributeThisDN [str] = "2214"
AttributeUserData [bstr] = KVList:
'ActivityID' [str] = "1-XXXXXX"
2015-01-07 13:14:26,574 TEST DEBUG [THREAD : 25] - Writing message RequestAttachUserData in 'proxy1' via '.StatePrimary proxy: proxy1'
2015-01-07 13:14:26,575 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection
2015-01-07 13:20:31,665 TEST DEBUG [THREAD : SelectorThread] - Proxy got message 'EventOnHook' ('87')
message attributes:
AttributeEventSequenceNumber [long] = 4899493
Time = ComplexClass(TimeStamp):
AttributeTimeinuSecs [int] = 573000
AttributeTimeinSecs [int] = 1420644031
AttributeThisDN [str] = "2214"
. Processing with state .StatePrimary proxy: proxy1
2015-01-07 14:14:26,666 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection

119
TimberWinR.UnitTests/MultilineTests.cs Normal file
View File

@@ -0,0 +1,119 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json;
using NUnit.Framework;
using TimberWinR.Inputs;
using TimberWinR.Parser;
using Newtonsoft.Json.Linq;
using System.Diagnostics;
namespace TimberWinR.UnitTests
{
[TestFixture]
public class MultilineTests
{
[Test(Description = "Test using next")]
public void TestMultiline1()
{
using (StreamReader sr = new StreamReader("Multiline1.txt"))
{
List<JObject> events = new List<JObject>();
Console.SetIn(sr);
Stdin sin = new Stdin();
sin.Codec = new Codec();
sin.Codec.Pattern = "\\\\$";
sin.Codec.What = Codec.WhatType.next;
sin.Codec.Type = Codec.CodecType.multiline;
var cancelTokenSource = new CancellationTokenSource();
using (var syncHandle = new ManualResetEventSlim())
{
try
{
StdinListener sl = new StdinListener(sin, cancelTokenSource.Token);
sl.OnMessageRecieved += o =>
{
events.Add(o);
if (events.Count >= 6)
cancelTokenSource.Cancel();
};
if (!cancelTokenSource.Token.IsCancellationRequested)
syncHandle.Wait(TimeSpan.FromSeconds(10000), cancelTokenSource.Token);
}
catch (OperationCanceledException oex)
{
}
}
Assert.AreEqual(events.Count, 6);
Assert.AreEqual(events[0]["message"].ToString(), "multiline1 \\\nml1_1 \\\nml1_2 \\\nml1_2 ");
Assert.AreEqual(events[1]["message"].ToString(), "singleline1");
Assert.AreEqual(events[2]["message"].ToString(), "singleline2");
Assert.AreEqual(events[3]["message"].ToString(), "multiline2 \\\nml2_1 \\\nml2_2");
Assert.AreEqual(events[4]["message"].ToString(), "multiline3 \\\nml3_1 \\\nml3_2");
Assert.AreEqual(events[5]["message"].ToString(), "singleline3");
}
}
[Test(Description = "Test using previous")]
public void TestMultiline2()
{
using (StreamReader sr = new StreamReader("Multiline2.txt"))
{
List<JObject> events = new List<JObject>();
Console.SetIn(sr);
Stdin sin = new Stdin();
sin.Codec = new Codec();
sin.Codec.Pattern = "^(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3})(.*)$";
sin.Codec.What = Codec.WhatType.previous;
sin.Codec.Type = Codec.CodecType.multiline;
sin.Codec.Negate = true;
var cancelTokenSource = new CancellationTokenSource();
using (var syncHandle = new ManualResetEventSlim())
{
try
{
StdinListener sl = new StdinListener(sin, cancelTokenSource.Token);
sl.OnMessageRecieved += o =>
{
events.Add(o);
if (events.Count >= 4)
cancelTokenSource.Cancel();
};
if (!cancelTokenSource.Token.IsCancellationRequested)
syncHandle.Wait(TimeSpan.FromSeconds(10000), cancelTokenSource.Token);
}
catch (OperationCanceledException oex)
{
}
}
Assert.AreEqual(events.Count, 4);
Assert.AreEqual(events[0]["message"].ToString(), "2015-01-07 13:14:26,572 TEST DEBUG [THREAD : 25] - Sending message to TServer - tcp://10.1111.11.111:1111\n'RequestAttachUserData' ('30')\nmessage attributes:\nAttributeConnID [long] = 00890\nAttributeReferenceID [int] = 88\nAttributeThisDN [str] = \"2214\"\nAttributeUserData [bstr] = KVList: \n\t\t'ActivityID' [str] = \"1-XXXXXX\"");
Assert.AreEqual(events[1]["message"].ToString(), "2015-01-07 13:14:26,574 TEST DEBUG [THREAD : 25] - Writing message RequestAttachUserData in 'proxy1' via '.StatePrimary proxy: proxy1'");
Assert.AreEqual(events[2]["message"].ToString(), "2015-01-07 13:14:26,575 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection");
Assert.AreEqual(events[3]["message"].ToString(), "2015-01-07 13:20:31,665 TEST DEBUG [THREAD : SelectorThread] - Proxy got message 'EventOnHook' ('87')\nmessage attributes:\nAttributeEventSequenceNumber [long] = 4899493\nTime = ComplexClass(TimeStamp):\n\tAttributeTimeinuSecs [int] = 573000\n\tAttributeTimeinSecs [int] = 1420644031\nAttributeThisDN [str] = \"2214\"\n. Processing with state .StatePrimary proxy: proxy1");
}
}
}
}

9
TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
View File

@@ -64,6 +64,7 @@
<Compile Include="Inputs\IisW3CRowReaderTests.cs" />
<Compile Include="JsonFilterTests.cs" />
<Compile Include="GrokFilterTests.cs" />
<Compile Include="MultilineTests.cs" />
<Compile Include="Parser\ElasticsearchOutputTests.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="TestBase.cs" />
@@ -82,6 +83,14 @@
<SubType>Designer</SubType>
</None>
</ItemGroup>
<ItemGroup>
<Content Include="Multiline2.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="Multiline1.txt">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<!-- To modify your build process, add your task inside one of the targets below and uncomment it.
Other similar extension points exist, see Microsoft.Common.targets.

2
TimberWinR/Inputs/InputListener.cs
View File

@@ -18,7 +18,7 @@ namespace TimberWinR.Inputs
private string _typeName;
public AutoResetEvent FinishedEvent { get; set; }
public string CheckpointFileName { get; set; }
public string InputType
{
get { return _typeName; }

249
TimberWinR/Inputs/LogsListener.cs
View File

@@ -2,7 +2,9 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Configuration;
using System.Runtime.InteropServices;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using System.IO;
@@ -16,9 +18,150 @@ using NLog;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using TextLineInputFormat = Interop.MSUtil.COMTextLineInputContextClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
using TimberWinR.Parser;
namespace TimberWinR.Inputs
{
public class LogsFileDatabase
{
private static readonly object _locker = new object();
private List<LogsFileDatabaseEntry> Entries { get; set; }
private string DatabaseDirectory { get; set; }
public string DatabaseFileName
{
get { return Path.Combine(DatabaseDirectory, ".timberwinrdb"); }
}
public static Manager Manager { get; set; }
private static LogsFileDatabase instance;
private bool ExistingFile(string logName)
{
lock (_locker)
{
return ExistingFileTest(logName);
}
}
private bool ExistingFileTest(string logName)
{
var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
return existingEntry != null;
}
private void RemoveFileEntry(string logName)
{
lock (_locker)
{
var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
if (existingEntry != null)
{
Entries.Remove(existingEntry);
WriteDatabaseFileNoLock();
}
}
}
private LogsFileDatabaseEntry AddFileEntry(string logName, TextLineInputFormat fmt)
{
LogsFileDatabaseEntry de = new LogsFileDatabaseEntry();
lock (_locker)
{
var lq = new LogQuery();
FileInfo fi = new FileInfo(logName);
de.FileName = logName;
de.Size = fi.Length;
de.SampleTime = DateTime.UtcNow;
de.CreationTime = fi.CreationTimeUtc;
if (fi.Exists)
{
var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
var rcount = lq.Execute(qcount, fmt);
var qr = rcount.getRecord();
var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
de.MaxRecords = lrn;
}
Entries.Add(de);
WriteDatabaseFileNoLock();
}
return de;
}
public static LogsFileDatabaseEntry AddLogFile(string logName, TextLineInputFormat fmt)
{
Instance.RemoveFileEntry(logName); // Remove if already exists, otherwise ignores.
return Instance.AddFileEntry(logName, fmt);
}
public static LogsFileDatabase Instance
{
get
{
if (instance == null)
{
instance = new LogsFileDatabase(Manager.LogfileDir);
lock (_locker)
{
if (!Directory.Exists(instance.DatabaseDirectory))
{
Directory.CreateDirectory(instance.DatabaseDirectory);
}
if (File.Exists(instance.DatabaseFileName))
instance.ReadDatabaseNoLock();
else
instance.WriteDatabaseFileNoLock();
}
}
return instance;
}
}
private void ReadDatabaseNoLock()
{
JsonSerializer serializer = new JsonSerializer();
if (File.Exists(DatabaseFileName))
Entries = JsonConvert.DeserializeObject<List<LogsFileDatabaseEntry>>(File.ReadAllText(DatabaseFileName));
}
private void WriteDatabaseFileNoLock()
{
File.WriteAllText(DatabaseFileName, JsonConvert.SerializeObject(instance.Entries), Encoding.UTF8);
}
private void ReadDatabaseLock()
{
lock (_locker)
{
ReadDatabaseNoLock();
}
}
private void WriteDatabaseLock()
{
lock (_locker)
{
WriteDatabaseFileNoLock();
}
}
private LogsFileDatabase(string databaseDirectory)
{
DatabaseDirectory = databaseDirectory;
Entries = new List<LogsFileDatabaseEntry>();
}
}
public class LogsFileDatabaseEntry
{
public string FileName { get; set; }
public Int64 MaxRecords { get; set; }
public DateTime CreationTime { get; set; }
public DateTime SampleTime { get; set; }
public long Size { get; set; }
}
/// <summary>
/// Tail a file.
/// </summary>
@@ -31,14 +174,17 @@ namespace TimberWinR.Inputs
private Dictionary<string, DateTime> _logFileCreationTimes;
private Dictionary<string, DateTime> _logFileSampleTimes;
private Dictionary<string, long> _logFileSizes;
private Codec _codec;
private List<string> _multiline { get; set; }
public bool Stop { get; set; }
public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-FileLog")
{
Stop = false;
_codec = arguments.Codec;
_logFileMaxRecords = new Dictionary<string, Int64>();
_logFileCreationTimes = new Dictionary<string, DateTime>();
_logFileSampleTimes = new Dictionary<string, DateTime>();
@@ -64,15 +210,18 @@ namespace TimberWinR.Inputs
public override JObject ToJson()
{
JObject json = new JObject(
new JProperty("log",
new JObject(
new JProperty("messages", _receivedMessages),
new JProperty("type", InputType),
new JProperty("location", _arguments.Location),
new JProperty("logSource", _arguments.LogSource),
new JProperty("codepage", _arguments.CodePage),
new JProperty("splitLongLines", _arguments.SplitLongLines),
new JProperty("recurse", _arguments.Recurse),
new JProperty("files",
new JArray(from f in _logFileMaxRecords.Keys
select new JValue(f))),
@@ -90,9 +239,96 @@ namespace TimberWinR.Inputs
select new JValue(f)))
)));
if (_codec != null)
{
var cp = new JProperty("codec",
new JArray(
new JObject(
new JProperty("type", _codec.Type.ToString()),
new JProperty("what", _codec.What.ToString()),
new JProperty("negate", _codec.Negate),
new JProperty("multilineTag", _codec.MultilineTag),
new JProperty("pattern", _codec.Pattern))));
json.Add(cp);
}
return json;
}
// return true to cancel codec
private void applyMultilineCodec(string msg)
{
if (_codec.Re == null)
_codec.Re = new Regex(_codec.Pattern);
Match match = _codec.Re.Match(msg);
bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
switch (_codec.What)
{
case Codec.WhatType.previous:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No Match
{
if (_multiline != null)
{
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
_multiline = new List<string>();
_multiline.Add(msg);
}
break;
case Codec.WhatType.next:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No match
{
if (_multiline != null)
{
_multiline.Add(msg);
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
else
{
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
}
break;
}
}
private void FileWatcher(string fileToWatch)
{
var iFmt = new TextLineInputFormat()
@@ -200,8 +436,13 @@ namespace TimberWinR.Inputs
string msg = json["Text"].ToString();
if (!string.IsNullOrEmpty(msg))
{
ProcessJson(json);
_receivedMessages++;
if (_codec != null && _codec.Type == Codec.CodecType.multiline)
applyMultilineCodec(msg);
else
{
ProcessJson(json);
_receivedMessages++;
}
}
var lrn = (Int64)record.getValueEx("Index");

115
TimberWinR/Inputs/StdinListener.cs
View File

@@ -1,21 +1,27 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using TimberWinR.Parser;
namespace TimberWinR.Inputs
{
public class StdinListener : InputListener
{
private Thread _listenThread;
private Codec _codec;
private List<string> _multiline { get; set; }
public StdinListener(CancellationToken cancelToken)
public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-Console")
{
_codec = arguments.Codec;
_listenThread = new Thread(new ThreadStart(ListenToStdin));
_listenThread.Start();
}
@@ -23,13 +29,28 @@ namespace TimberWinR.Inputs
public override JObject ToJson()
{
JObject json = new JObject(
new JProperty("stdin", "enabled"));
new JProperty("stdin", "enabled"));
if (_codec != null)
{
var cp = new JProperty("codec",
new JArray(
new JObject(
new JProperty("type", _codec.Type.ToString()),
new JProperty("what", _codec.What.ToString()),
new JProperty("negate", _codec.Negate),
new JProperty("multilineTag", _codec.MultilineTag),
new JProperty("pattern", _codec.Pattern))));
json.Add(cp);
}
return json;
}
public override void Shutdown()
{
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
base.Shutdown();
}
@@ -38,20 +59,92 @@ namespace TimberWinR.Inputs
LogManager.GetCurrentClassLogger().Info("StdIn Ready");
while (!CancelToken.IsCancellationRequested)
{
{
string line = Console.ReadLine();
if (line != null)
{
string msg = ToPrintable(line);
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
}
else
break;
if (_codec != null && _codec.Type == Codec.CodecType.multiline)
applyMultilineCodec(msg);
else
{
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
}
}
}
Finished();
}
// return true to cancel codec
private void applyMultilineCodec(string msg)
{
if (_codec.Re == null)
_codec.Re = new Regex(_codec.Pattern);
Match match = _codec.Re.Match(msg);
bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
switch (_codec.What)
{
case Codec.WhatType.previous:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No Match
{
if (_multiline != null)
{
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
}
_multiline = new List<string>();
_multiline.Add(msg);
}
break;
case Codec.WhatType.next:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No match
{
if (_multiline != null)
{
_multiline.Add(msg);
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
}
else
{
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
}
}
break;
}
}
}
}

9
TimberWinR/Manager.cs
View File

@@ -62,6 +62,8 @@ namespace TimberWinR
public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken)
{
LogsFileDatabase.Manager = this;
StartedOn = DateTime.UtcNow;
var vfi = new FileInfo(jsonConfigFile);
@@ -99,6 +101,11 @@ namespace TimberWinR
LogManager.GetCurrentClassLogger()
.Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());
LogManager.GetCurrentClassLogger()
.Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
try
{
// Is it a directory?
@@ -211,7 +218,7 @@ namespace TimberWinR
foreach (var stdin in Config.Stdins)
{
var elistner = new StdinListener(cancelToken);
var elistner = new StdinListener(stdin, cancelToken);
Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);

5
TimberWinR/Outputs/Redis.cs
View File

@@ -13,6 +13,7 @@ using System.Threading.Tasks;
using RapidRegex.Core;
using System.Text.RegularExpressions;
using System.Globalization;
using TimberWinR.Parser;
namespace TimberWinR.Outputs
{
@@ -174,10 +175,10 @@ namespace TimberWinR.Outputs
foreach (var filter in _manager.Config.Filters)
{
if (!filter.Apply(json))
{
{
LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
drop = true;
}
}
}
return drop;
}

14
TimberWinR/Outputs/Stdout.cs
View File

@@ -93,7 +93,10 @@ namespace TimberWinR.Outputs
protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage)
{
if (_manager.Config.Filters != null)
ApplyFilters(jsonMessage);
{
if (ApplyFilters(jsonMessage))
return;
}
var message = jsonMessage.ToString();
LogManager.GetCurrentClassLogger().Debug(message);
@@ -104,12 +107,17 @@ namespace TimberWinR.Outputs
}
}
private void ApplyFilters(JObject json)
private bool ApplyFilters(JObject json)
{
bool drop = false;
foreach (var filter in _manager.Config.Filters)
{
filter.Apply(json);
if (!filter.Apply(json))
drop = true;
}
return drop;
}
}

63
TimberWinR/Parser.cs
View File

@@ -23,7 +23,7 @@ namespace TimberWinR.Parser
{
void Validate();
}
public abstract class LogstashFilter : IValidateSchema
{
@@ -51,22 +51,21 @@ namespace TimberWinR.Parser
}
public abstract JObject ToJson();
protected bool EvaluateCondition(JObject json, string condition)
{
var cond = condition;
IList<string> keys = json.Properties().Select(pn => pn.Name).ToList();
foreach (string key in keys)
cond = cond.Replace(string.Format("[{0}]", key), string.Format("{0}", json[key].ToString()));
var p = Expression.Parameter(typeof (JObject), "");
var e = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] {p}, null, cond);
var p = Expression.Parameter(typeof(JObject), "");
var e = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] { p }, null, cond);
var result = e.Compile().DynamicInvoke(json);
return (bool) result;
return (bool)result;
}
protected void RemoveProperties(JToken token, string[] fields)
{
@@ -255,12 +254,49 @@ namespace TimberWinR.Parser
public class Stdin : IValidateSchema
{
[JsonProperty(PropertyName = "codec")]
public Codec Codec { get; set; }
public void Validate()
{
}
}
public class Codec
{
public enum CodecType
{
singleline,
multiline
};
public enum WhatType
{
previous,
next
};
[JsonProperty(PropertyName = "type")]
public CodecType Type { get; set; }
[JsonProperty(PropertyName = "pattern")]
public string Pattern { get; set; }
[JsonProperty(PropertyName = "what")]
public WhatType What { get; set; }
[JsonProperty(PropertyName = "negate")]
public bool Negate { get; set; }
[JsonProperty(PropertyName = "multiline_tag")]
public string MultilineTag { get; set; }
public Regex Re { get; set; }
public Codec()
{
Negate = false;
MultilineTag = "multiline";
}
}
public class Log : IValidateSchema
{
[JsonProperty(PropertyName = "location")]
@@ -277,6 +313,8 @@ namespace TimberWinR.Parser
public int Interval { get; set; }
[JsonProperty(PropertyName = "logSource")]
public string LogSource { get; set; }
[JsonProperty(PropertyName = "codec")]
public Codec Codec { get; set; }
public Log()
{
@@ -763,6 +801,7 @@ namespace TimberWinR.Parser
}
}
public partial class Json : LogstashFilter
{
public class JsonMissingSourceException : Exception
@@ -840,13 +879,13 @@ namespace TimberWinR.Parser
[JsonProperty("json")]
public Json Json { get; set; }
[JsonProperty("geoip")]
public GeoIP GeoIP { get; set; }
[JsonProperty("grokFilters")]
public Grok[] Groks { get; set; }
[JsonProperty("mutateFilters")]
public Mutate[] Mutates { get; set; }
@@ -855,9 +894,9 @@ namespace TimberWinR.Parser
[JsonProperty("jsonFilters")]
public Json[] Jsons { get; set; }
[JsonProperty("geoipFilters")]
public GeoIP[] GeoIPs { get; set; }
public GeoIP[] GeoIPs { get; set; }
}
public class TimberWinR
@@ -874,9 +913,9 @@ namespace TimberWinR.Parser
get
{
var list = new List<LogstashFilter>();
foreach (var filter in Filters)
{
{
foreach (var prop in filter.GetType().GetProperties())
{
object typedFilter = filter.GetType().GetProperty(prop.Name).GetValue(filter, null);

4
TimberWinR/ReleaseNotes.md
View File

@@ -4,6 +4,10 @@ A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version History
### 1.3.19.0 - 01/12/2015
1. Added support for Multiline codecs for Stdin and Logs listeners, addresses issue #23
### 1.3.18.0 - 12/22/2014
1. Fixed bug introduced in 1.3.17.0 which changed the meaning of the delay for Elasticsearch, Redis and Stdout

1
TimberWinR/TimberWinR.csproj
View File

@@ -122,6 +122,7 @@
<Content Include="GeoLite2City.mmdb">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<None Include="mdocs\Codec.md" />
<None Include="mdocs\DateFilter.md" />
<None Include="mdocs\Filters.md" />
<None Include="mdocs\GeoIPFilter.md" />

31
TimberWinR/mdocs/Codec.md Normal file
View File

@@ -0,0 +1,31 @@
# Codec
## Parameters
The following parameters are allowed when configuring the Codec.
| Parameter | Type | Description | Details | Default |
| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- |
| *type* | enum |Codec type 'multiline' | Must be 'multiline' | |
| *pattern* | regex |Regular expression to be matched | Must be legal .NET Regex | |
| *what* | enum |Value can be previous or next | If the pattern matched, does event belong to the next or previous event? | |
Example Input: Mutliline input log file
```json
{
"TimberWinR": {
"Inputs": {
"Logs": [
{
"location": "C:\\Logs1\\multiline.log",
"recurse": -1,
"codec": {
"type": "multiline",
"pattern": "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)",
"what": "previous"
}
}
}
}
}
```

1
TimberWinR/mdocs/Logs.md
View File

@@ -11,6 +11,7 @@ The following parameters are allowed when configuring WindowsEvents.
| *recurse* | integer |Max subdirectory recursion level. | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
| *splitLongLines* | boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false |
| *iCodepage* | integer |Codepage of the text file. | 0 is the system codepage, -1 is UNICODE. | 0 |
| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md) | object | Codec to use |
Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern. I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.

1
TimberWinR/mdocs/StdinInput.md
View File

@@ -26,4 +26,5 @@ A field: "type": "Win32-Stdin" is automatically appended, and the entire JSON is
| ---- |:-----| :-----------------------------------------------------------------------|
| type | STRING |Win32-Stdin |
| message | STRING | The message typed in |
| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md) | object | Codec to use |