TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

No functional changes - refactored to common code for Multiline codec

Browse Source
This commit is contained in:
Eric Fontana
2015-03-16 08:32:54 -04:00
parent b937d6ef45
commit 47f233f863
10 changed files with 183 additions and 269 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
TimberWinR.UnitTests/MultilineTests.cs
View File

@@ -17,7 +17,7 @@ namespace TimberWinR.UnitTests
[TestFixture]
public class MultilineTests
{
// [Test(Description = "Test using next")]
// [Test(Description = "Test using next")]
public void TestMultiline1()
{
using (StreamReader sr = new StreamReader("Multiline1.txt"))
@@ -28,10 +28,10 @@ namespace TimberWinR.UnitTests
Stdin sin = new Stdin();
sin.Codec = new Codec();
sin.Codec.Pattern = "\\\\$";
sin.Codec.What = Codec.WhatType.next;
sin.Codec.Type = Codec.CodecType.multiline;
sin.CodecArguments = new CodecArguments();
sin.CodecArguments.Pattern = "\\\\$";
sin.CodecArguments.What = CodecArguments.WhatType.next;
sin.CodecArguments.Type = CodecArguments.CodecType.multiline;
var cancelTokenSource = new CancellationTokenSource();
@@ -66,7 +66,7 @@ namespace TimberWinR.UnitTests
}
}
// [Test(Description = "Test using previous")]
// [Test(Description = "Test using previous")]
public void TestMultiline2()
{
using (StreamReader sr = new StreamReader("Multiline2.txt"))
@@ -77,11 +77,11 @@ namespace TimberWinR.UnitTests
Stdin sin = new Stdin();
sin.Codec = new Codec();
sin.Codec.Pattern = "^(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3})(.*)$";
sin.Codec.What = Codec.WhatType.previous;
sin.Codec.Type = Codec.CodecType.multiline;
sin.Codec.Negate = true;
sin.CodecArguments = new CodecArguments();
sin.CodecArguments.Pattern = "^(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3})(.*)$";
sin.CodecArguments.What = CodecArguments.WhatType.previous;
sin.CodecArguments.Type = CodecArguments.CodecType.multiline;
sin.CodecArguments.Negate = true;
var cancelTokenSource = new CancellationTokenSource();

90
TimberWinR/Codecs/Multiline.cs Normal file
View File

@@ -0,0 +1,90 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using Newtonsoft.Json.Linq;
using TimberWinR.Inputs;
using TimberWinR.Parser;
namespace TimberWinR.Codecs
{
public class Multiline : ICodec
{
private CodecArguments _codecArguments;
private List<string> _multiline { get; set; }
// return true to cancel codec
public Multiline(CodecArguments args)
{
_codecArguments = args;
}
public void Apply(string msg, InputListener listener)
{
if (_codecArguments.Re == null)
_codecArguments.Re = new Regex(_codecArguments.Pattern);
Match match = _codecArguments.Re.Match(msg);
bool isMatch = (match.Success && !_codecArguments.Negate) || (!match.Success && _codecArguments.Negate);
switch (_codecArguments.What)
{
case CodecArguments.WhatType.previous:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No Match
{
if (_multiline != null)
{
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codecArguments.MultilineTag));
listener.AddDefaultFields(jo);
listener.ProcessJson(jo);
}
_multiline = new List<string>();
_multiline.Add(msg);
}
break;
case CodecArguments.WhatType.next:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No match
{
if (_multiline != null)
{
_multiline.Add(msg);
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codecArguments.MultilineTag));
listener.AddDefaultFields(jo);
listener.ProcessJson(jo);
}
else
{
JObject jo = new JObject();
jo["message"] = msg;
listener.AddDefaultFields(jo);
listener.ProcessJson(jo);
}
}
break;
}
}
}
}

13
TimberWinR/ICodec.cs Normal file
View File

@@ -0,0 +1,13 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using TimberWinR.Inputs;
namespace TimberWinR
{
public interface ICodec
{
void Apply(string msg, InputListener listener);
}
}

4
TimberWinR/Inputs/InputListener.cs
View File

@@ -80,7 +80,7 @@ namespace TimberWinR.Inputs
}
}
protected virtual void AddDefaultFields(JObject json)
public virtual void AddDefaultFields(JObject json)
{
if (json["type"] == null)
json.Add(new JProperty("type", _typeName));
@@ -100,7 +100,7 @@ namespace TimberWinR.Inputs
json.Add(new JProperty("UtcTimestamp", utc.ToString("o")));
}
protected void ProcessJson(JObject json)
public void ProcessJson(JObject json)
{
if (OnMessageRecieved != null)
{

107
TimberWinR/Inputs/LogsListener.cs
View File

@@ -14,7 +14,7 @@ using Newtonsoft.Json.Linq;
using Newtonsoft.Json.Serialization;
using NLog;
using TimberWinR.Codecs;
using LogQuery = Interop.MSUtil.LogQueryClassClass;
using TextLineInputFormat = Interop.MSUtil.COMTextLineInputContextClass;
using LogRecordSet = Interop.MSUtil.ILogRecordset;
@@ -34,8 +34,8 @@ namespace TimberWinR.Inputs
private Dictionary<string, DateTime> _logFileCreationTimes;
private Dictionary<string, DateTime> _logFileSampleTimes;
private Dictionary<string, long> _logFileSizes;
private Codec _codec;
private List<string> _multiline { get; set; }
private CodecArguments _codecArguments;
private ICodec _codec;
public bool Stop { get; set; }
@@ -44,7 +44,12 @@ namespace TimberWinR.Inputs
{
Stop = false;
_codec = arguments.Codec;
_codecArguments = arguments.CodecArguments;
_codecArguments = arguments.CodecArguments;
if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
_codec = new Multiline(_codecArguments);
_logFileMaxRecords = new Dictionary<string, Int64>();
_logFileCreationTimes = new Dictionary<string, DateTime>();
_logFileSampleTimes = new Dictionary<string, DateTime>();
@@ -100,94 +105,22 @@ namespace TimberWinR.Inputs
)));
if (_codec != null)
if (_codecArguments != null)
{
var cp = new JProperty("codec",
new JArray(
new JObject(
new JProperty("type", _codec.Type.ToString()),
new JProperty("what", _codec.What.ToString()),
new JProperty("negate", _codec.Negate),
new JProperty("multilineTag", _codec.MultilineTag),
new JProperty("pattern", _codec.Pattern))));
new JProperty("type", _codecArguments.Type.ToString()),
new JProperty("what", _codecArguments.What.ToString()),
new JProperty("negate", _codecArguments.Negate),
new JProperty("multilineTag", _codecArguments.MultilineTag),
new JProperty("pattern", _codecArguments.Pattern))));
json.Add(cp);
}
return json;
}
// return true to cancel codec
private void applyMultilineCodec(string msg)
{
if (_codec.Re == null)
_codec.Re = new Regex(_codec.Pattern);
Match match = _codec.Re.Match(msg);
bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
switch (_codec.What)
{
case Codec.WhatType.previous:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No Match
{
if (_multiline != null)
{
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
_multiline = new List<string>();
_multiline.Add(msg);
}
break;
case Codec.WhatType.next:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No match
{
if (_multiline != null)
{
_multiline.Add(msg);
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
else
{
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
}
break;
}
}
}
private void FileWatcher(string fileToWatch)
{
@@ -298,8 +231,12 @@ namespace TimberWinR.Inputs
string msg = json["Text"].ToString();
if (!string.IsNullOrEmpty(msg))
{
if (_codec != null && _codec.Type == Codec.CodecType.multiline)
applyMultilineCodec(msg);
if (_codecArguments != null &&
_codecArguments.Type == CodecArguments.CodecType.multiline)
{
_codec.Apply(msg, this);
_receivedMessages++;
}
else
{
ProcessJson(json);

96
TimberWinR/Inputs/StdinListener.cs
View File

@@ -8,6 +8,7 @@ using System.Threading;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
using TimberWinR.Codecs;
using TimberWinR.Parser;
namespace TimberWinR.Inputs
@@ -15,13 +16,16 @@ namespace TimberWinR.Inputs
public class StdinListener : InputListener
{
private Thread _listenThread;
private Codec _codec;
private List<string> _multiline { get; set; }
private CodecArguments _codecArguments;
private ICodec _codec;
public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken)
: base(cancelToken, "Win32-Console")
{
_codec = arguments.Codec;
_codecArguments = arguments.CodecArguments;
if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
_codec = new Multiline(_codecArguments);
_listenThread = new Thread(new ThreadStart(ListenToStdin));
_listenThread.Start();
}
@@ -32,16 +36,16 @@ namespace TimberWinR.Inputs
new JProperty("stdin", "enabled"));
if (_codec != null)
if (_codecArguments != null)
{
var cp = new JProperty("codec",
new JArray(
new JObject(
new JProperty("type", _codec.Type.ToString()),
new JProperty("what", _codec.What.ToString()),
new JProperty("negate", _codec.Negate),
new JProperty("multilineTag", _codec.MultilineTag),
new JProperty("pattern", _codec.Pattern))));
new JProperty("type", _codecArguments.Type.ToString()),
new JProperty("what", _codecArguments.What.ToString()),
new JProperty("negate", _codecArguments.Negate),
new JProperty("multilineTag", _codecArguments.MultilineTag),
new JProperty("pattern", _codecArguments.Pattern))));
json.Add(cp);
}
@@ -65,8 +69,8 @@ namespace TimberWinR.Inputs
{
string msg = ToPrintable(line);
if (_codec != null && _codec.Type == Codec.CodecType.multiline)
applyMultilineCodec(msg);
if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
_codec.Apply(msg, this);
else
{
JObject jo = new JObject();
@@ -78,73 +82,5 @@ namespace TimberWinR.Inputs
}
Finished();
}
// return true to cancel codec
private void applyMultilineCodec(string msg)
{
if (_codec.Re == null)
_codec.Re = new Regex(_codec.Pattern);
Match match = _codec.Re.Match(msg);
bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
switch (_codec.What)
{
case Codec.WhatType.previous:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No Match
{
if (_multiline != null)
{
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
}
_multiline = new List<string>();
_multiline.Add(msg);
}
break;
case Codec.WhatType.next:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No match
{
if (_multiline != null)
{
_multiline.Add(msg);
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
}
else
{
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
}
}
break;
}
}
}
}

106
TimberWinR/Inputs/TailFileListener.cs
View File

@@ -16,6 +16,7 @@ using Newtonsoft.Json.Serialization;
using NLog;
using NLog.LayoutRenderers;
using TimberWinR.Codecs;
using TimberWinR.Parser;
namespace TimberWinR.Inputs
@@ -31,9 +32,9 @@ namespace TimberWinR.Inputs
private Dictionary<string, Int64> _logFileMaxRecords;
private Dictionary<string, DateTime> _logFileCreationTimes;
private Dictionary<string, DateTime> _logFileSampleTimes;
private Dictionary<string, long> _logFileSizes;
private Codec _codec;
private List<string> _multiline { get; set; }
private Dictionary<string, long> _logFileSizes;
private CodecArguments _codecArguments;
private ICodec _codec;
public bool Stop { get; set; }
@@ -42,7 +43,11 @@ namespace TimberWinR.Inputs
{
Stop = false;
_codec = arguments.Codec;
_codecArguments = arguments.CodecArguments;
if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
_codec = new Multiline(_codecArguments);
_logFileMaxRecords = new Dictionary<string, Int64>();
_logFileCreationTimes = new Dictionary<string, DateTime>();
_logFileSampleTimes = new Dictionary<string, DateTime>();
@@ -95,16 +100,16 @@ namespace TimberWinR.Inputs
)));
if (_codec != null)
if (_codecArguments != null)
{
var cp = new JProperty("codec",
new JArray(
new JObject(
new JProperty("type", _codec.Type.ToString()),
new JProperty("what", _codec.What.ToString()),
new JProperty("negate", _codec.Negate),
new JProperty("multilineTag", _codec.MultilineTag),
new JProperty("pattern", _codec.Pattern))));
new JProperty("type", _codecArguments.Type.ToString()),
new JProperty("what", _codecArguments.What.ToString()),
new JProperty("negate", _codecArguments.Negate),
new JProperty("multilineTag", _codecArguments.MultilineTag),
new JProperty("pattern", _codecArguments.Pattern))));
json.Add(cp);
}
@@ -112,77 +117,6 @@ namespace TimberWinR.Inputs
return json;
}
// return true to cancel codec
private void applyMultilineCodec(string msg)
{
if (_codec.Re == null)
_codec.Re = new Regex(_codec.Pattern);
Match match = _codec.Re.Match(msg);
bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
switch (_codec.What)
{
case Codec.WhatType.previous:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No Match
{
if (_multiline != null)
{
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
_multiline = new List<string>();
_multiline.Add(msg);
}
break;
case Codec.WhatType.next:
if (isMatch)
{
if (_multiline == null)
_multiline = new List<string>();
_multiline.Add(msg);
}
else // No match
{
if (_multiline != null)
{
_multiline.Add(msg);
string single = string.Join("\n", _multiline.ToArray());
_multiline = null;
JObject jo = new JObject();
jo["message"] = single;
jo.Add("tags", new JArray(_codec.MultilineTag));
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
else
{
JObject jo = new JObject();
jo["message"] = msg;
AddDefaultFields(jo);
ProcessJson(jo);
_receivedMessages++;
}
}
break;
}
}
private void TailFileContents(string fileName, long offset)
{
using (StreamReader reader = new StreamReader(new FileStream(fileName,
@@ -222,15 +156,17 @@ namespace TimberWinR.Inputs
json["Index"] = index;
json["LogFileName"] = fileName;
if (_codec != null && _codec.Type == Codec.CodecType.multiline)
applyMultilineCodec(line);
if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
{
_codec.Apply(line, this);
Interlocked.Increment(ref _receivedMessages);
}
else
{
ProcessJson(json);
Interlocked.Increment(ref _receivedMessages);
}
lineOffset += line.Length;
// Console.WriteLine("File: {0}:{1}: {2}", fileName, reader.BaseStream.Position, line);
lineOffset += line.Length;
}
//update the last max offset
lastMaxOffset = reader.BaseStream.Position;

10
TimberWinR/Parser.cs
View File

@@ -255,7 +255,7 @@ namespace TimberWinR.Parser
public class Stdin : IValidateSchema
{
[JsonProperty(PropertyName = "codec")]
public Codec Codec { get; set; }
public CodecArguments CodecArguments { get; set; }
public void Validate()
{
@@ -263,7 +263,7 @@ namespace TimberWinR.Parser
}
}
public class Codec
public class CodecArguments
{
public enum CodecType
{
@@ -290,7 +290,7 @@ namespace TimberWinR.Parser
public Regex Re { get; set; }
public Codec()
public CodecArguments()
{
Negate = false;
MultilineTag = "multiline";
@@ -310,7 +310,7 @@ namespace TimberWinR.Parser
[JsonProperty(PropertyName = "logSource")]
public string LogSource { get; set; }
[JsonProperty(PropertyName = "codec")]
public Codec Codec { get; set; }
public CodecArguments CodecArguments { get; set; }
public TailFile()
{
@@ -344,7 +344,7 @@ namespace TimberWinR.Parser
[JsonProperty(PropertyName = "logSource")]
public string LogSource { get; set; }
[JsonProperty(PropertyName = "codec")]
public Codec Codec { get; set; }
public CodecArguments CodecArguments { get; set; }
public LogParameters()
{

2
TimberWinR/TimberWinR.csproj
View File

@@ -82,6 +82,7 @@
</Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="Codecs\Multiline.cs" />
<Compile Include="Configuration.cs" />
<Compile Include="ConfigurationErrors.cs" />
<Compile Include="Diagnostics\Diagnostics.cs" />
@@ -91,6 +92,7 @@
<Compile Include="Filters\GeoIPFilter.cs" />
<Compile Include="Filters\JsonFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
<Compile Include="ICodec.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\IISW3CRowReader.cs" />
<Compile Include="Inputs\LogsFileDatabase.cs" />

2
chocolateyUninstall.ps1.template
View File

@@ -1,7 +1,7 @@
$packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
$installerType = 'msi' #only one of these: exe, msi, msu
$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
$silentArgs = '${PROJECTGUID} /quiet'
$silentArgs = '{129EA41B-D668-49B6-BC1D-D16A3CC97E2E} /quiet'
$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes