TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Finished up DateFilter

Browse Source 
lots of cleanup
This commit is contained in:
Eric Fontana
2014-07-28 13:15:07 -04:00
parent 99b6b29e99
commit 5bd3f53663
10 changed files with 139 additions and 110 deletions
Download Patch File Download Diff File Expand all files Collapse all files

46
TimberWinR.ServiceHost/Program.cs
View File

@@ -22,15 +22,7 @@ namespace TimberWinR.ServiceHost
private static void Main(string[] args)
{
Arguments arguments = new Arguments();
var text = "Nov 21 17:27:53";
var pattern = "MMM dd HH:mm:ss";
var match = Regex.Match(text, pattern);
Type x = Type.GetType("string");
Type x1 = Type.GetType("System.string");
HostFactory.Run(hostConfigurator =>
{
string cmdLine = Environment.CommandLine;
@@ -74,7 +66,8 @@ namespace TimberWinR.ServiceHost
readonly CancellationToken _cancellationToken;
readonly Task _serviceTask;
private readonly Arguments _args;
private TcpInputListener _nlogListener;
private TimberWinR.Manager _manager;
public TimberWinRService(Arguments args)
{
@@ -91,8 +84,10 @@ namespace TimberWinR.ServiceHost
public void Stop()
{
_cancellationTokenSource.Cancel();
_nlogListener.Shutdown();
_cancellationTokenSource.Cancel();
if (_manager != null)
_manager.Shutdown();
}
/// <summary>
@@ -100,32 +95,7 @@ namespace TimberWinR.ServiceHost
/// </summary>
private void RunService()
{
TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken);
#if false
var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken);
_nlogListener = new TcpInputListener(_cancellationToken, 5140);
outputRedis.Connect(_nlogListener);
foreach (Parser.IISW3CLog iisw3cConfig in manager.Config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
foreach (Parser.WindowsEvent eventConfig in manager.Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
foreach (var logConfig in manager.Config.Logs)
{
var elistner = new TailFileInputListener(logConfig, _cancellationToken);
outputRedis.Connect(elistner);
}
#endif
_manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken);
}
}
}

42
TimberWinR.ServiceHost/config.json
View File

@@ -35,20 +35,7 @@
}
]
},
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-FileLog\"",
"match": [
"Text",
""
],
"add_field": [
"host",
"%{ComputerName}"
]
}
},
"Filters": [
{
"grok": {
"condition": "[type] == \"Win32-Eventlog\"",
@@ -89,24 +76,29 @@
]
}
},
{
"mutate": {
"rename": [
"host", "Host",
"message","Message",
"SID", "Username"
]
}
},
{
"date": {
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
]
],
"target": "UtcTimestamp",
"convertToUTC": true
}
}
},
{
"mutate": {
"_comment": "Orion Rules",
"rename": [
"host", "Host",
"message","Message",
"type","Type",
"SID", "Username"
]
}
},
]
}
}

8
TimberWinR/Filters/DateFilter.cs
View File

@@ -71,10 +71,10 @@ namespace TimberWinR.Parser
if (ConvertToUTC)
ts = ts.ToUniversalTime();
//if (json[Target] == null)
// json.Add(Target, ts);
//else
// json[Target] = ts;
if (json[Target] == null)
json.Add(Target, ts);
else
json[Target] = ts;
}
}
}

14
TimberWinR/Inputs/IISW3CInputListener.cs
View File

@@ -33,20 +33,22 @@ namespace TimberWinR.Inputs
task.Start();
}
public override void Shutdown()
{
base.Shutdown();
}
private void IISW3CWatcher()
{
var oLogQuery = new LogQuery();
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
var iFmt = new IISW3CLogInputFormat()
{
codepage = _arguments.CodePage,
consolidateLogs = _arguments.ConsolidateLogs,
dirTime = _arguments.DirTime,
dQuotes = _arguments.DoubleQuotes,
iCheckpoint = checkpointFileName,
iCheckpoint = CheckpointFileName,
recurse = _arguments.Recurse,
useDoubleQuotes = _arguments.DoubleQuotes
};
@@ -106,6 +108,8 @@ namespace TimberWinR.Inputs
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
Finished();
}
}
}

32
TimberWinR/Inputs/InputListener.cs
View File

@@ -1,4 +1,5 @@
using System.Runtime.InteropServices;
using System.IO;
using System.Runtime.InteropServices;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
@@ -14,9 +15,14 @@ namespace TimberWinR.Inputs
public event Action<JObject> OnMessageRecieved;
private string _computerName;
private string _typeName;
public AutoResetEvent FinishedEvent { get; set; }
public string CheckpointFileName { get; set; }
public InputListener(CancellationToken token, string typeName)
{
CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString()));
this.FinishedEvent = new AutoResetEvent(false);
this.CancelToken = token;
this._typeName = typeName;
this._computerName = System.Environment.MachineName + "." +
@@ -26,20 +32,40 @@ namespace TimberWinR.Inputs
.ToString();
}
private void AddDefaultFileds(JObject json)
public void Finished()
{
FinishedEvent.Set();
}
public virtual void Shutdown()
{
FinishedEvent.WaitOne();
try
{
if (File.Exists(CheckpointFileName))
File.Delete(CheckpointFileName);
}
catch (Exception)
{
}
}
private void AddDefaultFields(JObject json)
{
if (json["type"] == null)
json.Add(new JProperty("type", _typeName));
if (json["host"] == null)
json.Add(new JProperty("host", _computerName));
if (json["@timestamp"] == null)
json.Add(new JProperty("@timestamp", DateTime.UtcNow));
}
protected void ProcessJson(JObject json)
{
if (OnMessageRecieved != null)
{
AddDefaultFileds(json);
AddDefaultFields(json);
OnMessageRecieved(json);
}
}

20
TimberWinR/Inputs/TailFileInputListener.cs
View File

@@ -34,16 +34,18 @@ namespace TimberWinR.Inputs
task.Start();
}
private void FileWatcher()
{
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
public override void Shutdown()
{
base.Shutdown();
}
private void FileWatcher()
{
var iFmt = new TextLineInputFormat()
{
iCodepage = _arguments.CodePage,
splitLongLines = _arguments.SplitLongLines,
iCheckpoint = checkpointFileName,
iCheckpoint = CheckpointFileName,
recurse = _arguments.Recurse
};
@@ -87,8 +89,10 @@ namespace TimberWinR.Inputs
}
else
json.Add(new JProperty(field.Name, v));
}
ProcessJson(json);
}
string msg = json["Text"].ToString();
if (!string.IsNullOrEmpty(msg))
ProcessJson(json);
}
}
// Close the recordset
@@ -106,6 +110,8 @@ namespace TimberWinR.Inputs
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
Finished();
}
}
}

23
TimberWinR/Inputs/TcpInputListener.cs
View File

@@ -24,13 +24,17 @@ namespace TimberWinR.Inputs
_tcpListener = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
_listenThread = new Thread(new ThreadStart(ListenForClients));
_listenThread.Start();
}
public void Shutdown()
{
this._tcpListener.Stop();
}
public override void Shutdown()
{
this._tcpListener.Stop();
Finished();
base.Shutdown();
}
private void ListenForClients()
{
this._tcpListener.Start();
@@ -62,13 +66,7 @@ namespace TimberWinR.Inputs
{
var tcpClient = (TcpClient)client;
NetworkStream clientStream = tcpClient.GetStream();
string computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
var message = new byte[bufferSize];
while (!CancelToken.IsCancellationRequested)
{
@@ -98,6 +96,7 @@ namespace TimberWinR.Inputs
ProcessJson(json);
}
tcpClient.Close();
Finished();
}
}
}

24
TimberWinR/Inputs/WindowsEvtInputListener.cs
View File

@@ -24,7 +24,7 @@ namespace TimberWinR.Inputs
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Parser.WindowsEvent _arguments;
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken, "Win32-Eventlog")
{
@@ -34,13 +34,17 @@ namespace TimberWinR.Inputs
task.Start();
}
public override void Shutdown()
{
base.Shutdown();
}
private void EventWatcher()
{
var oLogQuery = new LogQuery();
var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
string.Format("{0}.lpc", Guid.NewGuid().ToString()));
// Instantiate the Event Log Input Format object
var iFmt = new EventLogInputFormat()
{
@@ -52,15 +56,9 @@ namespace TimberWinR.Inputs
msgErrorMode = _arguments.MsgErrorMode.ToString(),
stringsSep = _arguments.StringsSep,
resolveSIDs = _arguments.ResolveSIDS,
iCheckpoint = checkpointFileName,
iCheckpoint = CheckpointFileName,
};
string computerName = System.Environment.MachineName + "." +
Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
@"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
.GetValue("Domain", "")
.ToString();
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Source);
@@ -99,6 +97,8 @@ namespace TimberWinR.Inputs
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
Finished();
}
}
}

17
TimberWinR/Manager.cs
View File

@@ -1,4 +1,5 @@
using System.IO;
using System.Net.Sockets;
using NLog;
using NLog.Config;
using NLog.Targets;
@@ -19,11 +20,19 @@ namespace TimberWinR
{
public Configuration Config { get; set; }
public List<OutputSender> Outputs { get; set; }
public List<TcpInputListener> Tcps { get; set; }
public List<InputListener> Listeners { get; set; }
public void Shutdown()
{
foreach (InputListener listener in Listeners)
listener.Shutdown();
}
public Manager(string xmlConfigFile, string jsonConfigFile, CancellationToken cancelToken)
{
Outputs = new List<OutputSender>();
Outputs = new List<OutputSender>();
Listeners = new List<InputListener>();
var loggingConfiguration = new LoggingConfiguration();
// Create our default targets
@@ -57,6 +66,7 @@ namespace TimberWinR
foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
Listeners.Add(elistner);
foreach(var output in Outputs)
output.Connect(elistner);
}
@@ -64,6 +74,7 @@ namespace TimberWinR
foreach (Parser.WindowsEvent eventConfig in Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);
Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
@@ -71,6 +82,7 @@ namespace TimberWinR
foreach (var logConfig in Config.Logs)
{
var elistner = new TailFileInputListener(logConfig, cancelToken);
Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
@@ -78,6 +90,7 @@ namespace TimberWinR
foreach (var tcp in Config.Tcps)
{
var elistner = new TcpInputListener(cancelToken, tcp.Port);
Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}

23
TimberWinR/Parser.cs
View File

@@ -455,11 +455,27 @@ namespace TimberWinR.Parser
public partial class DateFilter : LogstashFilter
{
public class DateFilterMatchException : Exception
{
public DateFilterMatchException()
: base("Date filter missing required match, must be 2 array entries.")
{
}
}
public class DateFilterTargetException : Exception
{
public DateFilterTargetException()
: base("Date filter missing target")
{
}
}
[JsonProperty("match")]
public string[] Match { get; set; }
[JsonProperty("target")]
public string[] Target { get; set; }
public string Target { get; set; }
[JsonProperty("convertToUTC")]
public bool ConvertToUTC { get; set; }
@@ -469,9 +485,12 @@ namespace TimberWinR.Parser
public override void Validate()
{
if (Match == null || Match.Length < 2)
throw new DateFilterMatchException();
if (string.IsNullOrEmpty(Target))
throw new DateFilterTargetException();
}
}
public partial class Mutate : LogstashFilter