diff --git a/TimberWinR.ServiceHost/Program.cs b/TimberWinR.ServiceHost/Program.cs
index ae8f015..a79f3a2 100644
--- a/TimberWinR.ServiceHost/Program.cs
+++ b/TimberWinR.ServiceHost/Program.cs
@@ -22,15 +22,7 @@ namespace TimberWinR.ServiceHost
private static void Main(string[] args)
{
Arguments arguments = new Arguments();
-
- var text = "Nov 21 17:27:53";
- var pattern = "MMM dd HH:mm:ss";
-
- var match = Regex.Match(text, pattern);
-
- Type x = Type.GetType("string");
- Type x1 = Type.GetType("System.string");
-
+
HostFactory.Run(hostConfigurator =>
{
string cmdLine = Environment.CommandLine;
@@ -74,7 +66,8 @@ namespace TimberWinR.ServiceHost
readonly CancellationToken _cancellationToken;
readonly Task _serviceTask;
private readonly Arguments _args;
- private TcpInputListener _nlogListener;
+
+ private TimberWinR.Manager _manager;
public TimberWinRService(Arguments args)
{
@@ -91,8 +84,10 @@ namespace TimberWinR.ServiceHost
public void Stop()
{
- _cancellationTokenSource.Cancel();
- _nlogListener.Shutdown();
+ _cancellationTokenSource.Cancel();
+
+ if (_manager != null)
+ _manager.Shutdown();
}
///
@@ -100,32 +95,7 @@ namespace TimberWinR.ServiceHost
///
private void RunService()
{
- TimberWinR.Manager manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken);
-
-#if false
- var outputRedis = new RedisOutput(manager, new string[] { "logaggregator.vistaprint.svc" }, _cancellationToken);
-
- _nlogListener = new TcpInputListener(_cancellationToken, 5140);
- outputRedis.Connect(_nlogListener);
-
- foreach (Parser.IISW3CLog iisw3cConfig in manager.Config.IISW3C)
- {
- var elistner = new IISW3CInputListener(iisw3cConfig, _cancellationToken);
- outputRedis.Connect(elistner);
- }
-
- foreach (Parser.WindowsEvent eventConfig in manager.Config.Events)
- {
- var elistner = new WindowsEvtInputListener(eventConfig, _cancellationToken);
- outputRedis.Connect(elistner);
- }
-
- foreach (var logConfig in manager.Config.Logs)
- {
- var elistner = new TailFileInputListener(logConfig, _cancellationToken);
- outputRedis.Connect(elistner);
- }
-#endif
+ _manager = new TimberWinR.Manager(_args.ConfigFile, _args.JsonFile, _cancellationToken);
}
}
}
diff --git a/TimberWinR.ServiceHost/config.json b/TimberWinR.ServiceHost/config.json
index 0e2d4cf..302f969 100644
--- a/TimberWinR.ServiceHost/config.json
+++ b/TimberWinR.ServiceHost/config.json
@@ -35,20 +35,7 @@
}
]
},
- "Filters": [
- {
- "grok": {
- "condition": "[type] == \"Win32-FileLog\"",
- "match": [
- "Text",
- ""
- ],
- "add_field": [
- "host",
- "%{ComputerName}"
- ]
- }
- },
+ "Filters": [
{
"grok": {
"condition": "[type] == \"Win32-Eventlog\"",
@@ -89,24 +76,29 @@
]
}
},
- {
- "mutate": {
- "rename": [
- "host", "Host",
- "message","Message",
- "SID", "Username"
- ]
- }
- },
{
"date": {
"match": [
"timestamp",
"MMM d HH:mm:sss",
"MMM dd HH:mm:ss"
- ]
+ ],
+ "target": "UtcTimestamp",
+ "convertToUTC": true
}
- }
+ },
+ {
+ "mutate": {
+ "_comment": "Orion Rules",
+ "rename": [
+ "host", "Host",
+ "message","Message",
+ "type","Type",
+ "SID", "Username"
+ ]
+ }
+ },
+
]
}
}
diff --git a/TimberWinR/Filters/DateFilter.cs b/TimberWinR/Filters/DateFilter.cs
index 139f44b..4563fce 100644
--- a/TimberWinR/Filters/DateFilter.cs
+++ b/TimberWinR/Filters/DateFilter.cs
@@ -71,10 +71,10 @@ namespace TimberWinR.Parser
if (ConvertToUTC)
ts = ts.ToUniversalTime();
- //if (json[Target] == null)
- // json.Add(Target, ts);
- //else
- // json[Target] = ts;
+ if (json[Target] == null)
+ json.Add(Target, ts);
+ else
+ json[Target] = ts;
}
}
}
diff --git a/TimberWinR/Inputs/IISW3CInputListener.cs b/TimberWinR/Inputs/IISW3CInputListener.cs
index a25128e..4c38191 100644
--- a/TimberWinR/Inputs/IISW3CInputListener.cs
+++ b/TimberWinR/Inputs/IISW3CInputListener.cs
@@ -33,20 +33,22 @@ namespace TimberWinR.Inputs
task.Start();
}
+ public override void Shutdown()
+ {
+ base.Shutdown();
+ }
+
private void IISW3CWatcher()
{
var oLogQuery = new LogQuery();
-
- var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
- string.Format("{0}.lpc", Guid.NewGuid().ToString()));
-
+
var iFmt = new IISW3CLogInputFormat()
{
codepage = _arguments.CodePage,
consolidateLogs = _arguments.ConsolidateLogs,
dirTime = _arguments.DirTime,
dQuotes = _arguments.DoubleQuotes,
- iCheckpoint = checkpointFileName,
+ iCheckpoint = CheckpointFileName,
recurse = _arguments.Recurse,
useDoubleQuotes = _arguments.DoubleQuotes
};
@@ -106,6 +108,8 @@ namespace TimberWinR.Inputs
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
+
+ Finished();
}
}
}
diff --git a/TimberWinR/Inputs/InputListener.cs b/TimberWinR/Inputs/InputListener.cs
index cbec092..4c3a402 100644
--- a/TimberWinR/Inputs/InputListener.cs
+++ b/TimberWinR/Inputs/InputListener.cs
@@ -1,4 +1,5 @@
-using System.Runtime.InteropServices;
+using System.IO;
+using System.Runtime.InteropServices;
using Newtonsoft.Json.Linq;
using System;
using System.Collections.Generic;
@@ -14,9 +15,14 @@ namespace TimberWinR.Inputs
public event Action OnMessageRecieved;
private string _computerName;
private string _typeName;
+ public AutoResetEvent FinishedEvent { get; set; }
+ public string CheckpointFileName { get; set; }
public InputListener(CancellationToken token, string typeName)
{
+ CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString()));
+
+ this.FinishedEvent = new AutoResetEvent(false);
this.CancelToken = token;
this._typeName = typeName;
this._computerName = System.Environment.MachineName + "." +
@@ -26,20 +32,40 @@ namespace TimberWinR.Inputs
.ToString();
}
- private void AddDefaultFileds(JObject json)
+ public void Finished()
+ {
+ FinishedEvent.Set();
+ }
+ public virtual void Shutdown()
+ {
+ FinishedEvent.WaitOne();
+ try
+ {
+ if (File.Exists(CheckpointFileName))
+ File.Delete(CheckpointFileName);
+ }
+ catch (Exception)
+ {
+ }
+ }
+
+ private void AddDefaultFields(JObject json)
{
if (json["type"] == null)
json.Add(new JProperty("type", _typeName));
if (json["host"] == null)
json.Add(new JProperty("host", _computerName));
+
+ if (json["@timestamp"] == null)
+ json.Add(new JProperty("@timestamp", DateTime.UtcNow));
}
protected void ProcessJson(JObject json)
{
if (OnMessageRecieved != null)
{
- AddDefaultFileds(json);
+ AddDefaultFields(json);
OnMessageRecieved(json);
}
}
diff --git a/TimberWinR/Inputs/TailFileInputListener.cs b/TimberWinR/Inputs/TailFileInputListener.cs
index ca58266..517891a 100644
--- a/TimberWinR/Inputs/TailFileInputListener.cs
+++ b/TimberWinR/Inputs/TailFileInputListener.cs
@@ -34,16 +34,18 @@ namespace TimberWinR.Inputs
task.Start();
}
- private void FileWatcher()
- {
- var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
- string.Format("{0}.lpc", Guid.NewGuid().ToString()));
+ public override void Shutdown()
+ {
+ base.Shutdown();
+ }
+ private void FileWatcher()
+ {
var iFmt = new TextLineInputFormat()
{
iCodepage = _arguments.CodePage,
splitLongLines = _arguments.SplitLongLines,
- iCheckpoint = checkpointFileName,
+ iCheckpoint = CheckpointFileName,
recurse = _arguments.Recurse
};
@@ -87,8 +89,10 @@ namespace TimberWinR.Inputs
}
else
json.Add(new JProperty(field.Name, v));
- }
- ProcessJson(json);
+ }
+ string msg = json["Text"].ToString();
+ if (!string.IsNullOrEmpty(msg))
+ ProcessJson(json);
}
}
// Close the recordset
@@ -106,6 +110,8 @@ namespace TimberWinR.Inputs
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
+
+ Finished();
}
}
}
diff --git a/TimberWinR/Inputs/TcpInputListener.cs b/TimberWinR/Inputs/TcpInputListener.cs
index a83b16f..da487aa 100644
--- a/TimberWinR/Inputs/TcpInputListener.cs
+++ b/TimberWinR/Inputs/TcpInputListener.cs
@@ -24,13 +24,17 @@ namespace TimberWinR.Inputs
_tcpListener = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
_listenThread = new Thread(new ThreadStart(ListenForClients));
_listenThread.Start();
- }
-
- public void Shutdown()
- {
- this._tcpListener.Stop();
}
+
+ public override void Shutdown()
+ {
+ this._tcpListener.Stop();
+ Finished();
+ base.Shutdown();
+ }
+
+
private void ListenForClients()
{
this._tcpListener.Start();
@@ -62,13 +66,7 @@ namespace TimberWinR.Inputs
{
var tcpClient = (TcpClient)client;
NetworkStream clientStream = tcpClient.GetStream();
-
- string computerName = System.Environment.MachineName + "." +
- Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
- @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
- .GetValue("Domain", "")
- .ToString();
-
+
var message = new byte[bufferSize];
while (!CancelToken.IsCancellationRequested)
{
@@ -98,6 +96,7 @@ namespace TimberWinR.Inputs
ProcessJson(json);
}
tcpClient.Close();
+ Finished();
}
}
}
diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs
index 46ad52d..cf6a1af 100644
--- a/TimberWinR/Inputs/WindowsEvtInputListener.cs
+++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs
@@ -24,7 +24,7 @@ namespace TimberWinR.Inputs
{
private int _pollingIntervalInSeconds = 1;
private TimberWinR.Parser.WindowsEvent _arguments;
-
+
public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
: base(cancelToken, "Win32-Eventlog")
{
@@ -34,13 +34,17 @@ namespace TimberWinR.Inputs
task.Start();
}
+ public override void Shutdown()
+ {
+ base.Shutdown();
+
+ }
+
private void EventWatcher()
{
var oLogQuery = new LogQuery();
- var checkpointFileName = Path.Combine(System.IO.Path.GetTempPath(),
- string.Format("{0}.lpc", Guid.NewGuid().ToString()));
-
+
// Instantiate the Event Log Input Format object
var iFmt = new EventLogInputFormat()
{
@@ -52,15 +56,9 @@ namespace TimberWinR.Inputs
msgErrorMode = _arguments.MsgErrorMode.ToString(),
stringsSep = _arguments.StringsSep,
resolveSIDs = _arguments.ResolveSIDS,
- iCheckpoint = checkpointFileName,
+ iCheckpoint = CheckpointFileName,
};
-
- string computerName = System.Environment.MachineName + "." +
- Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
- @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
- .GetValue("Domain", "")
- .ToString();
-
+
// Create the query
var query = string.Format("SELECT * FROM {0}", _arguments.Source);
@@ -99,6 +97,8 @@ namespace TimberWinR.Inputs
firstQuery = false;
System.Threading.Thread.Sleep(_pollingIntervalInSeconds * 1000);
}
+
+ Finished();
}
}
}
diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs
index 142992c..032802e 100644
--- a/TimberWinR/Manager.cs
+++ b/TimberWinR/Manager.cs
@@ -1,4 +1,5 @@
using System.IO;
+using System.Net.Sockets;
using NLog;
using NLog.Config;
using NLog.Targets;
@@ -19,11 +20,19 @@ namespace TimberWinR
{
public Configuration Config { get; set; }
public List Outputs { get; set; }
+ public List Tcps { get; set; }
+ public List Listeners { get; set; }
+ public void Shutdown()
+ {
+ foreach (InputListener listener in Listeners)
+ listener.Shutdown();
+ }
public Manager(string xmlConfigFile, string jsonConfigFile, CancellationToken cancelToken)
{
- Outputs = new List();
-
+ Outputs = new List();
+ Listeners = new List();
+
var loggingConfiguration = new LoggingConfiguration();
// Create our default targets
@@ -57,6 +66,7 @@ namespace TimberWinR
foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C)
{
var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
+ Listeners.Add(elistner);
foreach(var output in Outputs)
output.Connect(elistner);
}
@@ -64,6 +74,7 @@ namespace TimberWinR
foreach (Parser.WindowsEvent eventConfig in Config.Events)
{
var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);
+ Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
@@ -71,6 +82,7 @@ namespace TimberWinR
foreach (var logConfig in Config.Logs)
{
var elistner = new TailFileInputListener(logConfig, cancelToken);
+ Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
@@ -78,6 +90,7 @@ namespace TimberWinR
foreach (var tcp in Config.Tcps)
{
var elistner = new TcpInputListener(cancelToken, tcp.Port);
+ Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs
index d869348..302f690 100644
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -455,11 +455,27 @@ namespace TimberWinR.Parser
public partial class DateFilter : LogstashFilter
{
+ public class DateFilterMatchException : Exception
+ {
+ public DateFilterMatchException()
+ : base("Date filter missing required match, must be 2 array entries.")
+ {
+ }
+ }
+
+ public class DateFilterTargetException : Exception
+ {
+ public DateFilterTargetException()
+ : base("Date filter missing target")
+ {
+ }
+ }
+
[JsonProperty("match")]
public string[] Match { get; set; }
[JsonProperty("target")]
- public string[] Target { get; set; }
+ public string Target { get; set; }
[JsonProperty("convertToUTC")]
public bool ConvertToUTC { get; set; }
@@ -469,9 +485,12 @@ namespace TimberWinR.Parser
public override void Validate()
{
+ if (Match == null || Match.Length < 2)
+ throw new DateFilterMatchException();
+ if (string.IsNullOrEmpty(Target))
+ throw new DateFilterTargetException();
}
-
}
public partial class Mutate : LogstashFilter