diff --git a/README.md b/README.md index 2fa3906..5eff9f8 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ TimberWinR uses a configuration file to control how the logs are collected, filt These are broken down into: 1. Inputs (Collect data from different sources) 2. Filters (Are applied to all Inputs) - 3. Outputs (Currently ships only to Redis) + 3. Outputs (Redis, Elasticsearch or Stdout) ### Support ### Please use the TimberWinR Google Group for discussion and support: @@ -20,14 +20,15 @@ Please use the TimberWinR Google Group for discussion and support: https://groups.google.com/forum/#!forum/timberwinr -## Input Formats +## Inputs The current supported Input format sources are: 1. [Logs](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file) - 2. [Tcp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on a port for JSON messages) + 2. [Tcp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on TCP port for JSON messages) 3. [IISW3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/IISW3CInput.md)(Internet Information Services W3C Format) 4. [WindowsEvents](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/WindowsEvents.md) (Windows Event Viewer) 5. [Stdin](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) (Standard Input for Debugging) - 3. [W3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format) + 6. [W3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format) + 7. [Udp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/UdpInput.md) (listens for UDP on port for JSON messages) ## Filters The current list of supported filters are: @@ -41,7 +42,7 @@ The current list of supported filters are: Since TimberWinR only ships to Redis and Elasticsearch, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be represented as a JSON Property or Array. -## Supported Output Formats +## Outputs 1. [Redis](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md) 2. [Elasticsearch](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md) 3. [Stdout](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md) diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 1df06ee..4333ec8 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.3.7.0")] -[assembly: AssemblyFileVersion("1.3.7.0")] +[assembly: AssemblyVersion("1.3.9.0")] +[assembly: AssemblyFileVersion("1.3.9.0")] diff --git a/TimberWinR.ServiceHost/default.json b/TimberWinR.ServiceHost/default.json index 1b7eab4..043d20a 100644 --- a/TimberWinR.ServiceHost/default.json +++ b/TimberWinR.ServiceHost/default.json @@ -26,17 +26,17 @@ "drop": "true" } } - ] - }, - "Outputs": { - "Redis": [ - { - "_comment": "Change the host to your Redis instance", - "port": 6379, - "host": [ - "logaggregator.vistaprint.svc" - ] - } - ] + ], + "Outputs": { + "Redis": [ + { + "_comment": "Change the host to your Redis instance", + "port": 6379, + "host": [ + "logaggregator.vistaprint.svc" + ] + } + ] + } } } diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 36a0d31..d9edb69 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -55,7 +55,13 @@ namespace TimberWinR public IEnumerable Tcps { get { return _tcps; } - } + } + + private List _udps = new List(); + public IEnumerable Udps + { + get { return _udps; } + } private List _logs = new List(); public IEnumerable Logs @@ -144,6 +150,8 @@ namespace TimberWinR c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList()); if (x.TimberWinR.Inputs.Tcps != null) c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList()); + if (x.TimberWinR.Inputs.Udps != null) + c._udps.AddRange(x.TimberWinR.Inputs.Udps.ToList()); } if (x.TimberWinR.Outputs != null) @@ -192,6 +200,7 @@ namespace TimberWinR _elasticsearchOutputs = new List(); _stdoutOutputs = new List(); _tcps = new List(); + _udps = new List(); } public static Object GetPropValue(String name, Object obj) diff --git a/TimberWinR/Inputs/TcpInputListener.cs b/TimberWinR/Inputs/TcpInputListener.cs index 3734059..b4d9870 100644 --- a/TimberWinR/Inputs/TcpInputListener.cs +++ b/TimberWinR/Inputs/TcpInputListener.cs @@ -1,11 +1,9 @@ using System; -using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using System.Threading; using System.Net; using System.Net.Sockets; +using Newtonsoft.Json; using Newtonsoft.Json.Linq; using NLog; @@ -36,7 +34,7 @@ namespace TimberWinR.Inputs : base(cancelToken, "Win32-Tcp") { _port = port; - + LogManager.GetCurrentClassLogger().Info("Tcp Input(v4/v6) on Port {0} Ready", _port); @@ -67,7 +65,7 @@ namespace TimberWinR.Inputs listener.Start(); - + while (!CancelToken.IsCancellationRequested) { try @@ -92,27 +90,22 @@ namespace TimberWinR.Inputs private void HandleNewClient(object client) { var tcpClient = (TcpClient)client; - NetworkStream clientStream = null; try { - clientStream = tcpClient.GetStream(); - var stream = new StreamReader(clientStream); - string line; - while ((line = stream.ReadLine()) != null) + NetworkStream clientStream = tcpClient.GetStream(); + using (var stream = new StreamReader(clientStream)) { - try + //assume a continuous stream of JSON objects + using (var reader = new JsonTextReader(stream) { SupportMultipleContent = true }) { - JObject json = JObject.Parse(line); - ProcessJson(json); - _receivedMessages++; + while (reader.Read()) + { + if (CancelToken.IsCancellationRequested) break; + JObject json = JObject.Load(reader); + ProcessJson(json); + } } - catch (Exception ex) - { - LogManager.GetCurrentClassLogger().Error(ex); - } - if (CancelToken.IsCancellationRequested) - break; } } catch (Exception ex) @@ -120,9 +113,6 @@ namespace TimberWinR.Inputs LogManager.GetCurrentClassLogger().Error(ex); } - if (clientStream != null) - clientStream.Close(); - tcpClient.Close(); Finished(); } diff --git a/TimberWinR/Inputs/UdpInputListener.cs b/TimberWinR/Inputs/UdpInputListener.cs new file mode 100644 index 0000000..122da64 --- /dev/null +++ b/TimberWinR/Inputs/UdpInputListener.cs @@ -0,0 +1,89 @@ +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Net; +using System.Net.Sockets; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NLog; + +namespace TimberWinR.Inputs +{ + public class UdpInputListener : InputListener + { + private readonly System.Net.Sockets.UdpClient _udpListener; + private IPEndPoint groupV4; + private IPEndPoint groupV6; + + private Thread _listenThreadV4; + private Thread _listenThreadV6; + + private readonly int _port; + private long _receivedMessages; + + private struct listenProfile + { + public IPEndPoint endPoint; + public UdpClient client; + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("udp", + new JObject( + new JProperty("port", _port), + new JProperty("messages", _receivedMessages) + ))); + + return json; + } + + public UdpInputListener(CancellationToken cancelToken, int port = 5140) + : base(cancelToken, "Win32-Udp") + { + _port = port; + + LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port); + + _udpListener = new System.Net.Sockets.UdpClient(port); + + _listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener)); + _listenThreadV4.Start(new listenProfile() {endPoint = groupV4, client = _udpListener}); + + _listenThreadV6 = new Thread(new ParameterizedThreadStart(StartListener)); + _listenThreadV6.Start(new listenProfile() { endPoint = groupV6, client = _udpListener }); + } + + + public override void Shutdown() + { + Finished(); + base.Shutdown(); + } + + + private void StartListener(object useProfile) + { + var profile = (listenProfile)useProfile; + + try + { + while (!CancelToken.IsCancellationRequested) + { + byte[] bytes = profile.client.Receive(ref profile.endPoint); + var data = Encoding.ASCII.GetString(bytes, 0, bytes.Length); + JObject json = JObject.Parse(data); + ProcessJson(json); + } + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + } + + Finished(); + } + } +} diff --git a/TimberWinR/Inputs/WindowsEvtInputListener.cs b/TimberWinR/Inputs/WindowsEvtInputListener.cs index 90bfcfa..ce3155a 100644 --- a/TimberWinR/Inputs/WindowsEvtInputListener.cs +++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs @@ -26,11 +26,11 @@ namespace TimberWinR.Inputs private TimberWinR.Parser.WindowsEvent _arguments; private long _receivedMessages; - public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5) + public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken) : base(cancelToken, "Win32-Eventlog") { _arguments = arguments; - _pollingIntervalInSeconds = pollingIntervalInSeconds; + _pollingIntervalInSeconds = arguments.Interval; foreach (string eventHive in _arguments.Source.Split(',')) { @@ -52,6 +52,7 @@ namespace TimberWinR.Inputs new JProperty("messages", _receivedMessages), new JProperty("binaryFormat", _arguments.BinaryFormat.ToString()), new JProperty("direction", _arguments.Direction.ToString()), + new JProperty("interval", _arguments.Interval), new JProperty("formatMsg", _arguments.FormatMsg), new JProperty("fullEventCode", _arguments.FullEventCode), new JProperty("fullText", _arguments.FullText), @@ -67,8 +68,7 @@ namespace TimberWinR.Inputs { LogQuery oLogQuery = new LogQuery(); - LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready"); - + LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready"); // Instantiate the Event Log Input Format object var iFmt = new EventLogInputFormat() @@ -85,18 +85,17 @@ namespace TimberWinR.Inputs oLogQuery = null; - Dictionary logFileMaxRecords = new Dictionary(); - + Dictionary logFileMaxRecords = new Dictionary(); // Execute the query while (!CancelToken.IsCancellationRequested) { try { - oLogQuery = new LogQuery(); - Thread.CurrentThread.Priority = ThreadPriority.BelowNormal; + oLogQuery = new LogQuery(); + var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location); var rsfiles = oLogQuery.Execute(qfiles, iFmt); for (; !rsfiles.atEnd(); rsfiles.moveNext()) diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index abce515..cec594d 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -23,6 +23,7 @@ namespace TimberWinR public Configuration Config { get; set; } public List Outputs { get; set; } public List Tcps { get; set; } + public List Udps { get; set; } public List Listeners { get; set; } public DateTime StartedOn { get; set; } public string JsonConfig { get; set; } @@ -186,8 +187,15 @@ namespace TimberWinR output.Connect(elistner); } + foreach (var udp in Config.Udps) + { + var elistner = new UdpInputListener(cancelToken, udp.Port); + Listeners.Add(elistner); + foreach (var output in Outputs) + output.Connect(elistner); + } - foreach (var tcp in Config.Stdins) + foreach (var stdin in Config.Stdins) { var elistner = new StdinListener(cancelToken); Listeners.Add(elistner); diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 786b7c2..e0a7966 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -245,9 +245,12 @@ namespace TimberWinR.Parser public List Fields { get; set; } [JsonProperty(PropertyName = "formatMsg")] public bool FormatMsg { get; set; } - + [JsonProperty(PropertyName = "interval")] + public int Interval { get; set; } + public WindowsEvent() { + Interval = 60; // Every minute Source = "System"; StringsSep = "|"; FormatMsg = true; @@ -330,6 +333,22 @@ namespace TimberWinR.Parser } } + + public class Udp : IValidateSchema + { + [JsonProperty(PropertyName = "port")] + public int Port { get; set; } + + public Udp() + { + Port = 5142; + } + + public void Validate() + { + + } + } public class W3CLog : IValidateSchema { [JsonProperty(PropertyName = "location")] @@ -523,6 +542,9 @@ namespace TimberWinR.Parser [JsonProperty("Tcp")] public Tcp[] Tcps { get; set; } + [JsonProperty("Udp")] + public Udp[] Udps { get; set; } + [JsonProperty("IISW3CLogs")] public IISW3CLog[] IISW3CLogs { get; set; } diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index f8525a6..4f87b97 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -83,6 +83,7 @@ + @@ -120,6 +121,7 @@ + diff --git a/TimberWinR/mdocs/TcpInput.md b/TimberWinR/mdocs/TcpInput.md index 44d761f..cb1ea76 100644 --- a/TimberWinR/mdocs/TcpInput.md +++ b/TimberWinR/mdocs/TcpInput.md @@ -9,7 +9,7 @@ The following parameters are allowed when configuring the Tcp input. | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | | *port* | integer |Port number to open | Must be an available port | | -Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern. I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc. +Example Input: Listen on Port 5140 ```json { diff --git a/TimberWinR/mdocs/UdpInput.md b/TimberWinR/mdocs/UdpInput.md new file mode 100644 index 0000000..499f4fc --- /dev/null +++ b/TimberWinR/mdocs/UdpInput.md @@ -0,0 +1,28 @@ +# Input: Udp + +The Udp input will open a port and listen for properly formatted UDP datagrams to be broadcast. + +## Parameters +The following parameters are allowed when configuring the Udp input. + +| Parameter | Type | Description | Details | Default | +| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | +| *port* | integer |Port number to open | Must be an available port | | + +Example Input: Listen on Port 5142 + +```json +{ + "TimberWinR": { + "Inputs": { + "Udp": [ + { + "port": 5142 + } + ] + } + } +} +``` +## Fields +A field: "type": "Win32-Udp" is automatically appended, and the entire JSON is passed on vertabim. diff --git a/TimberWinR/mdocs/WindowsEvents.md b/TimberWinR/mdocs/WindowsEvents.md index be9164a..a16681c 100644 --- a/TimberWinR/mdocs/WindowsEvents.md +++ b/TimberWinR/mdocs/WindowsEvents.md @@ -2,7 +2,7 @@ The WindowsEvents input will collect events from the Windows Event Viewer. The source parameter indicates which event logs to collect data from. You can specify more than one log by using the comma, i.e. "Application,System" will collect -logs from the Application and System event logs. +logs from the Application and System event logs. The default interval for scanning for new Events is 60 seconds. ## Parameters The following parameters are allowed when configuring WindowsEvents. @@ -18,6 +18,7 @@ The following parameters are allowed when configuring WindowsEvents. | *fullText* | bool |Retrieve the full text message | true,false | **true** | | *resolveSIDS* | bool |Resolve SID values into full account names | true,false | **true** | | *formatMsg* | bool |Format the text message as a single line. | true,false | **true** | +| *interval* | integer | Interval in seconds to sleep during checks | Interval | 60 | ### source format The source indicates where to collect the event(s) from, it can be of these form(s):