diff --git a/README.md b/README.md index 2fa3906..5eff9f8 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ TimberWinR uses a configuration file to control how the logs are collected, filt These are broken down into: 1. Inputs (Collect data from different sources) 2. Filters (Are applied to all Inputs) - 3. Outputs (Currently ships only to Redis) + 3. Outputs (Redis, Elasticsearch or Stdout) ### Support ### Please use the TimberWinR Google Group for discussion and support: @@ -20,14 +20,15 @@ Please use the TimberWinR Google Group for discussion and support: https://groups.google.com/forum/#!forum/timberwinr -## Input Formats +## Inputs The current supported Input format sources are: 1. [Logs](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file) - 2. [Tcp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on a port for JSON messages) + 2. [Tcp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on TCP port for JSON messages) 3. [IISW3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/IISW3CInput.md)(Internet Information Services W3C Format) 4. [WindowsEvents](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/WindowsEvents.md) (Windows Event Viewer) 5. [Stdin](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) (Standard Input for Debugging) - 3. [W3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format) + 6. [W3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format) + 7. [Udp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/UdpInput.md) (listens for UDP on port for JSON messages) ## Filters The current list of supported filters are: @@ -41,7 +42,7 @@ The current list of supported filters are: Since TimberWinR only ships to Redis and Elasticsearch, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be represented as a JSON Property or Array. -## Supported Output Formats +## Outputs 1. [Redis](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md) 2. [Elasticsearch](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md) 3. [Stdout](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md) diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index 1df06ee..6cd2621 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.3.7.0")] -[assembly: AssemblyFileVersion("1.3.7.0")] +[assembly: AssemblyVersion("1.3.8.0")] +[assembly: AssemblyFileVersion("1.3.8.0")] diff --git a/TimberWinR/Configuration.cs b/TimberWinR/Configuration.cs index 36a0d31..d9edb69 100644 --- a/TimberWinR/Configuration.cs +++ b/TimberWinR/Configuration.cs @@ -55,7 +55,13 @@ namespace TimberWinR public IEnumerable Tcps { get { return _tcps; } - } + } + + private List _udps = new List(); + public IEnumerable Udps + { + get { return _udps; } + } private List _logs = new List(); public IEnumerable Logs @@ -144,6 +150,8 @@ namespace TimberWinR c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList()); if (x.TimberWinR.Inputs.Tcps != null) c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList()); + if (x.TimberWinR.Inputs.Udps != null) + c._udps.AddRange(x.TimberWinR.Inputs.Udps.ToList()); } if (x.TimberWinR.Outputs != null) @@ -192,6 +200,7 @@ namespace TimberWinR _elasticsearchOutputs = new List(); _stdoutOutputs = new List(); _tcps = new List(); + _udps = new List(); } public static Object GetPropValue(String name, Object obj) diff --git a/TimberWinR/Inputs/UdpInputListener.cs b/TimberWinR/Inputs/UdpInputListener.cs new file mode 100644 index 0000000..122da64 --- /dev/null +++ b/TimberWinR/Inputs/UdpInputListener.cs @@ -0,0 +1,89 @@ +using System; +using System.IO; +using System.Text; +using System.Threading; +using System.Net; +using System.Net.Sockets; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NLog; + +namespace TimberWinR.Inputs +{ + public class UdpInputListener : InputListener + { + private readonly System.Net.Sockets.UdpClient _udpListener; + private IPEndPoint groupV4; + private IPEndPoint groupV6; + + private Thread _listenThreadV4; + private Thread _listenThreadV6; + + private readonly int _port; + private long _receivedMessages; + + private struct listenProfile + { + public IPEndPoint endPoint; + public UdpClient client; + } + + public override JObject ToJson() + { + JObject json = new JObject( + new JProperty("udp", + new JObject( + new JProperty("port", _port), + new JProperty("messages", _receivedMessages) + ))); + + return json; + } + + public UdpInputListener(CancellationToken cancelToken, int port = 5140) + : base(cancelToken, "Win32-Udp") + { + _port = port; + + LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port); + + _udpListener = new System.Net.Sockets.UdpClient(port); + + _listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener)); + _listenThreadV4.Start(new listenProfile() {endPoint = groupV4, client = _udpListener}); + + _listenThreadV6 = new Thread(new ParameterizedThreadStart(StartListener)); + _listenThreadV6.Start(new listenProfile() { endPoint = groupV6, client = _udpListener }); + } + + + public override void Shutdown() + { + Finished(); + base.Shutdown(); + } + + + private void StartListener(object useProfile) + { + var profile = (listenProfile)useProfile; + + try + { + while (!CancelToken.IsCancellationRequested) + { + byte[] bytes = profile.client.Receive(ref profile.endPoint); + var data = Encoding.ASCII.GetString(bytes, 0, bytes.Length); + JObject json = JObject.Parse(data); + ProcessJson(json); + } + } + catch (Exception ex) + { + LogManager.GetCurrentClassLogger().Error(ex); + } + + Finished(); + } + } +} diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs index abce515..cec594d 100644 --- a/TimberWinR/Manager.cs +++ b/TimberWinR/Manager.cs @@ -23,6 +23,7 @@ namespace TimberWinR public Configuration Config { get; set; } public List Outputs { get; set; } public List Tcps { get; set; } + public List Udps { get; set; } public List Listeners { get; set; } public DateTime StartedOn { get; set; } public string JsonConfig { get; set; } @@ -186,8 +187,15 @@ namespace TimberWinR output.Connect(elistner); } + foreach (var udp in Config.Udps) + { + var elistner = new UdpInputListener(cancelToken, udp.Port); + Listeners.Add(elistner); + foreach (var output in Outputs) + output.Connect(elistner); + } - foreach (var tcp in Config.Stdins) + foreach (var stdin in Config.Stdins) { var elistner = new StdinListener(cancelToken); Listeners.Add(elistner); diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 786b7c2..16bda41 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -330,6 +330,22 @@ namespace TimberWinR.Parser } } + + public class Udp : IValidateSchema + { + [JsonProperty(PropertyName = "port")] + public int Port { get; set; } + + public Udp() + { + Port = 5142; + } + + public void Validate() + { + + } + } public class W3CLog : IValidateSchema { [JsonProperty(PropertyName = "location")] @@ -523,6 +539,9 @@ namespace TimberWinR.Parser [JsonProperty("Tcp")] public Tcp[] Tcps { get; set; } + [JsonProperty("Udp")] + public Udp[] Udps { get; set; } + [JsonProperty("IISW3CLogs")] public IISW3CLog[] IISW3CLogs { get; set; } diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj index f8525a6..4f87b97 100644 --- a/TimberWinR/TimberWinR.csproj +++ b/TimberWinR/TimberWinR.csproj @@ -83,6 +83,7 @@ + @@ -120,6 +121,7 @@ + diff --git a/TimberWinR/mdocs/TcpInput.md b/TimberWinR/mdocs/TcpInput.md index 44d761f..cb1ea76 100644 --- a/TimberWinR/mdocs/TcpInput.md +++ b/TimberWinR/mdocs/TcpInput.md @@ -9,7 +9,7 @@ The following parameters are allowed when configuring the Tcp input. | :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | | *port* | integer |Port number to open | Must be an available port | | -Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern. I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc. +Example Input: Listen on Port 5140 ```json { diff --git a/TimberWinR/mdocs/UdpInput.md b/TimberWinR/mdocs/UdpInput.md new file mode 100644 index 0000000..499f4fc --- /dev/null +++ b/TimberWinR/mdocs/UdpInput.md @@ -0,0 +1,28 @@ +# Input: Udp + +The Udp input will open a port and listen for properly formatted UDP datagrams to be broadcast. + +## Parameters +The following parameters are allowed when configuring the Udp input. + +| Parameter | Type | Description | Details | Default | +| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- | +| *port* | integer |Port number to open | Must be an available port | | + +Example Input: Listen on Port 5142 + +```json +{ + "TimberWinR": { + "Inputs": { + "Udp": [ + { + "port": 5142 + } + ] + } + } +} +``` +## Fields +A field: "type": "Win32-Udp" is automatically appended, and the entire JSON is passed on vertabim.