TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added Udp Input type.

Browse Source
This commit is contained in:
Eric Fontana
2014-11-04 08:01:48 -05:00
parent ff9026095a
commit a0c571b9c0
9 changed files with 166 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@@ -12,7 +12,7 @@ TimberWinR uses a configuration file to control how the logs are collected, filt
These are broken down into:
1. Inputs (Collect data from different sources)
2. Filters (Are applied to all Inputs)
3. Outputs (Currently ships only to Redis)
3. Outputs (Redis, Elasticsearch or Stdout)
### Support ###
Please use the TimberWinR Google Group for discussion and support:
@@ -20,14 +20,15 @@ Please use the TimberWinR Google Group for discussion and support:
https://groups.google.com/forum/#!forum/timberwinr
## Input Formats
## Inputs
The current supported Input format sources are:
1. [Logs](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file)
2. [Tcp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on a port for JSON messages)
2. [Tcp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on TCP port for JSON messages)
3. [IISW3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/IISW3CInput.md)(Internet Information Services W3C Format)
4. [WindowsEvents](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/WindowsEvents.md) (Windows Event Viewer)
5. [Stdin](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) (Standard Input for Debugging)
3. [W3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format)
6. [W3C](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format)
7. [Udp](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/UdpInput.md) (listens for UDP on port for JSON messages)
## Filters
The current list of supported filters are:
@@ -41,7 +42,7 @@ The current list of supported filters are:
Since TimberWinR only ships to Redis and Elasticsearch, the format generated by TimberWinR is JSON. All fields referenced by TimberWinR can be
represented as a JSON Property or Array.
## Supported Output Formats
## Outputs
1. [Redis](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md)
2. [Elasticsearch](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md)
3. [Stdout](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md)

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.7.0")]
[assembly: AssemblyFileVersion("1.3.7.0")]
[assembly: AssemblyVersion("1.3.8.0")]
[assembly: AssemblyFileVersion("1.3.8.0")]

11
TimberWinR/Configuration.cs
View File

@@ -55,7 +55,13 @@ namespace TimberWinR
public IEnumerable<Tcp> Tcps
{
get { return _tcps; }
}
}
private List<Udp> _udps = new List<Udp>();
public IEnumerable<Udp> Udps
{
get { return _udps; }
}
private List<Log> _logs = new List<Log>();
public IEnumerable<Log> Logs
@@ -144,6 +150,8 @@ namespace TimberWinR
c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList());
if (x.TimberWinR.Inputs.Tcps != null)
c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList());
if (x.TimberWinR.Inputs.Udps != null)
c._udps.AddRange(x.TimberWinR.Inputs.Udps.ToList());
}
if (x.TimberWinR.Outputs != null)
@@ -192,6 +200,7 @@ namespace TimberWinR
_elasticsearchOutputs = new List<ElasticsearchOutput>();
_stdoutOutputs = new List<StdoutOutput>();
_tcps = new List<Tcp>();
_udps = new List<Udp>();
}
public static Object GetPropValue(String name, Object obj)

89
TimberWinR/Inputs/UdpInputListener.cs Normal file
View File

@@ -0,0 +1,89 @@
using System;
using System.IO;
using System.Text;
using System.Threading;
using System.Net;
using System.Net.Sockets;
using Newtonsoft.Json;
using Newtonsoft.Json.Linq;
using NLog;
namespace TimberWinR.Inputs
{
public class UdpInputListener : InputListener
{
private readonly System.Net.Sockets.UdpClient _udpListener;
private IPEndPoint groupV4;
private IPEndPoint groupV6;
private Thread _listenThreadV4;
private Thread _listenThreadV6;
private readonly int _port;
private long _receivedMessages;
private struct listenProfile
{
public IPEndPoint endPoint;
public UdpClient client;
}
public override JObject ToJson()
{
JObject json = new JObject(
new JProperty("udp",
new JObject(
new JProperty("port", _port),
new JProperty("messages", _receivedMessages)
)));
return json;
}
public UdpInputListener(CancellationToken cancelToken, int port = 5140)
: base(cancelToken, "Win32-Udp")
{
_port = port;
LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port);
_udpListener = new System.Net.Sockets.UdpClient(port);
_listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener));
_listenThreadV4.Start(new listenProfile() {endPoint = groupV4, client = _udpListener});
_listenThreadV6 = new Thread(new ParameterizedThreadStart(StartListener));
_listenThreadV6.Start(new listenProfile() { endPoint = groupV6, client = _udpListener });
}
public override void Shutdown()
{
Finished();
base.Shutdown();
}
private void StartListener(object useProfile)
{
var profile = (listenProfile)useProfile;
try
{
while (!CancelToken.IsCancellationRequested)
{
byte[] bytes = profile.client.Receive(ref profile.endPoint);
var data = Encoding.ASCII.GetString(bytes, 0, bytes.Length);
JObject json = JObject.Parse(data);
ProcessJson(json);
}
}
catch (Exception ex)
{
LogManager.GetCurrentClassLogger().Error(ex);
}
Finished();
}
}
}

10
TimberWinR/Manager.cs
View File

@@ -23,6 +23,7 @@ namespace TimberWinR
public Configuration Config { get; set; }
public List<OutputSender> Outputs { get; set; }
public List<TcpInputListener> Tcps { get; set; }
public List<TcpInputListener> Udps { get; set; }
public List<InputListener> Listeners { get; set; }
public DateTime StartedOn { get; set; }
public string JsonConfig { get; set; }
@@ -186,8 +187,15 @@ namespace TimberWinR
output.Connect(elistner);
}
foreach (var udp in Config.Udps)
{
var elistner = new UdpInputListener(cancelToken, udp.Port);
Listeners.Add(elistner);
foreach (var output in Outputs)
output.Connect(elistner);
}
foreach (var tcp in Config.Stdins)
foreach (var stdin in Config.Stdins)
{
var elistner = new StdinListener(cancelToken);
Listeners.Add(elistner);

19
TimberWinR/Parser.cs
View File

@@ -330,6 +330,22 @@ namespace TimberWinR.Parser
}
}
public class Udp : IValidateSchema
{
[JsonProperty(PropertyName = "port")]
public int Port { get; set; }
public Udp()
{
Port = 5142;
}
public void Validate()
{
}
}
public class W3CLog : IValidateSchema
{
[JsonProperty(PropertyName = "location")]
@@ -523,6 +539,9 @@ namespace TimberWinR.Parser
[JsonProperty("Tcp")]
public Tcp[] Tcps { get; set; }
[JsonProperty("Udp")]
public Udp[] Udps { get; set; }
[JsonProperty("IISW3CLogs")]
public IISW3CLog[] IISW3CLogs { get; set; }

2
TimberWinR/TimberWinR.csproj
View File

@@ -83,6 +83,7 @@
<Compile Include="Filters\JsonFilter.cs" />
<Compile Include="Filters\MutateFilter.cs" />
<Compile Include="Inputs\FieldDefinitions.cs" />
<Compile Include="Inputs\UdpInputListener.cs" />
<Compile Include="Inputs\W3CInputListener.cs" />
<Compile Include="Inputs\IISW3CInputListener.cs" />
<Compile Include="Inputs\InputBase.cs" />
@@ -120,6 +121,7 @@
<None Include="mdocs\DateFilter.md" />
<None Include="mdocs\Filters.md" />
<None Include="mdocs\GeoIPFilter.md" />
<None Include="mdocs\UdpInput.md" />
<None Include="mdocs\W3CInput.md" />
<None Include="mdocs\JsonFilter.md" />
<None Include="mdocs\GrokFilter.md" />

2
TimberWinR/mdocs/TcpInput.md
View File

@@ -9,7 +9,7 @@ The following parameters are allowed when configuring the Tcp input.
| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- |
| *port* | integer |Port number to open | Must be an available port | |
Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern. I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
Example Input: Listen on Port 5140
```json
{

28
TimberWinR/mdocs/UdpInput.md Normal file
View File

@@ -0,0 +1,28 @@
# Input: Udp
The Udp input will open a port and listen for properly formatted UDP datagrams to be broadcast.
## Parameters
The following parameters are allowed when configuring the Udp input.
| Parameter | Type | Description | Details | Default |
| :---------------- |:---------------| :----------------------------------------------------------------------- | :--------------------------- | :-- |
| *port* | integer |Port number to open | Must be an available port | |
Example Input: Listen on Port 5142
```json
{
"TimberWinR": {
"Inputs": {
"Udp": [
{
"port": 5142
}
]
}
}
}
```
## Fields
A field: "type": "Win32-Udp" is automatically appended, and the entire JSON is passed on vertabim.