change DEPRECATION notice

Update README.md
Add deprecation notice.
2016-01-20 08:06:25 -05:00 · 2015-11-05 14:53:57 -05:00 · 2015-09-11 06:46:29 -04:00 · 2015-08-31 17:15:36 -07:00 · 2015-07-13 11:26:50 -04:00 · 2015-06-19 11:47:45 -04:00
227 changed files with 10783 additions and 157875 deletions
--- a/.gitignore
+++ b/.gitignore
@@ -155,3 +155,4 @@ $RECYCLE.BIN/
 # Mac desktop service store files
 .DS_Store
 packages
+*.msi
--- a/.nuget/NuGet.Config
+++ b/.nuget/NuGet.Config
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+</configuration>
--- a/.nuget/NuGet.exe
+++ b/.nuget/NuGet.exe
--- a/.nuget/NuGet.targets
+++ b/.nuget/NuGet.targets
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+    <PropertyGroup>
+        <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">$(MSBuildProjectDirectory)\..\</SolutionDir>
+
+        <!-- Enable the restore command to run before builds -->
+        <RestorePackages Condition="  '$(RestorePackages)' == '' ">false</RestorePackages>
+
+        <!-- Property that enables building a package from a project -->
+        <BuildPackage Condition=" '$(BuildPackage)' == '' ">false</BuildPackage>
+
+        <!-- Determines if package restore consent is required to restore packages -->
+        <RequireRestoreConsent Condition=" '$(RequireRestoreConsent)' != 'false' ">true</RequireRestoreConsent>
+
+        <!-- Download NuGet.exe if it does not already exist -->
+        <DownloadNuGetExe Condition=" '$(DownloadNuGetExe)' == '' ">false</DownloadNuGetExe>
+    </PropertyGroup>
+
+    <ItemGroup Condition=" '$(PackageSources)' == '' ">
+        <!-- Package sources used to restore packages. By default, registered sources under %APPDATA%\NuGet\NuGet.Config will be used -->
+        <!-- The official NuGet package source (https://www.nuget.org/api/v2/) will be excluded if package sources are specified and it does not appear in the list -->
+        <!--
+            <PackageSource Include="https://www.nuget.org/api/v2/" />
+            <PackageSource Include="https://my-nuget-source/nuget/" />
+        -->
+    </ItemGroup>
+
+    <PropertyGroup Condition=" '$(OS)' == 'Windows_NT'">
+        <!-- Windows specific commands -->
+        <NuGetToolsPath>$([System.IO.Path]::Combine($(SolutionDir), ".nuget"))</NuGetToolsPath>
+    </PropertyGroup>
+
+    <PropertyGroup Condition=" '$(OS)' != 'Windows_NT'">
+        <!-- We need to launch nuget.exe with the mono command if we're not on windows -->
+        <NuGetToolsPath>$(SolutionDir).nuget</NuGetToolsPath>
+    </PropertyGroup>
+
+    <PropertyGroup>
+        <PackagesProjectConfig Condition=" '$(OS)' == 'Windows_NT'">$(MSBuildProjectDirectory)\packages.$(MSBuildProjectName.Replace(' ', '_')).config</PackagesProjectConfig>
+        <PackagesProjectConfig Condition=" '$(OS)' != 'Windows_NT'">$(MSBuildProjectDirectory)\packages.$(MSBuildProjectName).config</PackagesProjectConfig>
+    </PropertyGroup>
+
+    <PropertyGroup>
+      <PackagesConfig Condition="Exists('$(MSBuildProjectDirectory)\packages.config')">$(MSBuildProjectDirectory)\packages.config</PackagesConfig>
+      <PackagesConfig Condition="Exists('$(PackagesProjectConfig)')">$(PackagesProjectConfig)</PackagesConfig>
+    </PropertyGroup>
+    
+    <PropertyGroup>
+        <!-- NuGet command -->
+        <NuGetExePath Condition=" '$(NuGetExePath)' == '' ">$(NuGetToolsPath)\NuGet.exe</NuGetExePath>
+        <PackageSources Condition=" $(PackageSources) == '' ">@(PackageSource)</PackageSources>
+
+        <NuGetCommand Condition=" '$(OS)' == 'Windows_NT'">"$(NuGetExePath)"</NuGetCommand>
+        <NuGetCommand Condition=" '$(OS)' != 'Windows_NT' ">mono --runtime=v4.0.30319 "$(NuGetExePath)"</NuGetCommand>
+
+        <PackageOutputDir Condition="$(PackageOutputDir) == ''">$(TargetDir.Trim('\\'))</PackageOutputDir>
+
+        <RequireConsentSwitch Condition=" $(RequireRestoreConsent) == 'true' ">-RequireConsent</RequireConsentSwitch>
+        <NonInteractiveSwitch Condition=" '$(VisualStudioVersion)' != '' AND '$(OS)' == 'Windows_NT' ">-NonInteractive</NonInteractiveSwitch>
+
+        <PaddedSolutionDir Condition=" '$(OS)' == 'Windows_NT'">"$(SolutionDir) "</PaddedSolutionDir>
+        <PaddedSolutionDir Condition=" '$(OS)' != 'Windows_NT' ">"$(SolutionDir)"</PaddedSolutionDir>
+
+        <!-- Commands -->
+        <RestoreCommand>$(NuGetCommand) install "$(PackagesConfig)" -source "$(PackageSources)"  $(NonInteractiveSwitch) $(RequireConsentSwitch) -solutionDir $(PaddedSolutionDir)</RestoreCommand>
+        <BuildCommand>$(NuGetCommand) pack "$(ProjectPath)" -Properties "Configuration=$(Configuration);Platform=$(Platform)" $(NonInteractiveSwitch) -OutputDirectory "$(PackageOutputDir)" -symbols</BuildCommand>
+
+        <!-- We need to ensure packages are restored prior to assembly resolve -->
+        <BuildDependsOn Condition="$(RestorePackages) == 'true'">
+            RestorePackages;
+            $(BuildDependsOn);
+        </BuildDependsOn>
+
+        <!-- Make the build depend on restore packages -->
+        <BuildDependsOn Condition="$(BuildPackage) == 'true'">
+            $(BuildDependsOn);
+            BuildPackage;
+        </BuildDependsOn>
+    </PropertyGroup>
+
+    <Target Name="CheckPrerequisites">
+        <!-- Raise an error if we're unable to locate nuget.exe  -->
+        <Error Condition="'$(DownloadNuGetExe)' != 'true' AND !Exists('$(NuGetExePath)')" Text="Unable to locate '$(NuGetExePath)'" />
+        <!--
+        Take advantage of MsBuild's build dependency tracking to make sure that we only ever download nuget.exe once.
+        This effectively acts as a lock that makes sure that the download operation will only happen once and all
+        parallel builds will have to wait for it to complete.
+        -->
+        <MsBuild Targets="_DownloadNuGet" Projects="$(MSBuildThisFileFullPath)" Properties="Configuration=NOT_IMPORTANT;DownloadNuGetExe=$(DownloadNuGetExe)" />
+    </Target>
+
+    <Target Name="_DownloadNuGet">
+        <DownloadNuGet OutputFilename="$(NuGetExePath)" Condition=" '$(DownloadNuGetExe)' == 'true' AND !Exists('$(NuGetExePath)')" />
+    </Target>
+
+    <Target Name="RestorePackages" DependsOnTargets="CheckPrerequisites">        
+        <Exec Command="$(RestoreCommand)"
+              Condition="'$(OS)' != 'Windows_NT' And Exists('$(PackagesConfig)')" />
+
+        <Exec Command="$(RestoreCommand)"
+              LogStandardErrorAsError="true"
+              Condition="'$(OS)' == 'Windows_NT' And Exists('$(PackagesConfig)')" />
+    </Target>
+
+    <Target Name="BuildPackage" DependsOnTargets="CheckPrerequisites">
+        <Exec Command="$(BuildCommand)"
+              Condition=" '$(OS)' != 'Windows_NT' " />
+
+        <Exec Command="$(BuildCommand)"
+              LogStandardErrorAsError="true"
+              Condition=" '$(OS)' == 'Windows_NT' " />
+    </Target>
+
+    <UsingTask TaskName="DownloadNuGet" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll">
+        <ParameterGroup>
+            <OutputFilename ParameterType="System.String" Required="true" />
+        </ParameterGroup>
+        <Task>
+            <Reference Include="System.Core" />
+            <Using Namespace="System" />
+            <Using Namespace="System.IO" />
+            <Using Namespace="System.Net" />
+            <Using Namespace="Microsoft.Build.Framework" />
+            <Using Namespace="Microsoft.Build.Utilities" />
+            <Code Type="Fragment" Language="cs">
+                <![CDATA[
+                try {
+                    OutputFilename = Path.GetFullPath(OutputFilename);
+
+                    Log.LogMessage("Downloading latest version of NuGet.exe...");
+                    WebClient webClient = new WebClient();
+                    webClient.DownloadFile("https://www.nuget.org/nuget.exe", OutputFilename);
+
+                    return true;
+                }
+                catch (Exception ex) {
+                    Log.LogErrorFromException(ex);
+                    return false;
+                }
+            ]]>
+            </Code>
+        </Task>
+    </UsingTask>
+</Project>
--- a/.nuget/packages.config
+++ b/.nuget/packages.config
@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="NUnit.Runners" version="2.6.3" />
+  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
+  <package id="NUnit.Runners" version="2.6.4" />
+  <package id="RapidRegex.Core" version="1.0.0.4" targetFramework="net40" />
+  <package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
 </packages>
--- a/README.md
+++ b/README.md
@@ -2,6 +2,14 @@ TimberWinR
 ==========
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.

+## Development and Roadmap
+
+TimberWinR's development has been and in the short term will continue to be on a hiatus. The project's ownership has been recently transferred, and thus the project will be slow to pick up steam again. Currently a roadmap is being established, and a vision for the project is currently being formulated.
+
+#### Getting Involved
+
+If you wish to be involved with TimberWinR in an on-going basis file an issue.
+
 ## Why have TimberWinR?
 TimberWinR is a native .NET implementation utilizing Microsoft's [LogParser](http://technet.microsoft.com/en-us/scriptcenter/dd919274.aspx).  This means
 no JVM/JRuby is required, and LogParser does all the heavy lifting.  TimberWinR collects
@@ -9,38 +17,48 @@ the data from LogParser and ships it to Logstash via Redis (or can ship direcly

 ## Release Notes

-[View Version History](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/ReleaseNotes.md)
+[View Version History](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/ReleaseNotes.md)

 ## Basics
 TimberWinR uses a configuration file to control how the logs are collected, filtered and shipped off.  
 These are broken down into:
 1. Inputs  (Collect data from different sources)
 2. Filters (Are applied to all Inputs)
- 3. Outputs (Redis, Elasticsearch or Stdout)
+ 3. Outputs (e.g. Redis, Elasticsearch, Stdout, StatsD)

 ### Support ###
 Please use the TimberWinR Google Group for discussion and support:

 https://groups.google.com/forum/#!forum/timberwinr

+Latest Build:
+
+![alt tag](https://ci.appveyor.com/api/projects/status/qi5x2lg153h1tim6/branch/master?svg=true)
+

 ## Inputs
 The current supported Input format sources are:
- 1. [Logs](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file)
- 2. [Tcp](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on TCP port for JSON messages)
- 3. [IISW3C](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/IISW3CInput.md)(Internet Information Services W3C Format)
- 4. [WindowsEvents](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/WindowsEvents.md) (Windows Event Viewer)
- 5. [Stdin](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) (Standard Input for Debugging)
- 6. [W3C](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format)
- 7. [Udp](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/UdpInput.md) (listens for UDP on port for JSON messages)
+ 1. [Logs](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file)
+ 2. [Tcp](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/TcpInput.md) (listens on TCP port for JSON messages)
+ 3. [IISW3C](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/IISW3CInput.md)(Internet Information Services W3C Format)
+ 4. [WindowsEvents](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/WindowsEvents.md) (Windows Event Viewer)
+ 5. [Stdin](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) (Standard Input for Debugging)
+ 6. [W3C](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format)
+ 7. [Udp](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/UdpInput.md) (listens for UDP on port for JSON messages)
+ 8. [TailFiles](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/TailFiles.md) (Tails log files efficiently)
+ 8. [Generator](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/Generator.md) (Generate logs for testing *New*)
+
+## Codecs
+The current list of supported codecs are:
+ 1. [Multiline](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)

 ## Filters
 The current list of supported filters are:
- 1. [Grok](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/GrokFilter.md)
- 2. [Mutate](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md)
- 3. [Date](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md)
- 4. [Json](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md)
- 5. [GeoIP](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/GeoIPFilter.md)
+ 1. [Grok](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/GrokFilter.md)
+ 2. [Mutate](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/MutateFilter.md)
+ 3. [Date](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/DateFilter.md)
+ 4. [Json](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/JsonFilter.md)
+ 5. [GeoIP](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/GeoIPFilter.md)

 Note that there are now two syntaxes for filters, array and single, if you want more than one filter of the same
 type then you must use the array tag instead of the singular tag. i.e:
@@ -61,8 +79,7 @@ A single Json filter using the single tag (this is only provided as a convienien
    }
 ]
 ```
-
-Multiple Json filters must use the jsonFilters and array syntax
+Multiple Json filters must use the jsonFilters and array syntax, also mutateFilters, grokFilters, dateFilters, geoipFilters.
 ```json
 "Filters": [
    {
@@ -95,9 +112,11 @@ Since TimberWinR only ships to Redis and Elasticsearch, the format generated by
 represented as a JSON Property or Array.

 ## Outputs
-1. [Redis](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md)
-2. [Elasticsearch](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md)
-3. [Stdout](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md)
+1. [Redis](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md)
+2. [Elasticsearch](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md)
+3. [Stdout](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md)
+4. [File](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/FileOutput.md)
+5. [StatsD](https://github.com/tparnell8/TimberWinR/blob/master/TimberWinR/mdocs/StatsD.md)

 ## Sample Configuration
 TimberWinR reads a JSON configuration file, an example file is shown here:
@@ -160,7 +179,7 @@ following options:
 TimberWinR.ServiceHost.exe -configFile:myconfig.json -logLevel:Debug
 ```

-## Automatic Installation via Chocolatey
+## Automatic Installation via Chocolatey (embedded)

 [TimbeWinR Chocolatey](https://chocolatey.org/packages/TimberWinR)

@@ -200,8 +219,8 @@ Options:
 -configFile:       Specifies the path to the JSON config file, or directory which contains .json file(s).
                   Default is -configFile:default.json
 -diagnosticPort:   Specifies the diagnostic port which can be used to get a health check of the service.
-                   Default Port is 5142, A value of 0 will disable it.  Open a browser
-                      http://localhost:5142
+                   Default Port is 5141, A value of 0 will disable it.  Open a browser
+                      http://localhost:5141
 ```
 #### -configFile
 This may be a single .json file or a directory containing .json file(s).  If it is a directory, all
@@ -212,8 +231,8 @@ order on disk.
 If you really just want to try it out, grab the binary distribution, extract the .zip file
 into a directory, e.g.  C:\TimberWinR

-Grab the [JSON example file](https://raw.githubusercontent.com/efontana/TimberWinR/master/TimberWinR.ServiceHost/default.json) and place it into C:\TimberWinR\default.json. 
-Edit the default.json file and change the Redis instance to match yours, replace 'tstlexiceapp006.vistaprint.svc' with the IP or DNS name
+Grab the [JSON example file](https://raw.githubusercontent.com/Cimpress-MCP/TimberWinR/master/TimberWinR.ServiceHost/default.json) and place it into C:\TimberWinR\default.json. 
+Edit the default.json file and change the Redis instance to match yours, replace 'tstlexiceapp006.mycompany.svc' with the IP or DNS name
 of the machine running redis.  Fire up the collector, enable the verbose debugging to see some Windows Events.

 ```
--- a/TimberWinR.ExtractID/Program.cs
+++ b/TimberWinR.ExtractID/Program.cs
@@ -100,8 +100,6 @@ namespace TimberWinR.ExtractID

                return 0;
            }           
-
-            Console.Error.WriteLine("Failed for some reason");
        }
    }
 }
--- a/TimberWinR.ServiceHost/Program.cs
+++ b/TimberWinR.ServiceHost/Program.cs
@@ -7,7 +7,9 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Xml;
 using Microsoft.Win32;
+
 using TimberWinR.Outputs;
 using TimberWinR.ServiceHost;
 using TimberWinR.Inputs;
@@ -39,6 +41,7 @@ namespace TimberWinR.ServiceHost
                    serviceConfigurator.WhenStopped(myService => myService.Stop());
                });

+                hostConfigurator.AddCommandLineDefinition("liveMonitor", c => arguments.LiveMonitor = bool.Parse(c.ToString()));
                hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c);
                hostConfigurator.AddCommandLineDefinition("logLevel", c => arguments.LogLevel = c);
                hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c);
@@ -60,6 +63,7 @@ namespace TimberWinR.ServiceHost
                        AddServiceParameter("-configFile", arguments.ConfigFile);
                        AddServiceParameter("-logLevel", arguments.LogLevel);
                        AddServiceParameter("-logDir", arguments.LogfileDir);
+                        AddServiceParameter("-liveMonitor", arguments.LiveMonitor);
                        if (arguments.DiagnosticPort > 0)
                            AddServiceParameter("-diagnosticPort", arguments.DiagnosticPort);
                    }
@@ -69,7 +73,6 @@ namespace TimberWinR.ServiceHost

        private static void AddServiceParameter(string paramName, string value)
        {
-         
            string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();

            if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0} ", paramName)))
@@ -81,7 +84,6 @@ namespace TimberWinR.ServiceHost

        private static void AddServiceParameter(string paramName, int value)
        {
-    
            string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();

            if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
@@ -91,6 +93,16 @@ namespace TimberWinR.ServiceHost
            }
        }

+        private static void AddServiceParameter(string paramName, bool value)
+        {
+            string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
+
+            if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
+            {
+                currentValue += string.Format(" {0} \"{1}\"", paramName, value.ToString());
+                Registry.SetValue(KeyPath, KeyName, currentValue);
+            }
+        }
    }

    internal class Arguments
@@ -99,9 +111,10 @@ namespace TimberWinR.ServiceHost
        public string LogLevel { get; set; }
        public string LogfileDir { get; set; }
        public int DiagnosticPort { get; set; }
-
+        public bool LiveMonitor { get; set; }
        public Arguments()
        {
+            LiveMonitor = false;
            DiagnosticPort = 5141;
            ConfigFile = "default.json";
            LogLevel = "Info";
@@ -118,6 +131,8 @@ namespace TimberWinR.ServiceHost
        private readonly Arguments _args;
        private TimberWinR.Diagnostics.Diagnostics _diags;
        private TimberWinR.Manager _manager;
+        public bool StartingUp { get; set; }
+        public bool Started { get; set; }

        public TimberWinRService(Arguments args)
        {
@@ -134,6 +149,8 @@ namespace TimberWinR.ServiceHost

        public void Stop()
        {
+            WaitForStartupToComplete();
+
            _cancellationTokenSource.Cancel();
            if (_diags != null)
                _diags.Shutdown();
@@ -142,14 +159,31 @@ namespace TimberWinR.ServiceHost
                _manager.Shutdown();
        }

+        // If you bounce the service too quickly, the shutdown can occur
+        // before the service has started, which results in a hang, this blocks until 
+        // all thread have properly started (waiting up to 10 seconds max) 
+        private void WaitForStartupToComplete()
+        {
+            int tries = 100; // 10 seconds max
+            if (StartingUp)
+            {
+                while (!Started && tries-- >= 0)
+                {                  
+                    Thread.Sleep(100);
+                }
+            }
+        }
+
        /// <summary>
        /// The Main body of the Service Worker Thread
        /// </summary>
        private void RunService()
        {
-            _manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _cancellationToken);
+            StartingUp = true;
+            _manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _args.LiveMonitor, _cancellationToken);
            if (_args.DiagnosticPort > 0)
                _diags = new Diagnostics.Diagnostics(_manager, _cancellationToken, _args.DiagnosticPort);
+            Started = true;
        }
    }
 }
--- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
+++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.19.0")]
-[assembly: AssemblyFileVersion("1.3.19.0")]
+[assembly: AssemblyVersion("1.3.26.0")]
+[assembly: AssemblyFileVersion("1.3.26.0")]
--- a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
+++ b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
@@ -12,6 +12,8 @@
    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <PlatformTarget>AnyCPU</PlatformTarget>
@@ -36,9 +38,16 @@
    <ApplicationIcon>timberwinr.ico</ApplicationIcon>
  </PropertyGroup>
  <ItemGroup>
-    <Reference Include="RapidRegex.Core, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
+    <Reference Include="RapidRegex.Core, Version=1.0.0.4, Culture=neutral, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\RapidRegex.Core.1.0.0.2\lib\net40\RapidRegex.Core.dll</HintPath>
+      <HintPath>..\packages\RapidRegex.Core.1.0.0.4\lib\net40\RapidRegex.Core.dll</HintPath>
+    </Reference>
+    <Reference Include="StatsdClient, Version=1.0.0.19, Culture=neutral, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.dll</HintPath>
+    </Reference>
+    <Reference Include="StatsdClient.Configuration">
+      <HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.Configuration.dll</HintPath>
    </Reference>
    <Reference Include="System" />
    <Reference Include="System.Configuration.Install" />
@@ -49,9 +58,8 @@
    <Reference Include="Microsoft.CSharp" />
    <Reference Include="System.Data" />
    <Reference Include="System.Xml" />
-    <Reference Include="Topshelf, Version=3.1.122.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll</HintPath>
+    <Reference Include="Topshelf">
+      <HintPath>..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll</HintPath>
    </Reference>
  </ItemGroup>
  <ItemGroup>
@@ -87,6 +95,13 @@
    </ProjectReference>
  </ItemGroup>
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
--- a/TimberWinR.ServiceHost/config.json
+++ b/TimberWinR.ServiceHost/config.json
@@ -90,7 +90,7 @@
                    "interval": 5000, 
                    "batch_count":  500,              
                    "host": [
-                        "tstlexiceapp006.vistaprint.svc"
+                        "tstlexiceapp006.mycompany.svc"
                    ]
                }
            ],            
@@ -99,7 +99,7 @@
                    "threads":  1,   
                    "interval": 5000,                          
                    "host": [
-                        "tstlexiceapp003.vistaprint.svc"
+                        "tstlexiceapp003.mycompany.svc"
                    ]
                }
            ]                           
--- a/TimberWinR.ServiceHost/default.json
+++ b/TimberWinR.ServiceHost/default.json
@@ -18,7 +18,7 @@
        "Filters": [
            {
                "grok": {
-                    "condition": "[EventTypeName] == \"Information Event\"",
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
                    "match": [
                        "Text",
                        ""
@@ -33,7 +33,7 @@
                    "_comment": "Change the host to your Redis instance",
                    "port": 6379,
                    "host": [
-                        "logaggregator.vistaprint.svc"
+                        "logaggregator.mycompany.svc"
                    ]
                }
            ]
--- a/TimberWinR.ServiceHost/packages.config
+++ b/TimberWinR.ServiceHost/packages.config
@@ -1,5 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
-  <package id="Topshelf" version="3.1.3" targetFramework="net40" />
+  <package id="RapidRegex.Core" version="1.0.0.4" targetFramework="net40" />
+  <package id="StatsdClient" version="1.0.0.19" targetFramework="net40" />
+  <package id="Topshelf" version="3.1.4" targetFramework="net40" />
 </packages>
--- a/TimberWinR.TestGenerator/CommandLineOptions.cs
+++ b/TimberWinR.TestGenerator/CommandLineOptions.cs
@@ -0,0 +1,88 @@
+using System.ComponentModel;
+using CommandLine;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using CommandLine.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class CommandLineOptions
+    {
+       // [Option('r', "read", Required = true, HelpText = "Input file to be processed.")]
+      //  public string InputFile { get; set; }
+        [Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")]
+        public string TimberWinRConfigFile { get; set; }
+
+        [Option("start", HelpText = "Start an instance of TimberWinR")]
+        public bool StartTimberWinR { get; set; }
+
+        [Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")]
+        public string TestDir { get; set; }
+
+        [Option("testFile", DefaultValue = "", HelpText = "Config file/directory to use")]
+        public string TestFile { get; set; }
+
+        [Option("resultsFile", HelpText = "Expected results Results json file")]
+        public string ExpectedResultsFile { get; set; }
+
+        [Option("totalMessages", DefaultValue = 0, HelpText = "The total number of messages to send to the output(s)")]
+        public int TotalMessages { get; set; }
+
+        [Option('n', "numMessages", DefaultValue = 1000, HelpText = "The number of messages to send to the output(s)")]
+        public int NumMessages { get; set; }
+
+        [Option('l', "logLevel", DefaultValue = "debug", HelpText = "Logging Level Debug|Error|Fatal|Info|Off|Trace|Warn")]
+        public string LogLevel { get; set; }
+
+        [Option('v', "verbose", DefaultValue = true, HelpText = "Prints all messages to standard output.")]
+        public bool Verbose { get; set; }
+
+        [Option("jsonLogDir", DefaultValue = ".", HelpText = "Json LogGenerator Log directory")]
+        public string JsonLogDir { get; set; }
+
+        [OptionArray('j', "json", DefaultValue = new string[] {})]
+        public string[] JsonLogFiles { get; set; }
+
+        [OptionArray("jroll", DefaultValue = new string[] { })]
+        public string[] JsonRollingLogFiles { get; set; }
+        
+        [Option("jsonRate", DefaultValue = 30, HelpText = "Json Rate in Milliseconds between generation of log lines")]
+        public int JsonRate { get; set; }
+
+        [Option('u', "udp", DefaultValue = 0, HelpText = "Enable UDP generator on this Port")]
+        public int Udp { get; set; }
+
+        [Option("udp-host", DefaultValue = "localhost", HelpText = "Host to send Udp data to")]
+        public string UdpHost { get; set; }
+
+        [Option("udp-rate", DefaultValue = 10, HelpText = "Udp Rate in Milliseconds between generation of log lines")]
+        public int UdpRate { get; set; }
+
+        [Option('t', "tcp", DefaultValue = 0, HelpText = "Enable Tcp generator on this Port")]
+        public int Tcp { get; set; }
+
+        [Option("tcp-host", DefaultValue = "localhost", HelpText = "Host to send Tcp data to")]
+        public string TcpHost { get; set; }
+
+        [Option("tcp-rate", DefaultValue = 10, HelpText = "Tcp Rate in Milliseconds between generation of log lines")]
+        public int TcpRate { get; set; }
+
+        [Option('r', "redis", DefaultValue = 0, HelpText = "Enable Redis generator on this Port")]
+        public int Redis { get; set; }
+
+        [Option("redis-host", DefaultValue = "", HelpText = "Host to send Redis data to")]
+        public string RedisHost { get; set; }
+
+        [ParserState]
+        public IParserState LastParserState { get; set; }
+
+        [HelpOption]
+        public string GetUsage()
+        {
+            return HelpText.AutoBuild(this,
+              (HelpText current) => HelpText.DefaultParsingErrorsHandler(this, current));
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/Dynamic.cs
+++ b/TimberWinR.TestGenerator/Dynamic.cs
--- a/TimberWinR.TestGenerator/JsonLogFileGenerator.cs
+++ b/TimberWinR.TestGenerator/JsonLogFileGenerator.cs
@@ -0,0 +1,251 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.IO;
+
+namespace TimberWinR.TestGenerator
+{
+    class JsonLogFileTestParameters
+    {
+        public int NumMessages { get; set; }
+        public string LogFileDir { get; set; }
+        public string LogFileName { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public JsonLogFileTestParameters()
+        {
+            SleepTimeMilliseconds = 30;
+            LogFileDir = ".";
+            NumMessages = 10;
+        }
+    }
+
+    class JsonLogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start JSON LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                {
+                    LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath);
+                    File.Delete(logFilePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var watch = Stopwatch.StartNew();
+
+            // This text is always added, making the file longer over time 
+            // if it is not deleted. 
+            using (StreamWriter sw = File.AppendText(logFilePath))
+            {               
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"LineNumber", i+1},
+                        {"Application", "jsonlogfile-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "jsonlog"},
+                        {"Message", string.Format("{0}: Testgenerator jsonlogfile message {1}", i+1, DateTime.UtcNow.ToString("o"))},
+                        {"Index", "logstash"}
+                    };
+                    sw.WriteLine(o.ToString(Formatting.None));
+
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+                LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+                watch.Reset();
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished JSON Log File Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+    }
+
+    class JsonRollingLogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start JSON RollingLogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                    File.Delete(logFilePath);
+
+                if (File.Exists(logFilePath + ".rolled"))
+                    File.Delete(logFilePath + ".rolled");
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+         
+            int quarters = parms.NumMessages/4;
+
+            int[] segments = new int[] {quarters, quarters, quarters, quarters + parms.NumMessages%4};
+            var watch = Stopwatch.StartNew();
+
+
+            int recordNumber = 0;
+            int currentTotal = 0;
+            for (int segment = 0; segment < 4; segment++)
+            {
+                currentTotal += segments[segment];
+             
+                // This text is always added, making the file longer over time 
+                // if it is not deleted. 
+                using (StreamWriter sw = File.AppendText(logFilePath))
+                {
+                    sw.AutoFlush = true;
+   
+                    var lwatch = Stopwatch.StartNew();
+
+                    // The Rolling Generator will roll 1/2 way through the log             
+                    for (int i = 0; i < segments[segment]; i++)
+                    {
+                        JObject o = new JObject
+                        {
+                            {"LineNumber", recordNumber + 1},
+                            {"Application", "jsonrollinglogfile-generator"},
+                            {"Host", hostName},
+                            {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                            {"Type", "jsonrollinglog"},
+                            {
+                                "Message",
+                                string.Format("{0}: Testgenerator jsonrollinglogfile message {1}", recordNumber + 1,
+                                    DateTime.UtcNow.ToString("o"))
+                            },
+                            {"Index", "logstash"}
+                        };
+                        sw.WriteLine(o.ToString(Formatting.None));
+                        recordNumber++;
+                        Thread.Sleep(parms.SleepTimeMilliseconds);
+                    }
+                    LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds for {2} logs", Path.GetFullPath(parms.LogFileName), lwatch.Elapsed, segments[segment]);
+                   
+                }
+               
+                // 
+                // We might not have yet processed all the lines from the first file, so wait till
+                // we catch up before rolling the log file.
+                //
+                LogManager.GetCurrentClassLogger().Info("{0}: Waiting for output to catch up: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, currentTotal);
+                WaitOutputToCatchUp(logFilePath, currentTotal);
+
+                //
+                // Roll the log + wait for the reader to catch up.
+                //  
+                
+                LogManager.GetCurrentClassLogger().Info("{0}: Rolling Log File: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, File.GetCreationTimeUtc(logFilePath));
+
+                RollLogFile(logFilePath); 
+                
+                LogManager.GetCurrentClassLogger().Info("{0}: Finished Rolling Log File: {1}", Thread.CurrentThread.ManagedThreadId, logFilePath);           
+            }
+            
+            watch.Stop();
+           
+            LogManager.GetCurrentClassLogger().Info("Finished JSON RollingLogFile File Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+
+        private static void WaitOutputToCatchUp(string logFilePath, int firstPart)
+        {
+            bool caughtUp = false;
+            do
+            {
+                var json = Program.Diagnostics.DiagnosticsOutput();
+
+                IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+                foreach (JToken t in inputs)
+                {
+                    JProperty inputProp = t.First as JProperty;
+                    if (inputProp.Name == "taillog" || inputProp.Name == "log")
+                    {
+                        var files = inputProp.Value["filedb"].Children().ToList();
+                        foreach (var file in files)
+                        {
+                            var fileName = file["FileName"].ToString();
+                            FileInfo fi1 = new FileInfo(fileName);
+                            FileInfo fi2 = new FileInfo(logFilePath);
+                            if (fi1.FullName == fi2.FullName)
+                            {
+                                var linesProcessed = file["LinesProcessed"].Value<int>();
+                                if (linesProcessed >= firstPart)
+                                {
+                                    caughtUp = true;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                Thread.Sleep(300);
+            } while (!caughtUp);
+
+            LogManager.GetCurrentClassLogger().Info("{0}:  Finished Waiting for output to catch up: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, firstPart);            
+
+        }
+
+        private static void RollLogFile(string logFilePath)
+        {
+            bool moved = false;           
+            do
+            {
+                try
+                {
+                    if (File.Exists(logFilePath + ".rolled"))
+                        File.Delete(logFilePath + ".rolled");
+
+                    File.Move(logFilePath, logFilePath + ".rolled");
+                    moved = true;                   
+                }
+                catch (Exception)
+                {
+                    Thread.Sleep(100);
+                }
+            } while (!moved);
+            Thread.Sleep(1000);
+        }
+    }
+
+}
--- a/TimberWinR.TestGenerator/LogFileGenerator.cs
+++ b/TimberWinR.TestGenerator/LogFileGenerator.cs
@@ -0,0 +1,93 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.IO;
+
+
+namespace TimberWinR.TestGenerator
+{
+    class LogFileTestParameters
+    {
+        public int NumMessages { get; set; }
+        public string LogFileDir { get; set; }
+        public string LogFileName { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public LogFileTestParameters()
+        {
+            SleepTimeMilliseconds = 30;
+            LogFileDir = ".";
+            NumMessages = 10;
+        }
+    }
+
+    class LogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                {
+                    LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath);
+                    File.Delete(logFilePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var watch = Stopwatch.StartNew();
+
+            // This text is always added, making the file longer over time 
+            // if it is not deleted. 
+            using (StreamWriter sw = File.AppendText(logFilePath))
+            {              
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"LineNumber", i+1},
+                        {"Application", "logfile-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "log"},
+                        {"Message", string.Format("{0}: Testgenerator logfile message {1}", i+1, DateTime.UtcNow.ToString("o"))},
+                        {"Index", "logstash"}
+                    };
+                    sw.WriteLine(o.ToString(Formatting.None));
+
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+                LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+                watch.Reset();
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished LogFile Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/Program.cs
+++ b/TimberWinR.TestGenerator/Program.cs
@@ -0,0 +1,646 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Net;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.CodeDom.Compiler;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+using ServiceStack.Text.Jsv;
+using TimberWinR.Parser;
+
+
+namespace TimberWinR.TestGenerator
+{
+    public class Program
+    {
+        private static List<Task> _tasks = new List<Task>();
+        private static CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
+        private static Manager _timberWinR;
+
+        public static Diagnostics.Diagnostics Diagnostics { get; set; }
+
+        private static PerformanceCounter cpuCounter = new PerformanceCounter();
+        private static PerformanceCounter ramCounter = new PerformanceCounter();
+        private static Task _monitorTask;
+
+        private static int _totalMessagesToSend;
+        private static int _cpuSampleCount;
+        private static double _avgCpuUsage;
+        private static double _totalCpuUsage;
+        private static double _maxCpuUsage;
+
+        private static int _memSampleCount;
+        private static double _avgMemUsage;
+        private static double _totalMemUsage;
+        private static double _maxMemUsage;
+
+        private static CommandLineOptions Options;
+
+        static int Main(string[] args)
+        {
+            _totalMessagesToSend = 0;
+
+            cpuCounter.CategoryName = "Processor";
+            cpuCounter.CounterName = "% Processor Time";
+            cpuCounter.InstanceName = "_Total";
+
+            ramCounter.CategoryName = "Memory";
+            ramCounter.CounterName = "% Committed Bytes In Use";
+
+            Options = new CommandLineOptions();
+
+            if (CommandLine.Parser.Default.ParseArguments(args, Options))
+            {
+                var testFile = Options.TestFile;
+                if (!string.IsNullOrEmpty(testFile))
+                {
+                    if (!File.Exists(Options.TestFile))
+                        throw new Exception(string.Format("No such test file: {0} found", Options.TestFile));
+
+                    var fargs = ParseTestArguments(testFile, ref Options);
+                    if (!CommandLine.Parser.Default.ParseArguments(fargs, Options))
+                        return 2;
+                }
+
+                SetupTestDirectory(Options);
+
+                var swOverall = Stopwatch.StartNew();
+                swOverall.Start();
+
+                InitializeLogging(Options.LogLevel);
+
+                LogManager.GetCurrentClassLogger().Info("Starting CPU Usage: {0}, RAM Usage: {1}", getCurrentCpuUsage(), getAvailableRAM());
+
+                // Reset the tests.
+                ResetTests(Options);
+
+                var sw = Stopwatch.StartNew();
+
+                // Startup TimberWinR
+                if (Options.StartTimberWinR)
+                    StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
+
+                // Run the Generators
+                var arrayOfTasks = RunGenerators(Options);
+
+                // Wait for all Generators to finish
+                try
+                {
+                    Task.WaitAll(arrayOfTasks);
+                }
+                catch (AggregateException aex)
+                {
+                    LogManager.GetCurrentClassLogger().Error(aex);
+                }
+
+
+                LogManager.GetCurrentClassLogger().Info("Generation Finished: " + sw.Elapsed);
+                sw.Reset();
+                sw.Start();
+
+                // All generators are finished, wait till senders are done.
+                WaitForOutputTransmission();
+
+                LogManager.GetCurrentClassLogger().Info("Finished Transmission: " + sw.Elapsed);
+                sw.Reset();
+                sw.Start();
+
+                // Get all the stats
+                JObject jsonTimberWinr;
+
+                if (Options.StartTimberWinR)
+                    jsonTimberWinr = ShutdownTimberWinR();
+                else
+                {
+                    jsonTimberWinr = GetDiagnosticsOutput();
+                    if (jsonTimberWinr == null)
+                        return 3;
+                }
+
+                LogManager.GetCurrentClassLogger().Info("Finished Shutdown: " + sw.Elapsed);
+                sw.Stop();
+
+                swOverall.Stop();
+                LogManager.GetCurrentClassLogger().Info("Total Elapsed Time: {0}", swOverall.Elapsed);
+
+                int results = VerifyResults(Options, jsonTimberWinr);
+
+                Console.ReadKey();
+                return results;
+            }
+
+            return 1;
+        }
+
+        public static string GET(string url)
+        {
+            try
+            {
+                HttpWebRequest request = (HttpWebRequest)WebRequest.Create(url);
+                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
+                Stream stream = response.GetResponseStream();
+                StreamReader reader = new StreamReader(stream);
+
+                string data = reader.ReadToEnd();
+
+                reader.Close();
+                stream.Close();
+
+                return data;
+            }
+            catch (Exception e)
+            {
+                LogManager.GetCurrentClassLogger().ErrorException("Error in GET", e);
+            }
+
+            return null;
+        }
+
+        private static void CopySourceFile(string fileName, string outputDir)
+        {
+            FileInfo fi = new FileInfo(fileName);
+            if (fi.Exists)
+                File.Copy(fileName, Path.Combine(outputDir, fi.Name));
+        }
+
+        private static void SetupTestDirectory(CommandLineOptions options)
+        {
+            if (options.TestDir != "." && Directory.Exists(options.TestDir))
+                Directory.Delete(options.TestDir, true);
+
+            if (!Directory.Exists(options.TestDir))
+                Directory.CreateDirectory(options.TestDir);
+
+            CopySourceFile(options.TestFile, options.TestDir);
+            CopySourceFile(options.TimberWinRConfigFile, options.TestDir);
+            CopySourceFile(options.ExpectedResultsFile, options.TestDir);
+
+            Directory.SetCurrentDirectory(options.TestDir);
+        }
+
+        private static string[] ParseTestArguments(string testFile, ref CommandLineOptions options)
+        {
+            options = new CommandLineOptions();
+            JObject jtest = JObject.Parse(File.ReadAllText(testFile));
+            IList<JToken> inputs = jtest["arguments"].Children().ToList();
+            List<string> testargs = new List<string>();
+            foreach (JProperty it in inputs)
+            {
+                testargs.Add(it.Name);
+
+                var cc = it.Value.Children().Count();
+                if (cc > 0)
+                {
+                    for (int i = 0; i < cc; i++)
+                    {
+                        testargs.Add(it.Value[i].ToString());
+                    }
+                }
+                else
+                {
+                    testargs.Add(it.Value.ToString());
+                }
+            }
+            var fargs = testargs.ToArray();
+            return fargs;
+        }
+
+        private static int VerifyResults(CommandLineOptions options, JObject json)
+        {
+            var jresult = JObject.Parse(File.ReadAllText(options.ExpectedResultsFile));
+
+            json["maxCpuUsage"] = _maxCpuUsage;
+            json["avgCpuUsage"] = _avgCpuUsage;
+
+            json["maxMemUsage"] = _maxMemUsage;
+            json["avgMemUsage"] = _avgMemUsage;
+
+            // TailLogs
+
+            IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+            foreach (JToken t in inputs)
+            {
+                JProperty inputProp = t.First as JProperty;
+                switch (inputProp.Name)
+                {
+                    case "udp":
+                        if (VerifyConditions(json, new string[] { "udp" }, inputProp, jresult) != 0)
+                            return 1;
+                        break;
+                    case "tcp":
+                        if (VerifyConditions(json, new string[] { "tcp" }, inputProp, jresult) != 0)
+                            return 1;
+                        break;
+                    case "log":
+                    case "taillog":
+                        if (VerifyConditions(json, new string[] { "log", "taillog" }, inputProp, jresult) != 0)
+                            return 1;
+                        break;
+                }
+            }
+
+            return 0;
+        }
+
+        private static int VerifyConditions(JObject json, string[] logTypes, JProperty inputProp, JObject jresult)
+        {
+            var ttail = inputProp.Value as JObject;
+            foreach (var resultInput in jresult["Results"]["Inputs"].Children().ToList())
+            {
+                JProperty rinputProp = resultInput.First as JProperty;
+                if (logTypes.Contains(rinputProp.Name))
+                {
+                    foreach (JProperty testProp in rinputProp.Value)
+                    {
+                        try
+                        {
+                            var cond1 = testProp.Value.ToString();
+                            IList<string> tkeys = ttail.Properties().Select(pn => pn.Name).ToList();
+                            foreach (string tkey in tkeys)
+                                cond1 = cond1.Replace(string.Format("[{0}]", tkey), string.Format("{0}", ttail[tkey].ToString()));
+
+                            // Add builtins
+                            cond1 = cond1.Replace("[avgCpuUsage]", json["avgCpuUsage"].ToString());
+                            cond1 = cond1.Replace("[maxCpuUsage]", json["maxCpuUsage"].ToString());
+                            cond1 = cond1.Replace("[avgMemUsage]", json["avgMemUsage"].ToString());
+                            cond1 = cond1.Replace("[maxMemUsage]", json["maxMemUsage"].ToString());
+
+                            var p1 = Expression.Parameter(typeof(JObject), "json");
+                            var e1 = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] { p1 },
+                                typeof(bool), cond1);
+                            bool r1 = (bool)e1.Compile().DynamicInvoke(ttail);
+                            if (!r1)
+                            {
+                                LogManager.GetCurrentClassLogger().Error("Test Failed: '{0}: ({1})'", testProp.Name, cond1);
+                                return 1;
+
+                            }
+                            else
+                            {
+                                LogManager.GetCurrentClassLogger()
+                                    .Info("PASSED({0}): '{1}: ({2})'", inputProp.Name, testProp.Name, cond1);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            LogManager.GetCurrentClassLogger()
+                                .Error("Error parsing expression '{0}': {1}", testProp.Value.ToString(),
+                                    ex.Message);
+                            return 2;
+                        }
+                    }
+                }
+            }
+            return 0;
+        }
+
+
+        private static JObject GetDiagnosticsOutput()
+        {
+            if (Diagnostics != null)
+                return Diagnostics.DiagnosticsOutput();
+            else
+            {
+                var jsonDiag = GET("http://localhost:5141");
+                if (jsonDiag == null)
+                {
+                    LogManager.GetCurrentClassLogger().Error("TimberWinR diagnostics port not responding.");
+                    return null;
+                }
+                return JObject.Parse(jsonDiag);
+            }        
+        }
+
+        // Wait till all output has been transmitted.
+        private static void WaitForOutputTransmission()
+        {
+            bool completed = false;
+            do
+            {
+                var json = GetDiagnosticsOutput();
+                if (json == null)
+                    return;
+
+                //Console.WriteLine(json.ToString(Formatting.Indented));
+
+                IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+                foreach (var so in inputs.Children())
+                {
+                    var token = so.First;
+                    var messages = token["messages"].Value<int>();
+                    //  Console.WriteLine("{0} messages", messages);
+                }
+
+
+                IList<JToken> outputs = json["timberwinr"]["outputs"].Children().ToList();
+                foreach (var so in outputs.Children())
+                {
+                    var outputToken = so.First;
+
+                    var mbc = outputToken["queuedMessageCount"].Value<int>();
+                    var smc = outputToken["sentMessageCount"].Value<int>();
+
+                    //LogManager.GetCurrentClassLogger().Info("Output: {2} Queued: {0}, Sent: {1}", mbc, smc, outputToken.ToString());
+
+                    completed = mbc == 0 && smc >= _totalMessagesToSend;
+                }
+                Thread.Sleep(250);
+            } while (!completed);
+        }
+
+        private static void sampleUsages()
+        {
+            getCurrentCpuUsage();
+            getAvailableRAM();
+        }
+
+        private static string getCurrentCpuUsage()
+        {
+            _cpuSampleCount++;
+            var v = cpuCounter.NextValue();
+            if (v > _maxCpuUsage)
+                _maxCpuUsage = v;
+
+            _totalCpuUsage += v;
+            _avgCpuUsage = _totalCpuUsage / _cpuSampleCount;
+
+            return v + "%";
+        }
+
+        private static string getAvailableRAM()
+        {
+            _memSampleCount++;
+            var v = ramCounter.NextValue();
+            if (v > _maxMemUsage)
+                _maxMemUsage = v;
+
+            _totalMemUsage += v;
+            _avgMemUsage = _totalMemUsage / _memSampleCount;
+            return v + "MB";
+        }
+
+        private static JObject ShutdownTimberWinR()
+        {
+            if (_timberWinR != null)
+            {
+                // Cancel any/all other threads
+                _cancellationTokenSource.Cancel();
+
+                _timberWinR.Shutdown();
+
+                var json = Diagnostics.DiagnosticsOutput();
+
+                LogManager.GetCurrentClassLogger()
+                    .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB",
+                        _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
+
+                LogManager.GetCurrentClassLogger().Info(json.ToString());
+
+                Diagnostics.Shutdown();
+
+                return json;
+            }
+
+            return new JObject();
+        }
+
+        static void StartTimberWinR(string configFile, string logLevel, string logFileDir, bool enableLiveMonitor)
+        {
+            _timberWinR = new TimberWinR.Manager(configFile, logLevel, logFileDir, enableLiveMonitor, _cancellationTokenSource.Token, false);
+            _timberWinR.OnConfigurationProcessed += TimberWinROnOnConfigurationProcessed;
+            _timberWinR.Start(_cancellationTokenSource.Token);
+            Diagnostics = new Diagnostics.Diagnostics(_timberWinR, _cancellationTokenSource.Token, 5141);
+        }
+
+        private static void TimberWinROnOnConfigurationProcessed(Configuration configuration)
+        {
+            if (!string.IsNullOrEmpty(Options.RedisHost) && configuration.RedisOutputs != null && configuration.RedisOutputs.Count() > 0)
+            {
+                foreach (var ro in configuration.RedisOutputs)
+                {
+                    ro.Host = new string[] { Options.RedisHost };
+                }
+            }
+
+        }
+
+        static void InitializeLogging(string logLevel)
+        {
+            var loggingConfiguration = new LoggingConfiguration();
+
+            // Create our default targets
+            var coloredConsoleTarget = new ColoredConsoleTarget();
+
+            var logFileDir = ".";
+
+            Target fileTarget = CreateDefaultFileTarget(logFileDir);
+
+            loggingConfiguration.AddTarget("Console", coloredConsoleTarget);
+            loggingConfiguration.AddTarget("DailyFile", fileTarget);
+
+            // The LogLevel.Trace means has to be at least Trace to show up on console
+            loggingConfiguration.LoggingRules.Add(new LoggingRule("*", LogLevel.Trace, coloredConsoleTarget));
+            // LogLevel.Debug means has to be at least Debug to show up in logfile
+            loggingConfiguration.LoggingRules.Add(new LoggingRule("*", LogLevel.Debug, fileTarget));
+
+            LogManager.Configuration = loggingConfiguration;
+            LogManager.EnableLogging();
+
+            LogManager.GlobalThreshold = LogLevel.FromString(logLevel);
+        }
+
+        static FileTarget CreateDefaultFileTarget(string logPath)
+        {
+            return new FileTarget
+            {
+                ArchiveEvery = FileArchivePeriod.None,
+                ArchiveAboveSize = 5 * 1024 * 1024,
+                MaxArchiveFiles = 5,
+                BufferSize = 10,
+                FileName = Path.Combine(logPath, "TimberWinR.TestGenerator", "TimberWinRTestGen.log"),
+                ArchiveFileName = Path.Combine(logPath, "TimberWinR-TestGenerator_log-{#######}.log"),
+            };
+        }
+
+        static void ResetTests(CommandLineOptions options)
+        {
+            if (File.Exists(".timberwinrdb"))
+                File.Delete(".timberwinrdb");
+
+            if (File.Exists("TimberWinR.TestGenerator\\TimberWinRTestGen.log"))
+                File.Delete("TimberWinR.TestGenerator\\TimberWinRTestGen.log");
+
+            if (File.Exists("TimberWinR\\TimberWinR.log"))
+                File.Delete("TimberWinR\\TimberWinR.log");
+
+            if (options.JsonLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonLogFiles)
+                {
+                    if (File.Exists(logFile))
+                        File.Delete(logFile);
+                }
+            }
+
+            if (options.JsonRollingLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonRollingLogFiles)
+                {
+                    if (File.Exists(logFile))
+                        File.Delete(logFile);
+                }
+            }
+        }
+
+        static Task[] RunGenerators(CommandLineOptions options)
+        {
+            _totalMessagesToSend = options.TotalMessages;
+
+            _monitorTask = Task.Factory.StartNew(() =>
+                {
+                    using (var syncHandle = new ManualResetEventSlim())
+                    {
+                        try
+                        {
+                            // Execute the query
+                            while (!_cancellationTokenSource.Token.IsCancellationRequested)
+                            {
+                                sampleUsages();
+                                // LogManager.GetCurrentClassLogger().Info("Starting CPU Usage: {0}, RAM Usage: {1}", getCurrentCpuUsage(), getAvailableRAM());
+                                syncHandle.Wait(TimeSpan.FromMilliseconds(options.JsonRate), _cancellationTokenSource.Token);
+                            }
+                        }
+                        catch (OperationCanceledException)
+                        {
+                        }
+                        catch (Exception ex)
+                        {
+                            LogManager.GetCurrentClassLogger().Error(ex);
+                        }
+                    }
+                }, _cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Current);
+
+            StartJson(options);
+            StartJsonRolling(options);
+            StartUdp(options);
+            StartTcp(options);
+
+            return _tasks.ToArray();
+        }
+
+        static void StartJson(CommandLineOptions options)
+        {
+            if (options.JsonLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonLogFiles)
+                {
+                    _totalMessagesToSend += options.NumMessages;
+
+                    if (options.Verbose)
+                        LogManager.GetCurrentClassLogger()
+                            .Info("Starting LogFile Generator for {0}",
+                                Path.GetFullPath(Path.Combine(options.JsonLogDir, logFile)));
+                    _tasks.Add(Task.Factory.StartNew(() =>
+                    {
+                        var p = new JsonLogFileTestParameters()
+                        {
+                            NumMessages = options.NumMessages,
+                            LogFileDir = options.JsonLogDir,
+                            LogFileName = logFile,
+                            SleepTimeMilliseconds = options.JsonRate
+                        };
+                        JsonLogFileGenerator.Generate(p);
+                        Thread.Sleep(250);
+                    }));
+
+                }
+            }
+        }
+
+        private static void StartJsonRolling(CommandLineOptions options)
+        {
+            if (options.JsonRollingLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonRollingLogFiles)
+                {
+                    _totalMessagesToSend += options.NumMessages;
+
+                    if (options.Verbose)
+                        LogManager.GetCurrentClassLogger()
+                            .Info("Starting RollingLogFile Generator for {0}",
+                                Path.GetFullPath(Path.Combine(options.JsonLogDir, logFile)));
+                    _tasks.Add(Task.Factory.StartNew(() =>
+                    {
+                        var p = new JsonLogFileTestParameters()
+                        {
+                            NumMessages = options.NumMessages,
+                            LogFileDir = options.JsonLogDir,
+                            LogFileName = logFile,
+                            SleepTimeMilliseconds = options.JsonRate
+                        };
+                        JsonRollingLogFileGenerator.Generate(p);
+                        Thread.Sleep(250);
+                    }));
+
+                }
+            }
+        }
+
+        static void StartUdp(CommandLineOptions options)
+        {
+            if (options.Udp > 0)
+            {
+                if (options.Verbose)
+                    LogManager.GetCurrentClassLogger()
+                        .Info("Starting UDP Generator for {0}:{1}", options.UdpHost, options.Udp);
+
+                _tasks.Add(Task.Factory.StartNew(() =>
+                {
+                    var p = new UdpTestParameters()
+                    {
+                        Port = options.Udp,
+                        Host = options.UdpHost,
+                        NumMessages = options.NumMessages,
+                        SleepTimeMilliseconds = options.UdpRate
+                    };
+                    _totalMessagesToSend += UdpTestGenerator.Generate(p);
+                }));
+            }
+        }
+
+        static void StartTcp(CommandLineOptions options)
+        {
+            if (options.Tcp > 0)
+            {
+                if (options.Verbose)
+                    LogManager.GetCurrentClassLogger()
+                        .Info("Starting Tcp Generator for {0}:{1}", options.TcpHost, options.Tcp);
+
+                _totalMessagesToSend += options.NumMessages;
+
+                _tasks.Add(Task.Factory.StartNew(() =>
+                {
+                    var p = new TcpTestParameters()
+                    {
+                        Port = options.Tcp,
+                        Host = options.TcpHost,
+                        NumMessages = options.NumMessages,
+                        SleepTimeMilliseconds = options.TcpRate
+                    };
+                    TcpTestGenerator.Generate(p);
+                }));
+            }
+        }
+
+    }
+}
--- a/TimberWinR.TestGenerator/Properties/AssemblyInfo.cs
+++ b/TimberWinR.TestGenerator/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("TimberWinR.TestGenerator")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("TimberWinR.TestGenerator")]
+[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("a56bf91c-c5f8-4771-8ef8-ab9ad28179c4")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
--- a/TimberWinR.TestGenerator/RedisTestGenerator.cs
+++ b/TimberWinR.TestGenerator/RedisTestGenerator.cs
@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+using ServiceStack.Redis;
+
+namespace TimberWinR.TestGenerator
+{
+    class RedisTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public RedisTestParameters()
+        {
+            NumMessages = 100;
+            Port = 6379;
+            Host = "localhost";
+        }
+    }
+
+    class RedisTestGenerator
+    {
+        public static void Generate(RedisTestParameters parms)
+        {
+            var hostName = System.Environment.MachineName + "." +
+               Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                   "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var rc = new RedisClient(parms.Host, parms.Port);
+
+            for (int i = 0; i < parms.NumMessages; i++)
+            {
+                JObject o = new JObject
+                {
+                    {"Application", "redis-generator"},               
+                    {"Host", hostName},
+                    {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                    {"Type", "redis"},                
+                    {"Message", "redis message " + DateTime.UtcNow.ToString("o")},
+                    {"Index", "logstash"}
+                };
+                byte[] bytes = System.Text.Encoding.UTF8.GetBytes(o.ToString());
+                var restult = rc.RPush("logstash", bytes);
+            }
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/TcpTestGenerator.cs
+++ b/TimberWinR.TestGenerator/TcpTestGenerator.cs
@@ -0,0 +1,62 @@
+using System.Threading;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using ServiceStack.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class TcpTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public TcpTestParameters()
+        {
+            NumMessages = 100;
+            Port = 5140;
+            Host = "localhost";
+            SleepTimeMilliseconds = 10;
+        }
+    }
+
+    class TcpTestGenerator
+    {
+        public static int Generate(TcpTestParameters parms)
+        {
+            TcpClient server = new TcpClient(parms.Host, parms.Port);
+            
+            var hostName = System.Environment.MachineName + "." +
+                       Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                           "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+
+            using (NetworkStream stream = server.GetStream())
+            {              
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"Application", "tcp-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "tcp"},
+                        {"Message", "tcp message " + DateTime.UtcNow.ToString("o")},
+                        {"Index", "logstash"}
+                    };
+                    byte[] data = Encoding.UTF8.GetBytes(string.Format("{0}\n", o.ToString()));
+                    stream.Write(data, 0, data.Length);
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+            }
+
+            return parms.NumMessages;
+        }
+
+    }
+}
--- a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
+++ b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
@@ -0,0 +1,163 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>TimberWinR.TestGenerator</RootNamespace>
+    <AssemblyName>TimberWinR.TestGenerator</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="CommandLine">
+      <HintPath>..\packages\CommandLineParser.1.9.71\lib\net40\CommandLine.dll</HintPath>
+    </Reference>
+    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="NLog, Version=3.2.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Common">
+      <HintPath>..\packages\ServiceStack.Common.Signed.4.0.38\lib\net40\ServiceStack.Common.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Interfaces">
+      <HintPath>..\packages\ServiceStack.Interfaces.4.0.38\lib\portable-wp80+sl5+net40+win8+monotouch+monoandroid\ServiceStack.Interfaces.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Redis">
+      <HintPath>..\packages\ServiceStack.Redis.Signed.4.0.38\lib\net40\ServiceStack.Redis.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Text">
+      <HintPath>..\packages\ServiceStack.Text.Signed.4.0.38\lib\net40\ServiceStack.Text.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CommandLineOptions.cs" />
+    <Compile Include="Dynamic.cs" />
+    <Compile Include="LogFileGenerator.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="RedisTestGenerator.cs" />
+    <Compile Include="TcpTestGenerator.cs" />
+    <Compile Include="JsonLogFileGenerator.cs" />
+    <Compile Include="UdpTestGenerator.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="default.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results1.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test1.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test1-twconfig.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <None Include="packages.config" />
+    <Content Include="test2-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test3.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test3-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results3.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test4-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results4.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test4.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test5-twconfig.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test5.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results5.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test7-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test7.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results7.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <None Include="sample-apache.log">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </None>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">
+      <Project>{4ef96a08-21db-4178-be44-70dae594632c}</Project>
+      <Name>TimberWinR</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
--- a/TimberWinR.TestGenerator/UdpTestGenerator.cs
+++ b/TimberWinR.TestGenerator/UdpTestGenerator.cs
@@ -0,0 +1,93 @@
+using System.Security.Cryptography;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class UdpTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public UdpTestParameters()
+        {
+            NumMessages = 100;
+            Port = 6379;
+            Host = "localhost";
+            SleepTimeMilliseconds = 1;
+        }
+    }
+
+    class UdpTestGenerator
+    {
+        public static int Generate(UdpTestParameters parms)
+        {
+            var hostName = System.Environment.MachineName + "." +
+               Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                   "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+     
+            IPAddress broadcast;
+            if (!IPAddress.TryParse(parms.Host, out broadcast))
+                broadcast = Dns.GetHostEntry(parms.Host).AddressList[0];
+
+            Socket s = new Socket(broadcast.AddressFamily, SocketType.Dgram, ProtocolType.Udp);
+
+            LogManager.GetCurrentClassLogger().Info("Start UDP Generation");
+
+            for (int i = 0; i < parms.NumMessages; i++)
+            {
+                JObject o = new JObject
+                {
+                    {"Application", "udp-generator"},  
+                    {"Executable", "VP.Common.SvcFrm.Services.Host, Version=29.7.0.0, Culture=neutral, PublicKeyToken=null"},
+                    {"RenderedMessage", "Responding to RequestSchedule message from 10.1.230.36 with Ack because: PRJ byte array is null."},
+                    {"Team", "Manufacturing Software"},   
+                    {"RecordNumber", i},
+                    {"Host", hostName},                   
+                    {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                    {"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"},                
+                    {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
+                    {"Index", "logstash"}
+                };
+
+                string hashedString = "";
+                foreach(var key in o)
+                {
+                    hashedString += key.ToString();
+                }
+
+                var source = ASCIIEncoding.ASCII.GetBytes(hashedString);
+                var md5 = new MD5CryptoServiceProvider().ComputeHash(source);
+                var hash = string.Concat(md5.Select(x => x.ToString("X2")));
+
+                o["md5"] = hash;
+
+                byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None));
+                IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
+                s.SendTo(sendbuf, ep);
+
+                if (i % 1000 == 0)
+                    LogManager.GetCurrentClassLogger().Info("Sent {0} of {1} messages", i, parms.NumMessages);
+
+                Thread.Sleep(parms.SleepTimeMilliseconds);
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished UDP Generation");
+
+            return parms.NumMessages;
+        }
+
+    }
+}
--- a/TimberWinR.TestGenerator/default.json
+++ b/TimberWinR.TestGenerator/default.json
@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "TailFiles": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.mycompany.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/packages.config
+++ b/TimberWinR.TestGenerator/packages.config
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="CommandLineParser" version="1.9.71" targetFramework="net40" />
+  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net40" />
+  <package id="NLog" version="3.2.0.0" targetFramework="net40" />
+  <package id="ServiceStack.Common.Signed" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Interfaces" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Redis.Signed" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Text.Signed" version="4.0.38" targetFramework="net40" />
+</packages>
--- a/TimberWinR.TestGenerator/results1.json
+++ b/TimberWinR.TestGenerator/results1.json
@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 30"                
+                }
+            },
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 1234",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 30"              
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/results2.json
+++ b/TimberWinR.TestGenerator/results2.json
@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 20"
+                }
+            },
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 1234",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 20"
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/results3.json
+++ b/TimberWinR.TestGenerator/results3.json
@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            },
+            {
+                "tcp": {
+                    "test4: message sent count": "[messages] == 1234",
+                    "test5: average cpu": "[avgCpuUsage] <= 30",
+                    "test6: maximum memory": "[maxMemUsage] <= 15"
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/results4.json
+++ b/TimberWinR.TestGenerator/results4.json
@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            },
+            {
+                "tcp": {
+                    "test4: message sent count": "[messages] == 1234",
+                    "test5: average cpu": "[avgCpuUsage] <= 30",
+                    "test6: maximum memory": "[maxMemUsage] <= 15"
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/results5.json
+++ b/TimberWinR.TestGenerator/results5.json
@@ -0,0 +1,13 @@
+{
+    "Results": {
+        "Inputs": [           
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 80000",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 30"              
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/results7.json
+++ b/TimberWinR.TestGenerator/results7.json
@@ -0,0 +1,13 @@
+{
+    "Results": {
+        "Inputs": [           
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 80000",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 30"              
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/sample-apache.log
+++ b/TimberWinR.TestGenerator/sample-apache.log
--- a/TimberWinR.TestGenerator/test1-twconfig.json
+++ b/TimberWinR.TestGenerator/test1-twconfig.json
@@ -0,0 +1,50 @@
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                },
+                "json": {
+                    "type": "Win32-TailFile",
+                    "source": "Text",
+                    "promote": "Text"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads": 1,
+                    "host": [
+                        "tstlexiceapp006.mycompany.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test1.json
+++ b/TimberWinR.TestGenerator/test1.json
@@ -0,0 +1,16 @@
+{
+    "test": "Test 1",
+    "arguments": {
+        "--start":  "",
+        "--testFile": "test1.json",
+        "--testDir": "test1",
+        "--timberWinRConfig": "test1-twconfig.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--udp-host":  "localhost",
+        "--udp": "5140",       
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results1.json"
+    }
+}
--- a/TimberWinR.TestGenerator/test2-tw.json
+++ b/TimberWinR.TestGenerator/test2-tw.json
@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "Logs": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.mycompany.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test2.json
+++ b/TimberWinR.TestGenerator/test2.json
@@ -0,0 +1,15 @@
+{
+    "test": "Test 2",
+    "arguments": {
+        "--start":  "",
+        "--testFile": "test2.json",
+        "--testDir": "test2",
+        "--timberWinRConfig": "test2-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--udp": "5140",
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results2.json"
+    }
+}
--- a/TimberWinR.TestGenerator/test3-tw.json
+++ b/TimberWinR.TestGenerator/test3-tw.json
@@ -0,0 +1,50 @@
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "d:\\logs\\sta\\sta.log",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                },
+                "json": {
+                    "type": "Win32-TailFile",
+                    "source": "Text",
+                    "promote": "Text"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads": 2,
+                    "host": [
+                        "tstlexiceapp006.mycompany.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test3.json
+++ b/TimberWinR.TestGenerator/test3.json
@@ -0,0 +1,12 @@
+{
+    "test": "Test 3",
+    "arguments": {
+        "--start":  "",
+        "--testFile": "test3.json",
+        "--testDir": "test3",
+        "--timberWinRConfig": "test3-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",         
+        "--resultsFile": "results3.json"
+    }
+}
--- a/TimberWinR.TestGenerator/test4-tw.json
+++ b/TimberWinR.TestGenerator/test4-tw.json
@@ -0,0 +1,50 @@
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "d:\\logs\\sta\\sta.log",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                },
+                "json": {
+                    "type": "Win32-TailFile",
+                    "source": "Text",
+                    "promote": "Text"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads": 2,
+                    "host": [
+                        "tstlexiceapp006.mycompany.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test4.json
+++ b/TimberWinR.TestGenerator/test4.json
@@ -0,0 +1,11 @@
+{
+    "test": "Test 4",
+    "arguments": {
+        "--testFile": "test4.json",                
+        "--testDir": "test4",
+        "--timberWinRConfig": "test4-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",         
+        "--resultsFile": "results4.json"
+    }
+}
--- a/TimberWinR.TestGenerator/test5-twconfig.json
+++ b/TimberWinR.TestGenerator/test5-twconfig.json
@@ -0,0 +1,36 @@
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140,
+                    "add_field": [
+                        "Environment",
+                        "PLANT_TST_TIMBERWINR"
+                    ],
+                    "rename": [
+                        "Type",
+                        "type"
+                    ]
+                }
+            ],
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Outputs": {
+            "File": [
+                {
+                    "format":  "indented",
+                    "file_name": "test5_output.txt"
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test5.json
+++ b/TimberWinR.TestGenerator/test5.json
@@ -0,0 +1,15 @@
+{
+    "test": "Test 5",
+    "arguments": {
+        "--start":  "",
+        "--testFile": "test5.json",
+        "--testDir": "test5",
+        "--timberWinRConfig": "test5-twconfig.json",
+        "--numMessages": 80000,
+        "--logLevel": "debug",
+        "--udp-host":  "localhost",
+        "--udp": "5140",  
+        "--udp-rate":  1,            
+        "--resultsFile": "results5.json"
+    }
+}
--- a/TimberWinR.TestGenerator/test7-tw.json
+++ b/TimberWinR.TestGenerator/test7-tw.json
@@ -0,0 +1,37 @@
+{
+    "TimberWinR": {
+        "Inputs": {
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "apache log files",
+                    "location": "..\\sample-apache.log",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "type": "Win32-TailLog",
+                    "match": [
+                        "Text",
+                        "%{COMBINEDAPACHELOG}"
+                    ]
+                }
+            }
+        ],
+        "Outputs": {            
+            "StatsD": [
+                {
+                    "type": "Win32-TailLog",
+                    "namespace": "timberwinrtest",
+                    "port": 8125,
+                    "host": "devlexicesnu003.mycompany.svc",
+                    "increment":  ["apache.response.%{response}"],
+                    "count":  ["apache.bytes", "%{bytes}"]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test7.json
+++ b/TimberWinR.TestGenerator/test7.json
@@ -0,0 +1,13 @@
+{
+    "test": "Test 7",
+    "arguments": {
+        "--totalMessages":  2223,
+        "--start":  "",
+        "--testFile": "test7.json",
+        "--testDir": "test7",
+        "--timberWinRConfig": "test7-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",       
+        "--resultsFile": "results7.json"
+    }
+}
--- a/TimberWinR.UnitTests/Configuration.cs
+++ b/TimberWinR.UnitTests/Configuration.cs
@@ -85,7 +85,7 @@ namespace TimberWinR.UnitTests
                        [{  
                            ""host"": 
                            [
-                                ""logaggregator.vistaprint.svc""
+                                ""logaggregator.mycompany.svc""
                            ]                   
 		                }]
 	                }
@@ -94,7 +94,7 @@ namespace TimberWinR.UnitTests


            Configuration c = Configuration.FromString(redisJson);
-            RedisOutput redis = c.RedisOutputs.First() as RedisOutput;
+            RedisOutputParameters redis = c.RedisOutputs.First() as RedisOutputParameters;
            Assert.IsTrue(redis.Host.Length >= 1);
        }

--- a/TimberWinR.UnitTests/FakeRediServer.cs
+++ b/TimberWinR.UnitTests/FakeRediServer.cs
@@ -0,0 +1,126 @@
+using System;
+using System.IO;
+using System.Threading;
+using System.Net;
+using System.Net.Sockets;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NUnit.Framework;
+using TimberWinR.Parser;
+using System.Text;
+using System.Collections.Generic;
+
+namespace TimberWinR.UnitTests
+{   
+    // Class which implements a Fake redis server for test purposes.
+    class FakeRediServer
+    {
+        private readonly System.Net.Sockets.TcpListener _tcpListenerV4;
+        private readonly System.Net.Sockets.TcpListener _tcpListenerV6;
+        private Thread _listenThreadV4;
+        private Thread _listenThreadV6;
+        private readonly int _port;
+        private CancellationToken _cancelToken;
+        private bool _shutdown;     
+
+        public FakeRediServer(CancellationToken cancelToken, int port = 6379)
+        {
+            _port = port;
+            _cancelToken = cancelToken;
+            _shutdown = false;
+
+            _tcpListenerV6 = new System.Net.Sockets.TcpListener(IPAddress.IPv6Any, port);
+            _tcpListenerV4 = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
+
+            _listenThreadV4 = new Thread(new ParameterizedThreadStart(ListenForClients));
+            _listenThreadV4.Start(_tcpListenerV4);
+
+            _listenThreadV6 = new Thread(new ParameterizedThreadStart(ListenForClients));
+            _listenThreadV6.Start(_tcpListenerV6);
+        }
+
+        public void Shutdown()
+        {
+            _shutdown = true;
+            this._tcpListenerV4.Stop();
+            this._tcpListenerV6.Stop();
+        }
+
+
+        private void ListenForClients(object olistener)
+        {
+            System.Net.Sockets.TcpListener listener = olistener as System.Net.Sockets.TcpListener;
+
+            listener.Start();
+
+
+            while (!_cancelToken.IsCancellationRequested && !_shutdown)
+            {
+                try
+                {
+                    //blocks until a client has connected to the server
+                    TcpClient client = listener.AcceptTcpClient();
+
+                    // Wait for a client, spin up a thread.
+                    var clientThread = new Thread(new ParameterizedThreadStart(HandleNewClient));
+                    clientThread.Start(client);
+                }
+                catch (SocketException ex)
+                {
+                    if (ex.SocketErrorCode == SocketError.Interrupted)
+                        break;
+                }
+            }
+        }
+
+        private void HandleNewClient(object client)
+        {
+            var tcpClient = (TcpClient)client;
+
+            try
+            {
+                NetworkStream clientStream = tcpClient.GetStream();
+                int i;
+                Byte[] bytes = new Byte[16535];
+                String data = null;
+
+                do
+                {
+                    try
+                    {
+                        // Loop to receive all the data sent by the client.
+                        while ((i = clientStream.Read(bytes, 0, bytes.Length)) != 0)
+                        {
+                            // Translate data bytes to a ASCII string.
+                            data = System.Text.Encoding.ASCII.GetString(bytes, 0, i);
+                            //System.Diagnostics.Debug.WriteLine(String.Format("Received: {0}", data));
+
+                            // Process the data sent by the client.
+                            data = ":1000\r\n";
+
+                            byte[] msg = System.Text.Encoding.ASCII.GetBytes(data);
+
+                            // Send back a response.
+                            clientStream.Write(msg, 0, msg.Length);
+                          //  System.Diagnostics.Debug.WriteLine(String.Format("Sent: {0}", data));
+                        }
+                    }
+                    catch (IOException)
+                    {                       
+                    }
+                } while (true);
+            }
+            catch (Exception ex)
+            {
+                System.Diagnostics.Debug.WriteLine(ex.ToString());
+            }
+            tcpClient.Close();
+        }
+
+        private void ProcessJson(JObject json)
+        {
+            Console.WriteLine(json.ToString());
+        }
+
+    }
+}
--- a/TimberWinR.UnitTests/GrokFilterTests.cs
+++ b/TimberWinR.UnitTests/GrokFilterTests.cs
@@ -22,7 +22,7 @@ namespace TimberWinR.UnitTests
                {"Index", 7},
                {"Text", null},
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
            };

            string grokJson = @"{  
@@ -55,7 +55,7 @@ namespace TimberWinR.UnitTests
            Assert.IsTrue(grok.Apply(json));

            // Verify host field added
-            Assert.AreEqual(json["host"].ToString(), "dev.vistaprint.net");
+            Assert.AreEqual(json["host"].ToString(), "dev.mycompany.net");

            // Verify two tags added
            Assert.AreEqual(json["tags"][0].ToString(), "rn_7");
@@ -71,7 +71,7 @@ namespace TimberWinR.UnitTests
                {"Index", 7},
                {"Text", "crap"},
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
            };

            string grokJson = @"{  
@@ -123,7 +123,7 @@ namespace TimberWinR.UnitTests
                    }
                },
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
            };

            JObject json2 = new JObject
@@ -140,7 +140,7 @@ namespace TimberWinR.UnitTests
                    }
                },
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
            };


@@ -223,7 +223,8 @@ namespace TimberWinR.UnitTests
                    }
                },
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"Type", "Win32-MyType"},
+                {"ComputerName", "dev.mycompany.net"}
            };

            string grokJson1 = @"{  
@@ -281,11 +282,35 @@ namespace TimberWinR.UnitTests
                }
            }";

-            // Positive Tests
-            Configuration c = Configuration.FromString(grokJson1);
+            string grokJson4 = @"{  
+            ""TimberWinR"":{        
+                ""Filters"":[  
+	                {  
+		            ""grok"":{  
+		                ""condition"": ""!\""[Type]\"".StartsWith(\""[\"") && !\""[Type]\"".EndsWith(\""]\"") && (\""[type]\"" == \""Win32-FileLog\"")"",
+		                ""match"":[  
+			                ""Text"",
+			                """"
+		                ],
+                        ""remove_tag"":[  
+                            ""tag1""                          
+                        ]		               
+		            }
+	                }]
+                }
+            }";
+
+            
+            Configuration c = Configuration.FromString(grokJson4);
            Grok grok = c.Filters.First() as Grok;
            Assert.IsTrue(grok.Apply(json));

+
+            // Positive Tests
+            c = Configuration.FromString(grokJson1);
+            grok = c.Filters.First() as Grok;
+            Assert.IsTrue(grok.Apply(json));
+
            c = Configuration.FromString(grokJson2);
            grok = c.Filters.First() as Grok;
            Assert.IsTrue(grok.Apply(json));
@@ -311,7 +336,7 @@ namespace TimberWinR.UnitTests
                    }
                },
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
            };

            string grokJson = @"{  
--- a/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs
+++ b/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs
@@ -1,60 +0,0 @@
-namespace TimberWinR.UnitTests.Inputs
-{
-    using System;
-    using System.Collections.Generic;
-
-    using Interop.MSUtil;
-
-    using Moq;
-
-    using NUnit.Framework;
-
-    using TimberWinR.Inputs;
-    using TimberWinR.Parser;
-
-    [TestFixture]
-    public class IisW3CRowReaderTests : TestBase
-    {
-        private IisW3CRowReader reader;
-
-        public override void Setup()
-        {
-            base.Setup();
-            var fields = new List<Field>
-                             {
-                                 new Field("date", "DateTime"),
-                                 new Field("time", "DateTime"),
-                                 new Field("uri")
-                             };
-            this.reader = new IisW3CRowReader(fields);
-
-            var recordset = this.GetRecordsetMock();
-            this.reader.ReadColumnMap(recordset.Object);
-        }
-
-        [Test]
-        public void GivenValidRowAddsTimestampColumn()
-        {
-            var record = this.MockRepository.Create<ILogRecord>();
-            record.Setup(x => x.getValue("date")).Returns(new DateTime(2014, 11, 30));
-            record.Setup(x => x.getValue("time")).Returns(new DateTime(1, 1, 1, 18, 45, 37, 590));
-            record.Setup(x => x.getValue("uri")).Returns("http://somedomain.com/someurl");
-
-            var json = this.reader.ReadToJson(record.Object);
-
-            Assert.AreEqual("2014-11-30T18:45:37.000Z", json["@timestamp"].ToString());
-            Assert.AreEqual("http://somedomain.com/someurl", json["uri"].ToString());
-        }
-
-        private Mock<ILogRecordset> GetRecordsetMock()
-        {
-            var recordset = this.MockRepository.Create<ILogRecordset>();
-            recordset.Setup(x => x.getColumnCount()).Returns(3);
-
-            recordset.Setup(x => x.getColumnName(0)).Returns("date");
-            recordset.Setup(x => x.getColumnName(1)).Returns("time");
-            recordset.Setup(x => x.getColumnName(2)).Returns("uri");
-            return recordset;
-        }
-    }
-}
--- a/TimberWinR.UnitTests/JsonFilterTests.cs
+++ b/TimberWinR.UnitTests/JsonFilterTests.cs
@@ -20,14 +20,14 @@ namespace TimberWinR.UnitTests
            JObject jsonInputLine1 = new JObject
            {               
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"},
+                {"ComputerName", "dev.mycompany.net"},
                {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
            };

            JObject jsonInputLine2 = new JObject
            {               
                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"},
+                {"ComputerName", "dev.mycompany.net"},
                {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
            };

@@ -77,5 +77,56 @@ namespace TimberWinR.UnitTests
            Assert.IsNull(nostuff);
            Assert.AreEqual(6, jsonInputLine2.Count);            
        }
+
+        [Test]
+        public void TestRenameAndAdds()
+        {
+            JObject jsonInputLine1 = new JObject
+            {               
+                {"type", "Win32-FileLog"},
+                {"ComputerName", "dev.mycompany.net"},
+                {"Text", "{\"log4net:Username\" : \"NT AUTHORITY\",\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
+            };
+
+
+            string jsonFilter = @"{  
+            ""TimberWinR"":{        
+                ""Filters"":[  
+	                {  
+		            ""json"":{  
+		                ""type"": ""Win32-FileLog"",		               
+                        ""source"": ""Text"",
+                        ""promote"": ""Text"",
+                        ""add_field"":[  
+                            ""test1"",
+                            ""value1"",
+                            ""test2"",
+                            ""value2""
+                        ],       
+                        ""rename"":[  
+                            ""Active"",
+                            ""active"",
+                            ""log4net:Username"",
+                            ""lusername""
+                        ]          
+		            }
+	              }]
+                }
+            }";
+            
+
+            // Positive Tests
+            Configuration c = Configuration.FromString(jsonFilter);
+            Json jf = c.Filters.First() as Json;
+            Assert.IsTrue(jf.Apply(jsonInputLine1));
+
+            Assert.AreEqual("NT AUTHORITY", jsonInputLine1["lusername"].ToString());
+            Assert.AreEqual("True", jsonInputLine1["active"].ToString());
+            Assert.IsNotNull(jsonInputLine1["test1"]);
+            Assert.IsNotNull(jsonInputLine1["test2"]);
+            Assert.AreEqual("value1", jsonInputLine1["test1"].ToString());
+            Assert.AreEqual("value2", jsonInputLine1["test2"].ToString());
+        }
+ 
    }
 }
--- a/TimberWinR.UnitTests/MultilineTests.cs
+++ b/TimberWinR.UnitTests/MultilineTests.cs
@@ -3,7 +3,6 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
-using System.Linq;
 using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json;
@@ -29,10 +28,10 @@ namespace TimberWinR.UnitTests

                Stdin sin = new Stdin();

-                sin.Codec = new Codec();
-                sin.Codec.Pattern = "\\\\$";
-                sin.Codec.What = Codec.WhatType.next;
-                sin.Codec.Type = Codec.CodecType.multiline;
+                sin.CodecArguments = new CodecArguments();
+                sin.CodecArguments.Pattern = "\\\\$";
+                sin.CodecArguments.What = CodecArguments.WhatType.next;
+                sin.CodecArguments.Type = CodecArguments.CodecType.multiline;

                var cancelTokenSource = new CancellationTokenSource();

@@ -52,7 +51,7 @@ namespace TimberWinR.UnitTests
                        if (!cancelTokenSource.Token.IsCancellationRequested)
                            syncHandle.Wait(TimeSpan.FromSeconds(10000), cancelTokenSource.Token);
                    }
-                    catch (OperationCanceledException oex)
+                    catch (OperationCanceledException)
                    {
                    }
                }
@@ -78,11 +77,11 @@ namespace TimberWinR.UnitTests

                Stdin sin = new Stdin();

-                sin.Codec = new Codec();
-                sin.Codec.Pattern = "^(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3})(.*)$";              
-                sin.Codec.What = Codec.WhatType.previous;
-                sin.Codec.Type = Codec.CodecType.multiline;
-                sin.Codec.Negate = true;
+                sin.CodecArguments = new CodecArguments();
+                sin.CodecArguments.Pattern = "^(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3})(.*)$";              
+                sin.CodecArguments.What = CodecArguments.WhatType.previous;
+                sin.CodecArguments.Type = CodecArguments.CodecType.multiline;
+                sin.CodecArguments.Negate = true;

                var cancelTokenSource = new CancellationTokenSource();

@@ -102,7 +101,7 @@ namespace TimberWinR.UnitTests
                        if (!cancelTokenSource.Token.IsCancellationRequested)
                            syncHandle.Wait(TimeSpan.FromSeconds(10000), cancelTokenSource.Token);
                    }
-                    catch (OperationCanceledException oex)
+                    catch (OperationCanceledException)
                    {
                    }
                }
--- a/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs
+++ b/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs
@@ -1,4 +1,6 @@
-namespace TimberWinR.UnitTests.Parser
+using TimberWinR.Outputs;
+
+namespace TimberWinR.UnitTests.Parser
 {
    using System;

@@ -10,12 +12,12 @@

    public class ElasticsearchOutputTests
    {
-        private ElasticsearchOutput parser;
+        private ElasticsearchOutputParameters parser;

        [SetUp]
        public void Setup()
        {
-            this.parser = new ElasticsearchOutput();
+            this.parser = new ElasticsearchOutputParameters();
        }

        [Test]
@@ -51,5 +53,51 @@

            Assert.AreEqual("someindex-" + DateTime.UtcNow.ToString("yyyy.MM.dd"), result);
        }
+
+        [Test]
+        public void Given_no_ssl_then_validate_does_not_throw()
+        {
+            parser.Ssl = false;
+            Assert.That(() => parser.Validate(), Throws.Nothing);
+        }
+
+        [Test]
+        public void Given_ssl_and_no_username_then_validate_throws()
+        {
+            parser.Ssl = true;
+            parser.Password = "pass";
+
+            Assert.That(() => parser.Validate(), Throws.Exception.InstanceOf<ElasticsearchOutputParameters.ElasticsearchBasicAuthException>());
+        }
+
+        [Test]
+        public void Given_ssl_and_no_password_then_validate_throws()
+        {
+            parser.Ssl = true;
+            parser.Username = "user";
+
+            Assert.That(() => parser.Validate(), Throws.Exception.InstanceOf<ElasticsearchOutputParameters.ElasticsearchBasicAuthException>());
+        }
+
+        [Test]
+        public void Given_ssl_and_username_and_password_then_validate_does_not_throw()
+        {
+            parser.Ssl = true;
+            parser.Username = "user";
+            parser.Password = "pass";
+
+            Assert.That(() => parser.Validate(), Throws.Nothing);
+        }
+
+        [Test]
+        [TestCase("host", 1234, false, null, null, "http://host:1234/")]
+        [TestCase("host", 1234, true, "user", "pass", "https://user:pass@host:1234/")]
+        [TestCase("host", 1234, true, "user:", "pass@", "https://user%3A:pass%40@host:1234/")]
+        public void ComposeUri_Matches_Expected(string host, int port, bool ssl, string username, string password, string expectedUri)
+        {
+            var uri = ElasticsearchOutput.ComposeUri(host, port, ssl, username, password);
+
+            Assert.That(uri.ToString(), Is.EqualTo(expectedUri));
+        }
    }
 }
--- a/TimberWinR.UnitTests/TailFileTests.cs
+++ b/TimberWinR.UnitTests/TailFileTests.cs
@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TimberWinR.Inputs;
+using TimberWinR.Parser;
+using Newtonsoft.Json.Linq;
+using System.Diagnostics;
+
+namespace TimberWinR.UnitTests
+{
+    [TestFixture]
+    public class TailFileTests
+    {
+        [Test]
+        public void TestTailFile()
+        {
+            List<JObject> events = new List<JObject>();
+
+            if (File.Exists(".timberwinrdb"))
+                File.Delete(".timberwinrdb");
+
+            var mgr = new Manager();
+            mgr.LogfileDir = ".";
+
+            var tf = new TailFileArguments();
+            var cancelTokenSource = new CancellationTokenSource();
+            tf.Location = "TestTailFile1.log";
+
+   
+            if (File.Exists(tf.Location))
+                File.Delete(tf.Location);
+
+            try
+            {
+                var listener = new TailFileListener(tf, cancelTokenSource.Token);
+
+                listener.OnMessageRecieved += o =>
+                {
+                    events.Add(o);
+                    if (events.Count >= 100)
+                        cancelTokenSource.Cancel();
+                };
+
+                GenerateLogFile(tf.Location);
+
+                bool createdFile = false;
+                while (!listener.Stop && !cancelTokenSource.IsCancellationRequested)
+                {
+                    Thread.Sleep(100);
+                    if (!createdFile)
+                    {
+                        GenerateLogFile(tf.Location);
+                        createdFile = true;
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                Console.WriteLine("Done!");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.ToString());
+
+            }
+            finally
+            {
+                Assert.AreEqual(100, events.Count);
+            }
+        }
+
+        private static void GenerateLogFile(string fileName)
+        {
+            using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
+            {
+                for (int i = 0; i < 100; i++)
+                {
+                    file.WriteLine("Log Line Number {0}", i);
+                }
+            }
+        }
+    }
+}
--- a/TimberWinR.UnitTests/TestBase.cs
+++ b/TimberWinR.UnitTests/TestBase.cs
@@ -1,23 +0,0 @@
-namespace TimberWinR.UnitTests
-{
-    using Moq;
-
-    using NUnit.Framework;
-
-    public class TestBase
-    {
-        public MockRepository MockRepository { get; private set; }
-
-        [SetUp]
-        public virtual void Setup()
-        {
-            this.MockRepository = new MockRepository(MockBehavior.Default);
-        }
-
-        [TearDown]
-        public virtual void TearDown()
-        {
-            this.MockRepository.VerifyAll();
-        }
-    }
-}
--- a/TimberWinR.UnitTests/TestDynamicBatchCount.cs
+++ b/TimberWinR.UnitTests/TestDynamicBatchCount.cs
@@ -0,0 +1,89 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TimberWinR.Parser;
+using Newtonsoft.Json.Linq;
+using System.Threading;
+
+
+namespace TimberWinR.UnitTests
+{
+    [TestFixture]
+    public class TestDynamicBatchCount
+    {
+   //     [Test]
+        public void TestDynamicBatch()
+        {
+            var mgr = new Manager();
+            mgr.LogfileDir = ".";
+
+            mgr.Config = new Configuration();
+
+            CancellationTokenSource cancelTokenSource = new CancellationTokenSource();
+
+            var cancelToken = cancelTokenSource.Token;
+
+            FakeRediServer fr = new FakeRediServer(cancelToken);
+
+            var redisParams = new RedisOutputParameters();
+            redisParams.BatchCount = 10;
+            redisParams.MaxBatchCount = 40;
+            redisParams.Interval = 100;
+
+            var redisOutput = new Outputs.RedisOutput(mgr, redisParams, cancelToken);
+
+
+            // Message is irrelavant
+            JObject jsonMessage = new JObject
+            {               
+                {"type", "Win32-FileLog"},
+                {"ComputerName", "dev.vistaprint.net"},
+                {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
+            };
+            
+                     
+            // Send 1000 messages at max throttle
+            for (int i = 0; i < 1000; i++)
+            {
+                Thread.Sleep(10);
+                redisOutput.Startup(jsonMessage);             
+            }
+
+            while (redisOutput.SentMessages < 1000)
+            {
+                System.Diagnostics.Debug.WriteLine(redisOutput.SentMessages);
+                Thread.Sleep(1000);
+            }
+
+            fr.Shutdown();
+
+            cancelTokenSource.Cancel();
+
+            System.Diagnostics.Debug.WriteLine(redisOutput.ToJson());
+            System.Diagnostics.Debug.WriteLine(redisOutput.QueueDepth);
+
+            JObject json = redisOutput.ToJson();
+            var mbc = json["redis"]["reachedMaxBatchCountTimes"].Value<int>();
+            var sm = json["redis"]["sentMessageCount"].Value<int>();
+            var errs = json["redis"]["errors"].Value<int>();
+            var cbc = json["redis"]["currentBatchCount"].Value<int>();
+            
+            // No errors
+            Assert.AreEqual(0, errs);          
+              
+            // Should have reached max at least 1 time
+            Assert.GreaterOrEqual(mbc, 1);
+
+            // Should have sent 1000 messages
+            Assert.AreEqual(1000, sm);
+
+            // Should reset back down to original
+            Assert.AreEqual(cbc, 10);
+        }
+    }
+}
--- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
+++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
@@ -12,6 +12,8 @@
    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <DebugSymbols>true</DebugSymbols>
@@ -38,16 +40,13 @@
      <EmbedInteropTypes>False</EmbedInteropTypes>
      <HintPath>..\TimberWinR\lib\com-logparser\Interop.MSUtil.dll</HintPath>
    </Reference>
-    <Reference Include="Moq">
-      <HintPath>..\packages\Moq.4.2.1409.1722\lib\net40\Moq.dll</HintPath>
-    </Reference>
    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <Private>True</Private>
    </Reference>
-    <Reference Include="nunit.framework, Version=2.6.3.13283, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\NUnit.2.6.3\lib\nunit.framework.dll</HintPath>
+    <Reference Include="nunit.framework">
+      <HintPath>..\packages\NUnit.2.6.4\lib\nunit.framework.dll</HintPath>
    </Reference>
    <Reference Include="System" />
    <Reference Include="System.Core" />
@@ -60,14 +59,15 @@
  <ItemGroup>
    <Compile Include="Configuration.cs" />
    <Compile Include="DateFilterTests.cs" />
+    <Compile Include="FakeRediServer.cs" />
    <Compile Include="GeoIPFilterTests.cs" />
-    <Compile Include="Inputs\IisW3CRowReaderTests.cs" />
    <Compile Include="JsonFilterTests.cs" />
    <Compile Include="GrokFilterTests.cs" />
    <Compile Include="MultilineTests.cs" />
    <Compile Include="Parser\ElasticsearchOutputTests.cs" />
    <Compile Include="Properties\AssemblyInfo.cs" />
-    <Compile Include="TestBase.cs" />
+    <Compile Include="TailFileTests.cs" />
+    <Compile Include="TestDynamicBatchCount.cs" />
  </ItemGroup>
  <ItemGroup>
    <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">
@@ -79,6 +79,7 @@
    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
  </ItemGroup>
  <ItemGroup>
+    <None Include="app.config" />
    <None Include="packages.config">
      <SubType>Designer</SubType>
    </None>
@@ -91,7 +92,17 @@
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
    </Content>
  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Inputs\" />
+  </ItemGroup>
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
--- a/TimberWinR.UnitTests/app.config
+++ b/TimberWinR.UnitTests/app.config
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.5.0.0" newVersion="4.5.0.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>
--- a/TimberWinR.UnitTests/packages.config
+++ b/TimberWinR.UnitTests/packages.config
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages> 
-  <package id="Moq" version="4.2.1409.1722" targetFramework="net40" />
-  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
-  <package id="NUnit" version="2.6.3" targetFramework="net40" />
+  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net40" />
+  <package id="NUnit" version="2.6.4" targetFramework="net40" />
 </packages>
--- a/TimberWinR.sln
+++ b/TimberWinR.sln
@@ -7,6 +7,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR", "TimberWinR\Ti
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{2C4AC7DB-018D-4CCA-9579-06AC5AD2C9D9}"
 	ProjectSection(SolutionItems) = preProject
+		.nuget\NuGet.Config = .nuget\NuGet.Config
+		.nuget\NuGet.exe = .nuget\NuGet.exe
+		.nuget\NuGet.targets = .nuget\NuGet.targets
 		.nuget\packages.config = .nuget\packages.config
 	EndProjectSection
 EndProject
@@ -21,6 +24,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		chocolateyInstall.ps1.template = chocolateyInstall.ps1.template
 		chocolateyUninstall.ps1.guid = chocolateyUninstall.ps1.guid
 		chocolateyUninstall.ps1.template = chocolateyUninstall.ps1.template
+		chocolateyUninstall.ps1.template.orig = chocolateyUninstall.ps1.template.orig
 		LICENSE.txt = LICENSE.txt
 		Performance1.psess = Performance1.psess
 		README.md = README.md
@@ -34,6 +38,8 @@ Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "TimberWinR.Wix", "TimberWix
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR.ExtractID", "TimberWinR.ExtractID\TimberWinR.ExtractID.csproj", "{99096939-E9DD-4499-883D-4726745A5843}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR.TestGenerator", "TimberWinR.TestGenerator\TimberWinR.TestGenerator.csproj", "{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -96,6 +102,16 @@ Global
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|x86.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|x86.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/TimberWinR/Codecs/JsonCodec.cs
+++ b/TimberWinR/Codecs/JsonCodec.cs
@@ -0,0 +1,26 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+using TimberWinR.Parser;
+
+namespace TimberWinR.Codecs
+{
+    class JsonCodec : ICodec
+    {
+        private CodecArguments _codecArguments;
+
+        public void Apply(string msg, Inputs.InputListener listener)
+        {
+            JObject jobject = JObject.Parse(msg);
+            listener.AddDefaultFields(jobject);
+            listener.ProcessJson(jobject);
+        }
+
+        public JsonCodec(CodecArguments args)
+        {
+            _codecArguments = args;
+        }
+    }
+}
--- a/TimberWinR/Codecs/Multiline.cs
+++ b/TimberWinR/Codecs/Multiline.cs
@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Text.RegularExpressions;
+using Newtonsoft.Json.Linq;
+using TimberWinR.Inputs;
+using TimberWinR.Parser;
+
+namespace TimberWinR.Codecs
+{
+    public class Multiline : ICodec
+    {
+        private CodecArguments _codecArguments;
+        private List<string> _multiline { get; set; }
+
+        // return true to cancel codec
+        public Multiline(CodecArguments args)
+        {
+            _codecArguments = args;
+        }
+
+        public void Apply(string msg, InputListener listener)
+        {
+            if (_codecArguments.Re == null)
+                _codecArguments.Re = new Regex(_codecArguments.Pattern);
+
+            Match match = _codecArguments.Re.Match(msg);
+
+            bool isMatch = (match.Success && !_codecArguments.Negate) || (!match.Success && _codecArguments.Negate);
+
+            switch (_codecArguments.What)
+            {
+                case CodecArguments.WhatType.previous:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+
+                        _multiline.Add(msg);
+                    }
+                    else // No Match
+                    {
+                        if (_multiline != null)
+                        {
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codecArguments.MultilineTag));
+                            listener.AddDefaultFields(jo);
+                            listener.ProcessJson(jo);
+                        }
+                        _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    break;
+                case CodecArguments.WhatType.next:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    else // No match
+                    {
+                        if (_multiline != null)
+                        {
+                            _multiline.Add(msg);
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codecArguments.MultilineTag));
+                            listener.AddDefaultFields(jo);
+                            listener.ProcessJson(jo);
+                        }
+                        else
+                        {
+                            JObject jo = new JObject();
+                            jo["message"] = msg;
+                            listener.AddDefaultFields(jo);
+                            listener.ProcessJson(jo);
+                        }
+                    }
+                    break;
+            }
+        }
+    }
+}
--- a/TimberWinR/Codecs/PlainCodec.cs
+++ b/TimberWinR/Codecs/PlainCodec.cs
@@ -0,0 +1,39 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+using TimberWinR.Parser;
+
+namespace TimberWinR.Codecs
+{
+    public class PlainCodec : ICodec
+    {
+        private CodecArguments _codecArguments;
+
+        public void Apply(string msg, Inputs.InputListener listener)
+        {
+            JObject json = new JObject();
+            listener.AddDefaultFields(json);
+            json["message"] = ExpandField(msg, json);
+            listener.ProcessJson(json);
+        }
+
+        protected string ExpandField(string fieldName, JObject json)
+        {
+            foreach (var token in json.Children())
+            {
+                string replaceString = "%{" + token.Path + "}";
+                fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
+            }
+            return fieldName;
+        }
+
+
+        public PlainCodec(CodecArguments args)
+        {
+            _codecArguments = args;
+        }
+
+    }
+}
--- a/TimberWinR/Configuration.cs
+++ b/TimberWinR/Configuration.cs
@@ -4,6 +4,8 @@ using System.Data.Odbc;
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
 using System.Text;
 using System.Xml;
 using System.Xml.Linq;
@@ -16,69 +18,92 @@ using Newtonsoft.Json.Linq;
 using TimberWinR.Inputs;
 using TimberWinR.Filters;

+
 using NLog;
 using TimberWinR.Parser;
 using Topshelf.Configurators;
-using IISW3CLog = TimberWinR.Parser.IISW3CLog;
 using WindowsEvent = TimberWinR.Parser.WindowsEvent;

 namespace TimberWinR
 {
    public class Configuration
    {       
+        private CancellationToken _cancelToken;
+        
+        private FileSystemWatcher _dirWatcher;
+        private Manager _manager;
+ 
        private List<WindowsEvent> _events = new List<WindowsEvent>();
        public IEnumerable<WindowsEvent> Events
        {
            get { return _events; }
        }

-        private List<RedisOutput> _redisOutputs = new List<RedisOutput>();
-        public IEnumerable<RedisOutput> RedisOutputs
+        private List<StatsDOutputParameters> _statsdOutputs = new List<StatsDOutputParameters>();
+        public IEnumerable<StatsDOutputParameters> StatsDOutputs
+        {
+            get { return _statsdOutputs; }
+        }
+
+        private List<RedisOutputParameters> _redisOutputs = new List<RedisOutputParameters>();
+        public IEnumerable<RedisOutputParameters> RedisOutputs
        {
            get { return _redisOutputs; }
        }
       

-        private List<ElasticsearchOutput> _elasticsearchOutputs = new List<ElasticsearchOutput>();
-        public IEnumerable<ElasticsearchOutput> ElasticsearchOutputs
+        private List<ElasticsearchOutputParameters> _elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
+        public IEnumerable<ElasticsearchOutputParameters> ElasticsearchOutputs
        {
            get { return _elasticsearchOutputs; }
        }

-        private List<StdoutOutput> _stdoutOutputs = new List<StdoutOutput>();
-        public IEnumerable<StdoutOutput> StdoutOutputs
+        private List<StdoutOutputParameters> _stdoutOutputs = new List<StdoutOutputParameters>();
+        public IEnumerable<StdoutOutputParameters> StdoutOutputs
        {
            get { return _stdoutOutputs; }
        }

-        private List<Tcp> _tcps = new List<Tcp>();
-        public IEnumerable<Tcp> Tcps
+        private List<FileOutputParameters> _fileOutputs = new List<FileOutputParameters>();
+        public IEnumerable<FileOutputParameters> FileOutputs
+        {
+            get { return _fileOutputs; }
+        }
+
+        private List<TcpParameters> _tcps = new List<TcpParameters>();
+        public IEnumerable<TcpParameters> Tcps
        {
            get { return _tcps; }
        }

-        private List<Udp> _udps = new List<Udp>();
-        public IEnumerable<Udp> Udps
+        private List<UdpParameters> _udps = new List<UdpParameters>();
+        public IEnumerable<UdpParameters> Udps
        {
            get { return _udps; }
        }     

-        private List<Log> _logs = new List<Log>();
-        public IEnumerable<Log> Logs
+        private List<LogParameters> _logs = new List<LogParameters>();
+        public IEnumerable<LogParameters> Logs
        {
            get { return _logs; }
        }

-        private List<IISW3CLog> _iisw3clogs = new List<IISW3CLog>();
+        private List<TailFileArguments> _tails = new List<TailFileArguments>();
+        public IEnumerable<TailFileArguments> TailFiles
+        {
+            get { return _tails; }
+        }    

-        public IEnumerable<IISW3CLog> IISW3C
+        private List<IISW3CLogParameters> _iisw3clogs = new List<IISW3CLogParameters>();
+
+        public IEnumerable<IISW3CLogParameters> IISW3C
        {
            get { return _iisw3clogs; }
        }

-        private List<W3CLog> _w3clogs = new List<W3CLog>();
+        private List<W3CLogParameters> _w3clogs = new List<W3CLogParameters>();

-        public IEnumerable<W3CLog> W3C
+        public IEnumerable<W3CLogParameters> W3C
        {
            get { return _w3clogs; }
        }
@@ -89,6 +114,12 @@ namespace TimberWinR
            get { return _stdins; }
        }

+        private List<GeneratorParameters> _generators = new List<GeneratorParameters>();
+        public IEnumerable<GeneratorParameters> Generators
+        {
+            get { return _generators; }
+        }
+
        private List<LogstashFilter> _filters = new List<LogstashFilter>();

        public IEnumerable<LogstashFilter> Filters
@@ -96,8 +127,84 @@ namespace TimberWinR
            get { return _filters; }
        }

+        private void MonitorDirectory(string directoryToWatch, CancellationToken cancelToken, Manager manager)
+        {
+            _manager = manager;
+            _cancelToken = cancelToken;
+            if (_dirWatcher == null)
+            {
+                _dirWatcher = new FileSystemWatcher();
+                _dirWatcher.Path = directoryToWatch;
+                _dirWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName;
+                // Only watch json files.
+                _dirWatcher.Filter = "*.json";
+                _dirWatcher.Created += DirWatcherOnCreated;
+                _dirWatcher.Changed += DirWatcherOnChanged;
+                _dirWatcher.Renamed += DirWatcherOnRenamed;
+                _dirWatcher.EnableRaisingEvents = true;
+            }       
+        }

-        public static Configuration FromDirectory(string jsonDirectory)
+        private void DirWatcherOnRenamed(object sender, RenamedEventArgs e)
+        {
+            // The Renamed file could be a different name from .json
+            FileInfo fi = new FileInfo(e.FullPath);
+            if (fi.Extension == ".json")
+            {
+                LogManager.GetCurrentClassLogger().Info("File: OnRenamed " + e.FullPath + " " + e.ChangeType);
+                ProcessNewJson(e.FullPath);
+            }
+        }
+
+        private void DirWatcherOnCreated(object sender, FileSystemEventArgs e)
+        {
+            FileInfo fi = new FileInfo(e.FullPath);
+            if (fi.Extension == ".json")
+            {
+                LogManager.GetCurrentClassLogger().Info("File: OnCreated " + e.FullPath + " " + e.ChangeType);
+                ProcessNewJson(e.FullPath);
+            }
+        }
+
+        private void DirWatcherOnChanged(object sender, FileSystemEventArgs e)
+        {
+            FileInfo fi = new FileInfo(e.FullPath);
+            if (fi.Extension == ".json")
+            {
+                // Specify what is done when a file is changed, created, or deleted.
+                LogManager.GetCurrentClassLogger()
+                    .Info("File: OnChanged " + e.ChangeType.ToString() + " " + e.FullPath + " " + e.ChangeType);
+                ProcessNewJson(e.FullPath);
+            }
+        }
+      
+        private void ProcessNewJson(string fileName)
+        {
+            try
+            {
+                Configuration c = new Configuration();
+                var config = Configuration.FromFile(fileName, c);
+                _manager.ProcessConfiguration(_cancelToken, config);
+
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);                         
+            }
+        }
+
+        private void ShutdownDirectoryMonitor()
+        {           
+            _dirWatcher.EnableRaisingEvents = false;
+            LogManager.GetCurrentClassLogger().Info("Stopping Directory Monitor");            
+        }
+
+        private void DirectoryWatcher(string directoryToWatch)
+        {
+            LogManager.GetCurrentClassLogger().Info("Starting Directory Monitor {0}", directoryToWatch);                       
+        }
+
+        public static Configuration FromDirectory(string jsonDirectory, CancellationToken cancelToken, Manager manager)
        {
            Configuration c = null;
      
@@ -109,6 +216,10 @@ namespace TimberWinR
                }
            }

+            // Startup Directory Monitor
+            if (manager.LiveMonitor)           
+                c.MonitorDirectory(jsonDirectory, cancelToken, manager);            
+
            return c;
        }

@@ -146,8 +257,12 @@ namespace TimberWinR
                    c._iisw3clogs.AddRange(x.TimberWinR.Inputs.IISW3CLogs.ToList());
                if (x.TimberWinR.Inputs.Stdins != null)
                    c._stdins.AddRange(x.TimberWinR.Inputs.Stdins.ToList());
+                if (x.TimberWinR.Inputs.Generators != null)
+                    c._generators.AddRange(x.TimberWinR.Inputs.Generators.ToList());
                if (x.TimberWinR.Inputs.Logs != null)
                    c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList());
+                if (x.TimberWinR.Inputs.TailFilesArguments != null)
+                    c._tails.AddRange(x.TimberWinR.Inputs.TailFilesArguments.ToList());
                if (x.TimberWinR.Inputs.Tcps != null)
                    c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList());
                if (x.TimberWinR.Inputs.Udps != null)
@@ -156,12 +271,16 @@ namespace TimberWinR

            if (x.TimberWinR.Outputs != null)
            {
+                if (x.TimberWinR.Outputs.StatsD != null)
+                    c._statsdOutputs.AddRange(x.TimberWinR.Outputs.StatsD.ToList());
                if (x.TimberWinR.Outputs.Redis != null)
                    c._redisOutputs.AddRange(x.TimberWinR.Outputs.Redis.ToList());
                if (x.TimberWinR.Outputs.Elasticsearch != null)
                    c._elasticsearchOutputs.AddRange(x.TimberWinR.Outputs.Elasticsearch.ToList());
                if (x.TimberWinR.Outputs.Stdout != null)
                    c._stdoutOutputs.AddRange(x.TimberWinR.Outputs.Stdout.ToList());
+                if (x.TimberWinR.Outputs.File != null)
+                    c._fileOutputs.AddRange(x.TimberWinR.Outputs.File.ToList());
            }

            if (x.TimberWinR.Filters != null)
@@ -194,13 +313,15 @@ namespace TimberWinR
        {
            _filters = new List<LogstashFilter>();
            _events = new List<WindowsEvent>();
-            _iisw3clogs = new List<IISW3CLog>();
-            _logs = new List<Log>();
-            _redisOutputs = new List<RedisOutput>();
-            _elasticsearchOutputs = new List<ElasticsearchOutput>();
-            _stdoutOutputs = new List<StdoutOutput>();
-            _tcps = new List<Tcp>();
-            _udps = new List<Udp>();
+            _iisw3clogs = new List<IISW3CLogParameters>();
+            _logs = new List<LogParameters>();
+            _statsdOutputs = new List<StatsDOutputParameters>();
+            _redisOutputs = new List<RedisOutputParameters>();
+            _elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
+            _stdoutOutputs = new List<StdoutOutputParameters>();
+            _fileOutputs = new List<FileOutputParameters>();
+            _tcps = new List<TcpParameters>();
+            _udps = new List<UdpParameters>();
        }

        public static Object GetPropValue(String name, Object obj)
--- a/TimberWinR/Diagnostics/Diagnostics.cs
+++ b/TimberWinR/Diagnostics/Diagnostics.cs
@@ -21,7 +21,7 @@ namespace TimberWinR.Diagnostics
        private CancellationToken CancelToken { get; set; }
        public int Port { get; set; }
        public Manager Manager { get; set; }
-
+        public bool Stop { get; set; }
        private HttpListener web;

        public Diagnostics(Manager manager, CancellationToken cancelToken, int port = 5141)
@@ -49,18 +49,12 @@ namespace TimberWinR.Diagnostics
        }


-        private void DiagnosticCallback(IAsyncResult result)
+        public JObject DiagnosticsOutput()
        {
-            if (web == null)         
-                return;
-
-            var context = web.EndGetContext(result);
-            var response = context.Response;
-
            JObject json = new JObject(
                           new JProperty("timberwinr",
                               new JObject(
-                        new JProperty("version", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+                                   new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
                                   new JProperty("messages", Manager.NumMessages),
                                   new JProperty("startedon", Manager.StartedOn),
                                   new JProperty("configfile", Manager.JsonConfig),
@@ -78,15 +72,37 @@ namespace TimberWinR.Diagnostics
                                       new JArray(
                                           from o in Manager.Outputs
                                           select new JObject(o.ToJson()))))));
+            return json;
+        }

-            response.StatusCode = (int)HttpStatusCode.OK;
+        private void DiagnosticCallback(IAsyncResult result)
+        {
+            if (web == null)
+                return;
+
+            try
+            {
+                var context = web.EndGetContext(result);
+                var response = context.Response;
+                var json = DiagnosticsOutput();
+
+                response.StatusCode = (int) HttpStatusCode.OK;
                response.StatusDescription = HttpStatusCode.OK.ToString();
                byte[] buffer = Encoding.UTF8.GetBytes(json.ToString());
                response.ContentLength64 = buffer.Length;
                response.OutputStream.Write(buffer, 0, buffer.Length);
                response.OutputStream.Close();
            }
-
+            catch (SocketException)
+            {
+                // Shutdown
+            }
+            catch (Exception ex)
+            {
+                if (!Stop)
+                    LogManager.GetCurrentClassLogger().Error(ex);
+            }
+        }

        private void HttpListen(object o)
        {
@@ -101,8 +117,13 @@ namespace TimberWinR.Diagnostics
                    processRequest();
                }
            }
+            catch (SocketException)
+            {
+                // Shutdown
+            }
            catch (Exception ex)
            {
+                if (!Stop)
                LogManager.GetCurrentClassLogger().Error("Diagnostic Listener Error: {0}", ex.ToString());
            }
        }
@@ -140,7 +161,7 @@ namespace TimberWinR.Diagnostics
            var tcpClient = (TcpClient)client;
            NetworkStream clientStream = null;

-            Console.WriteLine("Handle new diag client: {0}, {1}", tcpClient.Connected, tcpClient.Client.RemoteEndPoint.ToString());
+            //     Console.WriteLine("Handle new diag client: {0}, {1}", tcpClient.Connected, tcpClient.Client.RemoteEndPoint.ToString());
            try
            {
                using (clientStream = tcpClient.GetStream())
@@ -183,7 +204,7 @@ namespace TimberWinR.Diagnostics

        public void Shutdown()
        {
-            
+            Stop = true;
            try
            {
                if (web != null && web.IsListening)
@@ -193,11 +214,9 @@ namespace TimberWinR.Diagnostics
                    web = null;
                }
            }
-            catch (Exception ex)
+            catch (Exception)
            {
-                LogManager.GetCurrentClassLogger().Error(ex);
            }
        }
-
    }
 }
--- a/TimberWinR/Filters/GrokFilter.cs
+++ b/TimberWinR/Filters/GrokFilter.cs
@@ -30,8 +30,6 @@ namespace TimberWinR.Parser
        }
    }

-
-
    public partial class Grok : LogstashFilter
    {
        public override JObject ToJson()
@@ -47,6 +45,7 @@ namespace TimberWinR.Parser
                         new JProperty("condition", Condition),
                         new JProperty("addfields", AddField),
                         new JProperty("addtags", AddTag),
+                         new JProperty("drop", DropIfMatch),
                         new JProperty("type", Type),
                         new JProperty("removefields", RemoveField),
                         new JProperty("removetag", RemoveTag)
@@ -91,10 +90,13 @@ namespace TimberWinR.Parser
            return true;
        }

+        // Test for any true matching condition(s)
        private bool Matches(Newtonsoft.Json.Linq.JObject json)
        {
-            string field = Match[0];
-            string expr = Match[1];
+            for (int i = 0; i < Match.Length; i += 2)
+            {
+                string field = Match[i];
+                string expr = Match[i + 1];

                JToken token = null;
                if (json.TryGetValue(field, out token))
@@ -118,8 +120,7 @@ namespace TimberWinR.Parser
                    }
                    if (string.IsNullOrEmpty(expr))
                        return true; // Empty field is no match                 
-                else
-                    return false;
+                }
            }
            return false; // Not specified is failure
        }
--- a/TimberWinR/Filters/JsonFilter.cs
+++ b/TimberWinR/Filters/JsonFilter.cs
@@ -3,6 +3,7 @@ using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using System.Xml.Linq;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
@@ -12,6 +13,8 @@ namespace TimberWinR.Parser
 {
    public partial class Json : LogstashFilter
    {
+        private long _errorCount;
+
        public Json()
        {
            RemoveSource = true;
@@ -22,6 +25,7 @@ namespace TimberWinR.Parser
               new JProperty("json",
                   new JObject(
                       new JProperty("condition", Condition),
+                       new JProperty("errors", _errorCount),
                       new JProperty("source", Source),
                       new JProperty("promote", Source),
                       new JProperty("target", Target),
@@ -88,10 +92,13 @@ namespace TimberWinR.Parser

                    if (Rename != null && Rename.Length > 0)
                    {
-                        string oldName = ExpandField(Rename[0], json);
-                        string newName = ExpandField(Rename[1], json);
+                        for (int i = 0; i < Rename.Length; i += 2)
+                        {
+                            string oldName = ExpandField(Rename[i], json);
+                            string newName = ExpandField(Rename[i+1], json);
                            RenameProperty(json, oldName, newName);
                        }
+                    }

                    if (RemoveSource)
                    {
@@ -102,6 +109,7 @@ namespace TimberWinR.Parser
                catch (Exception ex)
                {
                    LogManager.GetCurrentClassLogger().Error(ex);
+                    Interlocked.Increment(ref _errorCount);
                    return true;
                }
            }
--- a/TimberWinR/ICodec.cs
+++ b/TimberWinR/ICodec.cs
@@ -0,0 +1,13 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using TimberWinR.Inputs;
+
+namespace TimberWinR
+{
+    public interface ICodec
+    {
+        void Apply(string msg, InputListener listener);
+    }
+}
--- a/TimberWinR/Inputs/GeneratorInput.cs
+++ b/TimberWinR/Inputs/GeneratorInput.cs
@@ -0,0 +1,88 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using RestSharp.Extensions;
+using TimberWinR.Codecs;
+using TimberWinR.Parser;
+
+
+namespace TimberWinR.Inputs
+{
+    public class GeneratorInput : InputListener
+    {
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("message", _params.Message),
+                new JProperty("messages", _sentMessages),
+                new JProperty("generator", "enabled"));
+            return json;
+        }
+
+        private TimberWinR.Parser.GeneratorParameters _params;
+        private Thread _listenThread;
+        private ICodec _codec;
+        private int _sentMessages;
+
+        public GeneratorInput(TimberWinR.Parser.GeneratorParameters parameters, CancellationToken cancelToken)
+            : base(cancelToken, "Win32-InputGen")
+        {
+            _params = parameters;
+
+            if (_params.CodecArguments != null)
+            {
+                switch (_params.CodecArguments.Type)
+                {
+                    case CodecArguments.CodecType.json:
+                        _codec = new JsonCodec(_params.CodecArguments);
+                        break;
+                    case CodecArguments.CodecType.multiline:
+                        _codec = new Multiline(_params.CodecArguments);
+                        break;
+                    case CodecArguments.CodecType.plain:
+                        _codec = new PlainCodec(_params.CodecArguments);
+                        break;
+                }
+            }
+
+            _listenThread = new Thread(new ThreadStart(GenerateData));
+            _listenThread.Start();
+        }
+
+        private void GenerateData()
+        {
+            LogManager.GetCurrentClassLogger().Info("Generator Creating {0} Lines", _params.Count);
+
+            int numMessages = _params.Count;         
+
+            // Infinite or until done.
+            for (int i = 0; (_params.Count == 0 || i < numMessages); i++)
+            {
+                if (CancelToken.IsCancellationRequested)
+                    break;
+
+                string msg = ToPrintable(_params.Message);
+
+                if (_codec != null)
+                    _codec.Apply(msg, this);
+                else
+                {
+                    JObject jo = new JObject();
+                    jo["Message"] = msg;
+                    AddDefaultFields(jo);
+                    ProcessJson(jo);
+                }
+
+                Thread.Sleep(_params.Rate);
+            }
+
+            Finished();
+        }
+    }
+}
--- a/TimberWinR/Inputs/IISW3CInputListener.cs
+++ b/TimberWinR/Inputs/IISW3CInputListener.cs
@@ -16,12 +16,12 @@ namespace TimberWinR.Inputs
    public class IISW3CInputListener : InputListener
    {
        private readonly int _pollingIntervalInSeconds;
-        private readonly Parser.IISW3CLog _arguments;
+        private readonly Parser.IISW3CLogParameters _arguments;
        private long _receivedMessages;
        public bool Stop { get; set; }
        private IisW3CRowReader rowReader;      

-        public IISW3CInputListener(Parser.IISW3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
+        public IISW3CInputListener(Parser.IISW3CLogParameters arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
            : base(cancelToken, "Win32-IISLog")
        {          
            _arguments = arguments;
@@ -138,7 +138,7 @@ namespace TimberWinR.Inputs
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                        }
-                        catch (OperationCanceledException oce)
+                        catch (OperationCanceledException)
                        {
                            break;
                        }
--- a/TimberWinR/Inputs/InputListener.cs
+++ b/TimberWinR/Inputs/InputListener.cs
@@ -1,5 +1,6 @@
 using System.IO;
 using System.Runtime.InteropServices;
+using Microsoft.Win32;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -18,6 +19,8 @@ namespace TimberWinR.Inputs
        private string _typeName;
        public AutoResetEvent FinishedEvent { get; set; }
        public string CheckpointFileName { get; set; }
+        private object _locker = new object();
+        public List<string> Files { get; set; }

        public string InputType
        {
@@ -28,6 +31,7 @@ namespace TimberWinR.Inputs

        public InputListener(CancellationToken token, string typeName)
        {
+            Files = new List<string>();
            CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString()));          
          
            this.FinishedEvent = new AutoResetEvent(false);
@@ -40,6 +44,24 @@ namespace TimberWinR.Inputs
                             .ToString();    
        }

+        public void SetTypeName(string newType)
+        {
+            _typeName = newType;
+        }
+
+        public bool HaveSeenFile(string fileName)
+        {
+            return Files.Contains(fileName);
+        }
+
+        protected void SaveVisitedFileName(string fileName)
+        {
+            lock (_locker)
+            {
+                if (!HaveSeenFile(fileName))
+                    Files.Add(fileName);
+            }
+        }
        protected string ToPrintable(string inputString)
        {
            string asAscii = Encoding.ASCII.GetString(
@@ -58,17 +80,17 @@ namespace TimberWinR.Inputs

        public void Finished()
        {
-            LogManager.GetCurrentClassLogger().Info("Signaling Event Shutdown {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Signalling Event Shutdown {1}", Thread.CurrentThread.ManagedThreadId, InputType);
            FinishedEvent.Set();
-            LogManager.GetCurrentClassLogger().Info("Finished signaling Shutdown {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Finished signalling Shutdown {1}", Thread.CurrentThread.ManagedThreadId, InputType);
        }
+
        public virtual void Shutdown()
        {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1}", Thread.CurrentThread.ManagedThreadId, InputType);

            FinishedEvent.WaitOne();
          
-            LogManager.GetCurrentClassLogger().Info("Finished Wait For {0}", InputType);
            try
            {
                if (File.Exists(CheckpointFileName))
@@ -80,7 +102,62 @@ namespace TimberWinR.Inputs
            }          
        }
       
-        protected virtual void AddDefaultFields(JObject json)
+        protected void AddOrModify(JObject json, string fieldName, string fieldValue)
+        {
+            if (json[fieldName] == null)
+                json.Add(fieldName, fieldValue);
+            else
+                json[fieldName] = fieldValue;
+        }
+
+        protected void RenameProperty(JObject json, string oldName, string newName)
+        {
+            JToken token = json[oldName];
+            if (token != null)
+            {
+                json.Add(newName, token.DeepClone());
+                json.Remove(oldName);
+            }
+        }
+
+        protected string ExpandField(string fieldName, JObject json)
+        {
+            foreach (var token in json.Children().ToList())
+            {
+                string replaceString = "%{" + token.Path + "}";
+                fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
+            }
+            return fieldName;
+        }
+
+        protected void EnsureRollingCaught()
+        {
+            try
+            {
+                const string mteKey = @"SYSTEM\CurrentControlSet\Control\FileSystem";
+
+                var mte = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(mteKey).GetValue("MaximumTunnelEntries");
+                if (mte == null || (int)mte != 0)
+                {
+                    LogManager.GetCurrentClassLogger()
+                        .Error(
+                            "HKLM\\{0}\\MaximumTunnelEntries is not set to accurately detect log rolling, a DWORD value of 0 is required.",
+                            mteKey);
+                    Microsoft.Win32.Registry.LocalMachine.CreateSubKey(mteKey).SetValue("MaximumTunnelEntries", 0, RegistryValueKind.DWord);
+                    LogManager.GetCurrentClassLogger()
+                      .Error(
+                          "HKLM\\{0}\\MaximumTunnelEntries is now set to 0, A reboot is now required to fix this issue.  See http://support.microsoft.com/en-us/kb/172190 for details",
+                          mteKey);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }           
+        }
+
+
+        public virtual void AddDefaultFields(JObject json)
        {
            if (json["type"] == null)
                json.Add(new JProperty("type", _typeName));
@@ -100,7 +177,7 @@ namespace TimberWinR.Inputs
                json.Add(new JProperty("UtcTimestamp", utc.ToString("o")));
        }

-        protected void ProcessJson(JObject json)
+        public void ProcessJson(JObject json)
        {
            if (OnMessageRecieved != null)
            {
--- a/TimberWinR/Inputs/LogsFileDatabase.cs
+++ b/TimberWinR/Inputs/LogsFileDatabase.cs
@@ -0,0 +1,278 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using TimberWinR.Parser;
+
+
+namespace TimberWinR.Inputs
+{
+    //
+    // Maintain persistent state for Log files (to be used across restarts)
+    //
+    public class LogsFileDatabase
+    {
+        private static readonly object _locker = new object();
+        private List<LogsFileDatabaseEntry> Entries { get; set; }
+        private string DatabaseDirectory { get; set; }
+        public string DatabaseFileName
+        {
+            get { return Path.Combine(DatabaseDirectory, ".timberwinrdb"); }
+        }
+
+        public static Manager Manager { get; set; }
+
+        private static LogsFileDatabase instance;
+
+        private bool ExistingFile(string logName)
+        {
+            lock (_locker)
+            {
+                return ExistingFileTest(logName);
+            }
+        }
+       
+        //
+        // Lookup the database entry for this log file, returns null if there isnt one.
+        //
+        private LogsFileDatabaseEntry FindFile(string logName)
+        {
+            lock (_locker)
+            {
+                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+                return existingEntry;
+            }
+        }
+
+        private bool ExistingFileTest(string logName)
+        {
+            var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+            return existingEntry != null;
+        }
+
+        private void RemoveFileEntry(string logName)
+        {
+            lock (_locker)
+            {
+                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+                if (existingEntry != null)
+                {
+                    Entries.Remove(existingEntry);
+                    WriteDatabaseFileNoLock();
+                }
+            }
+        }
+
+        private LogsFileDatabaseEntry AddFileEntry(string logName)
+        {
+            var de = new LogsFileDatabaseEntry();
+            lock (_locker)
+            {
+                var fi = new FileInfo(logName);             
+                de.FileName = logName;
+                de.LogFileExists = fi.Exists;
+                de.Previous = "";
+                de.NewFile = true;
+                de.ProcessedFile = false;
+                de.LastPosition = fi.Length;
+                de.SampleTime = DateTime.UtcNow;
+                de.CreationTimeUtc = fi.CreationTimeUtc;               
+
+                Entries.Add(de);
+                WriteDatabaseFileNoLock();
+            }
+            return de;
+        }      
+
+        public static LogsFileDatabaseEntry LookupLogFile(string logName)
+        {
+            LogsFileDatabaseEntry dbe = Instance.FindFile(logName);
+            if (dbe == null)
+                dbe = Instance.AddFileEntry(logName);
+
+            FileInfo fi = new FileInfo(logName);          
+
+            dbe.LogFileExists = fi.Exists;
+            var creationTime = fi.CreationTimeUtc;
+
+            if (dbe.LogFileExists && creationTime != dbe.CreationTimeUtc)
+            {
+                dbe.NewFile = true;
+                dbe.Previous = "";
+            }
+            dbe.CreationTimeUtc = creationTime;
+      
+            return dbe;
+        }
+
+        // Find all the non-existent entries and remove them.
+        private void PruneFiles()
+        {
+            lock (_locker)
+            {
+                foreach(var entry in Entries.ToList())
+                {
+                    FileInfo fi = new FileInfo(entry.FileName);
+                    if (!fi.Exists)
+                        Entries.Remove(entry);
+                }
+                WriteDatabaseFileNoLock();
+            }
+        }
+
+        public static void Update(LogsFileDatabaseEntry dbe, bool processedFile, long lastOffset)
+        {
+            dbe.ProcessedFile = processedFile;
+            dbe.LogFileExists = File.Exists(dbe.FileName);
+            Instance.UpdateEntry(dbe, lastOffset);           
+        }
+
+        public static void Roll(LogsFileDatabaseEntry dbe)
+        {
+            dbe.ProcessedFile = false;
+            dbe.LastPosition = 0;
+            dbe.Previous = "";
+            Instance.UpdateEntry(dbe, 0);
+            dbe.NewFile = true;           
+        }
+
+        private void UpdateEntry(LogsFileDatabaseEntry dbe, long lastOffset)
+        {
+            lock (_locker)
+            {
+                var fi = new FileInfo(dbe.FileName);                
+                dbe.NewFile = !fi.Exists;
+                dbe.CreationTimeUtc = fi.CreationTimeUtc;
+                dbe.SampleTime = DateTime.UtcNow;                 
+                dbe.LastPosition = lastOffset;
+              
+                WriteDatabaseFileNoLock();
+            }
+        }
+        public static LogsFileDatabase Instance
+        {
+            get
+            {
+                if (instance == null)
+                {
+                    instance = new LogsFileDatabase(Manager.LogfileDir);
+                    lock (_locker)
+                    {
+                        if (!Directory.Exists(instance.DatabaseDirectory))
+                            Directory.CreateDirectory(instance.DatabaseDirectory);
+                        // If it exists, read the current state, otherwise create an empty database.
+                        if (File.Exists(instance.DatabaseFileName))
+                            instance.ReadDatabaseNoLock();
+                        else
+                            instance.WriteDatabaseFileNoLock();
+
+                        if (instance.Entries == null)
+                            instance.Entries = new List<LogsFileDatabaseEntry>();
+
+                        instance.PruneFiles();
+                    }
+                }
+                return instance;
+            }
+        }
+
+
+        // Serialize in the Database
+        private void ReadDatabaseNoLock()
+        {
+            try
+            {
+                var serializer = new JsonSerializer();
+                if (File.Exists(DatabaseFileName))
+                    Entries =
+                        JsonConvert.DeserializeObject<List<LogsFileDatabaseEntry>>(File.ReadAllText(DatabaseFileName));
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger()
+                    .Error("Error reading database '{0}': {1}", DatabaseFileName, ex.ToString());
+                try
+                {
+                    if (File.Exists(DatabaseFileName))
+                        File.Delete(DatabaseFileName);
+                    LogManager.GetCurrentClassLogger().Info("Creating New Database '{0}'", DatabaseFileName);
+                    WriteDatabaseLock();
+                }
+                catch (Exception ex2)
+                {
+                    LogManager.GetCurrentClassLogger().Info("Error Creating New Database '{0}': {1}", DatabaseFileName, ex2.ToString());
+                }
+            }
+        }
+        private void WriteDatabaseFileNoLock()
+        {
+            try
+            {
+                File.WriteAllText(DatabaseFileName, JsonConvert.SerializeObject(instance.Entries), Encoding.UTF8);
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger()
+                    .Error("Error saving database '{0}': {1}", DatabaseFileName, ex.ToString());
+            }
+        }
+
+
+        private void ReadDatabaseLock()
+        {
+            lock (_locker)
+            {
+                ReadDatabaseNoLock();
+            }
+
+        }
+        private void WriteDatabaseLock()
+        {
+            lock (_locker)
+            {
+                WriteDatabaseFileNoLock();
+            }
+        }
+
+        private LogsFileDatabase(string databaseDirectory)
+        {
+            DatabaseDirectory = databaseDirectory;
+            Entries = new List<LogsFileDatabaseEntry>();
+        }
+
+    }
+
+
+    //
+    // Represents a log file to be tailed
+    // 
+    public class LogsFileDatabaseEntry
+    {
+        [JsonIgnore]
+        public bool NewFile { get; set; }
+        public bool ProcessedFile { get; set; }
+        public bool LogFileExists { get; set; }
+        public string FileName { get; set; }       
+        public DateTime CreationTimeUtc { get; set; }
+        public DateTime SampleTime { get; set; }
+        public long LastPosition { get; set; }
+        public long LinesProcessed
+        {
+            get { return _linesProcessed; }
+        }
+
+        private int _linesProcessed;
+        public void IncrementLineCount()
+        {
+            Interlocked.Increment(ref _linesProcessed);
+        }
+        public string Previous { get; set; }
+    }
+
+}
--- a/TimberWinR/Inputs/LogsListener.cs
+++ b/TimberWinR/Inputs/LogsListener.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Net.Configuration;
 using System.Runtime.InteropServices;
@@ -9,12 +10,13 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.IO;
 using Interop.MSUtil;
+using Microsoft.Win32;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;

 using NLog;
-
+using TimberWinR.Codecs;
 using LogQuery = Interop.MSUtil.LogQueryClassClass;
 using TextLineInputFormat = Interop.MSUtil.COMTextLineInputContextClass;
 using LogRecordSet = Interop.MSUtil.ILogRecordset;
@@ -22,182 +24,53 @@ using TimberWinR.Parser;

 namespace TimberWinR.Inputs
 {
-    public class LogsFileDatabase
-    {
-        private static readonly object _locker = new object();
-        private List<LogsFileDatabaseEntry> Entries { get; set; }
-        private string DatabaseDirectory { get; set; }
-        public string DatabaseFileName
-        {
-            get { return Path.Combine(DatabaseDirectory, ".timberwinrdb"); }
-        }
-
-        public static Manager Manager { get; set; }
-
-        private static LogsFileDatabase instance;
-
-        private bool ExistingFile(string logName)
-        {
-            lock (_locker)
-            {
-                return ExistingFileTest(logName);
-            }
-        }
-
-        private bool ExistingFileTest(string logName)
-        {
-            var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
-            return existingEntry != null;
-        }
-
-        private void RemoveFileEntry(string logName)
-        {
-            lock (_locker)
-            {
-                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
-                if (existingEntry != null)
-                {
-                    Entries.Remove(existingEntry);
-                    WriteDatabaseFileNoLock();
-                }
-            }
-        }
-
-        private LogsFileDatabaseEntry AddFileEntry(string logName, TextLineInputFormat fmt)
-        {
-            LogsFileDatabaseEntry de = new LogsFileDatabaseEntry();
-            lock (_locker)
-            {
-                var lq = new LogQuery();
-                FileInfo fi = new FileInfo(logName);
-                de.FileName = logName;
-                de.Size = fi.Length;
-                de.SampleTime = DateTime.UtcNow;
-                de.CreationTime = fi.CreationTimeUtc;
-                if (fi.Exists)
-                {
-                    var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
-                    var rcount = lq.Execute(qcount, fmt);
-                    var qr = rcount.getRecord();
-                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
-                    de.MaxRecords = lrn;
-                }
-                Entries.Add(de);
-                WriteDatabaseFileNoLock();
-            }
-            return de;
-        }
-
-        public static LogsFileDatabaseEntry AddLogFile(string logName, TextLineInputFormat fmt)
-        {
-            Instance.RemoveFileEntry(logName); // Remove if already exists, otherwise ignores.
-            return Instance.AddFileEntry(logName, fmt);
-        }
-
-        public static LogsFileDatabase Instance
-        {
-            get
-            {
-                if (instance == null)
-                {
-                    instance = new LogsFileDatabase(Manager.LogfileDir);
-                    lock (_locker)
-                    {
-                        if (!Directory.Exists(instance.DatabaseDirectory))
-                        {
-                            Directory.CreateDirectory(instance.DatabaseDirectory);
-                        }
-                        if (File.Exists(instance.DatabaseFileName))
-                            instance.ReadDatabaseNoLock();
-                        else
-                            instance.WriteDatabaseFileNoLock();
-                    }
-                }
-                return instance;
-            }
-        }
-
-        private void ReadDatabaseNoLock()
-        {
-            JsonSerializer serializer = new JsonSerializer();
-            if (File.Exists(DatabaseFileName))
-                Entries = JsonConvert.DeserializeObject<List<LogsFileDatabaseEntry>>(File.ReadAllText(DatabaseFileName));
-        }
-        private void WriteDatabaseFileNoLock()
-        {
-            File.WriteAllText(DatabaseFileName, JsonConvert.SerializeObject(instance.Entries), Encoding.UTF8);
-        }
-
-
-        private void ReadDatabaseLock()
-        {
-            lock (_locker)
-            {
-                ReadDatabaseNoLock();
-            }
-
-        }
-        private void WriteDatabaseLock()
-        {
-            lock (_locker)
-            {
-                WriteDatabaseFileNoLock();
-            }
-        }
-
-        private LogsFileDatabase(string databaseDirectory)
-        {
-            DatabaseDirectory = databaseDirectory;
-            Entries = new List<LogsFileDatabaseEntry>();
-        }
-
-    }
-
-    public class LogsFileDatabaseEntry
-    {
-        public string FileName { get; set; }
-        public Int64 MaxRecords { get; set; }
-        public DateTime CreationTime { get; set; }
-        public DateTime SampleTime { get; set; }
-        public long Size { get; set; }
-    }
-
    /// <summary>
    /// Tail a file.
    /// </summary>
    public class LogsListener : InputListener
    {
+        private object _locker = new object();
        private int _pollingIntervalInSeconds;
-        private TimberWinR.Parser.Log _arguments;
+        private TimberWinR.Parser.LogParameters _arguments;
        private long _receivedMessages;
-        private Dictionary<string, Int64> _logFileMaxRecords;
-        private Dictionary<string, DateTime> _logFileCreationTimes;
-        private Dictionary<string, DateTime> _logFileSampleTimes;
-        private Dictionary<string, long> _logFileSizes;
-        private Codec _codec;
-        private List<string> _multiline { get; set; }
+        private CodecArguments _codecArguments;
+        private ICodec _codec;
      
        public bool Stop { get; set; }
+        public bool IsWildcardFilePattern { get; set; }

-        public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken)
+
+        public LogsListener(TimberWinR.Parser.LogParameters arguments, CancellationToken cancelToken)
            : base(cancelToken, "Win32-FileLog")
        {
            Stop = false;
        
-            _codec = arguments.Codec;
-            _logFileMaxRecords = new Dictionary<string, Int64>();
-            _logFileCreationTimes = new Dictionary<string, DateTime>();
-            _logFileSampleTimes = new Dictionary<string, DateTime>();
-            _logFileSizes = new Dictionary<string, long>();
+            EnsureRollingCaught();
+
+             _codecArguments = arguments.CodecArguments;
+
+            _codecArguments = arguments.CodecArguments;
+            if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
+                _codec = new Multiline(_codecArguments);
+
+            if (!string.IsNullOrEmpty(arguments.Type))
+                SetTypeName(arguments.Type);

            _receivedMessages = 0;
            _arguments = arguments;
            _pollingIntervalInSeconds = arguments.Interval;

+            IsWildcardFilePattern = arguments.Location.Contains('*');
+
            foreach (string srcFile in _arguments.Location.Split(','))
            {
                string file = srcFile.Trim();
-                Task.Factory.StartNew(() => FileWatcher(file));
+                string dir = Path.GetDirectoryName(file);
+                if (string.IsNullOrEmpty(dir))
+                    dir = Environment.CurrentDirectory;
+                string fileSpec = Path.Combine(dir, file);
+
+                Task.Factory.StartNew(() => FileWatcher(fileSpec));
            }
        }

@@ -208,9 +81,9 @@ namespace TimberWinR.Inputs
            base.Shutdown();
        }

+      
        public override JObject ToJson()
        {
-
            JObject json = new JObject(
                new JProperty("log",
                    new JObject(
@@ -221,35 +94,25 @@ namespace TimberWinR.Inputs
                        new JProperty("codepage", _arguments.CodePage),
                        new JProperty("splitLongLines", _arguments.SplitLongLines),
                        new JProperty("recurse", _arguments.Recurse),
-
+                        new JProperty("filedb",
+                            new JArray(from f in Files.ToList()
+                                       select JObject.FromObject(LogsFileDatabase.LookupLogFile(f)))),
                        new JProperty("files",
-                            new JArray(from f in _logFileMaxRecords.Keys
-                                       select new JValue(f))),
-                        new JProperty("fileSampleTimes",
-                            new JArray(from f in _logFileSampleTimes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileSizes",
-                            new JArray(from f in _logFileSizes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileIndices",
-                            new JArray(from f in _logFileMaxRecords.Values
-                                       select new JValue(f))),
-                        new JProperty("fileCreationDates",
-                            new JArray(from f in _logFileCreationTimes.Values
+                            new JArray(from f in Files.ToList()
                                       select new JValue(f)))
                        )));


-            if (_codec != null)
+            if (_codecArguments != null)
            {
                var cp = new JProperty("codec",
                    new JArray(
                        new JObject(
-                            new JProperty("type", _codec.Type.ToString()),
-                            new JProperty("what", _codec.What.ToString()),
-                            new JProperty("negate", _codec.Negate),
-                            new JProperty("multilineTag", _codec.MultilineTag),
-                            new JProperty("pattern", _codec.Pattern))));
+                            new JProperty("type", _codecArguments.Type.ToString()),
+                            new JProperty("what", _codecArguments.What.ToString()),
+                            new JProperty("negate", _codecArguments.Negate),
+                            new JProperty("multilineTag", _codecArguments.MultilineTag),
+                            new JProperty("pattern", _codecArguments.Pattern))));
                json.Add(cp);
            }

@@ -257,78 +120,6 @@ namespace TimberWinR.Inputs
            return json;
        }

-        // return true to cancel codec
-        private void applyMultilineCodec(string msg)
-        {
-            if (_codec.Re == null)
-                _codec.Re = new Regex(_codec.Pattern);
-
-            Match match = _codec.Re.Match(msg);
-
-            bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
-
-            switch (_codec.What)
-            {
-                case Codec.WhatType.previous:
-                    if (isMatch)
-                    {
-                        if (_multiline == null)
-                            _multiline = new List<string>();
-
-                        _multiline.Add(msg);
-                    }
-                    else // No Match
-                    {
-                        if (_multiline != null)
-                        {
-                            string single = string.Join("\n", _multiline.ToArray());
-                            _multiline = null;
-                            JObject jo = new JObject();
-                            jo["message"] = single;
-                            jo.Add("tags", new JArray(_codec.MultilineTag));
-                            AddDefaultFields(jo);
-                            ProcessJson(jo);
-                            _receivedMessages++;
-                        }
-                        _multiline = new List<string>();
-                        _multiline.Add(msg);
-                    }
-                    break;
-                case Codec.WhatType.next:
-                    if (isMatch)
-                    {
-                        if (_multiline == null)
-                            _multiline = new List<string>();
-                        _multiline.Add(msg);
-                    }
-                    else // No match
-                    {
-                        if (_multiline != null)
-                        {
-                            _multiline.Add(msg);
-                            string single = string.Join("\n", _multiline.ToArray());
-                            _multiline = null;
-                            JObject jo = new JObject();
-                            jo["message"] = single;
-                            jo.Add("tags", new JArray(_codec.MultilineTag));
-                            AddDefaultFields(jo);
-                            ProcessJson(jo);
-                            _receivedMessages++;
-                        }
-                        else
-                        {
-                            JObject jo = new JObject();
-                            jo["message"] = msg;
-                            AddDefaultFields(jo);
-                            ProcessJson(jo);
-                            _receivedMessages++;
-                        }
-                    }
-                    break;
-            }
-        }
-
-
        private void FileWatcher(string fileToWatch)
        {
            var iFmt = new TextLineInputFormat()
@@ -338,6 +129,8 @@ namespace TimberWinR.Inputs
                recurse = _arguments.Recurse
            };

+            Dictionary<string, string> _fnfmap = new Dictionary<string, string>();
+
            using (var syncHandle = new ManualResetEventSlim())
            {
                // Execute the query
@@ -356,44 +149,32 @@ namespace TimberWinR.Inputs
                                string logName = record.getValue("LogFilename") as string;
                                FileInfo fi = new FileInfo(logName);                             

-                                if (!fi.Exists)
-                                {
-                                    _logFileCreationTimes.Remove(logName);
-                                    _logFileMaxRecords.Remove(logName);
-                                    _logFileSizes.Remove(logName);
-                                }
+                                var dbe = LogsFileDatabase.LookupLogFile(logName);
                    
-                                _logFileSampleTimes[logName] = DateTime.UtcNow;
+                                SaveVisitedFileName(dbe.FileName);

                                DateTime creationTime = fi.CreationTimeUtc;
-                                bool logHasRolled = (_logFileCreationTimes.ContainsKey(logName) &&
-                                                     creationTime > _logFileCreationTimes[logName]) ||
-                                                    (_logFileSizes.ContainsKey(logName) &&
-                                                     fi.Length < _logFileSizes[logName]);
+                                bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition);

-
-                                if (!_logFileMaxRecords.ContainsKey(logName) || logHasRolled)
-                                {
-                                    _logFileCreationTimes[logName] = creationTime;
-                                    _logFileSizes[logName] = fi.Length;
-                                    var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
-                                    var rcount = oLogQuery.Execute(qcount, iFmt);
-                                    var qr = rcount.getRecord();
-                                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
                                if (logHasRolled)
                                {
                                    LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
-                                        lrn = 0;
-                                    }
-                                    _logFileMaxRecords[logName] = lrn;
+                                    LogsFileDatabase.Roll(dbe);
                                }

-                                _logFileSizes[logName] = fi.Length;
+                                // Log has rolled or this is a new file, or we haven't processed yet.
+                                bool processWholeFile = logHasRolled || !dbe.ProcessedFile;
+
+                                if (processWholeFile)
+                                    LogsFileDatabase.Update(dbe, true, 0);
+
                            }
                            rsfiles.close();
-                            foreach (string fileName in _logFileMaxRecords.Keys.ToList())
+                            foreach (string fileName in Files.ToList())
                            {
-                                var lastRecordNumber = _logFileMaxRecords[fileName];
+                                var dbe = LogsFileDatabase.LookupLogFile(fileName);
+
+                                var lastRecordNumber = dbe.LastPosition;
                                var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName,
                                    lastRecordNumber);

@@ -436,17 +217,22 @@ namespace TimberWinR.Inputs
                                    string msg = json["Text"].ToString();
                                    if (!string.IsNullOrEmpty(msg))
                                    {
-                                        if (_codec != null && _codec.Type == Codec.CodecType.multiline)
-                                            applyMultilineCodec(msg);
+                                        if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
+                                        {
+                                            _codec.Apply(msg, this);
+                                            _receivedMessages++;
+                                            dbe.IncrementLineCount();
+                                        }
                                        else
                                        {
                                            ProcessJson(json);
+                                            dbe.IncrementLineCount();
                                             _receivedMessages++;
                                        }
                                    }

                                    var lrn = (Int64)record.getValueEx("Index");
-                                    _logFileMaxRecords[fileName] = lrn;
+                                    LogsFileDatabase.Update(dbe, true, lrn);
                                    GC.Collect();
                                }

@@ -455,16 +241,20 @@ namespace TimberWinR.Inputs
                                rs.close();
                                rs = null;
                                GC.Collect();
+
                            }
-                            // Sleep 
-                            if (!Stop)
-                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                        }
                        catch (FileNotFoundException fnfex)
+                        {
+                            string fn = fnfex.FileName;
+
+                            if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn))
                            {
                                LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
+                                _fnfmap[fn] = fn;
                            }
-                        catch (OperationCanceledException oce)
+                        }
+                        catch (OperationCanceledException)
                        {
                            break;
                        }
@@ -473,8 +263,21 @@ namespace TimberWinR.Inputs
                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                        finally
+                        {
+                            try
                            {
                                oLogQuery = null;
+                                // Sleep 
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                            }
+                            catch (OperationCanceledException)
+                            {
+                            }
+                            catch (Exception ex1)
+                            {
+                                LogManager.GetCurrentClassLogger().Warn(ex1);
+                            }
                        }
                    }
                }
--- a/TimberWinR/Inputs/StdinListener.cs
+++ b/TimberWinR/Inputs/StdinListener.cs
@@ -2,26 +2,36 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
+using TimberWinR.Codecs;
 using TimberWinR.Parser;

 namespace TimberWinR.Inputs
 {
    public class StdinListener : InputListener
    {
+        [DllImport("User32.Dll", EntryPoint = "PostMessageA")]
+        private static extern bool PostMessage(IntPtr hWnd, uint msg, int wParam, int lParam);
+  
        private Thread _listenThread;
-        private Codec _codec;
-        private List<string> _multiline { get; set; }
+        private CodecArguments _codecArguments;
+        private ICodec _codec;
+        const int VK_RETURN = 0x0D;
+        const int WM_KEYDOWN = 0x100;

        public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken)
            : base(cancelToken, "Win32-Console")
        {
-            _codec = arguments.Codec;
+            _codecArguments = arguments.CodecArguments;
+            if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
+                _codec = new Multiline(_codecArguments);
+
            _listenThread = new Thread(new ThreadStart(ListenToStdin));
            _listenThread.Start();
        }
@@ -32,16 +42,16 @@ namespace TimberWinR.Inputs
                new JProperty("stdin", "enabled"));

            
-            if (_codec != null)
+            if (_codecArguments != null)
            {
                var cp = new JProperty("codec",
                    new JArray(
                        new JObject(
-                            new JProperty("type", _codec.Type.ToString()),
-                            new JProperty("what", _codec.What.ToString()),
-                            new JProperty("negate", _codec.Negate),
-                            new JProperty("multilineTag", _codec.MultilineTag),
-                            new JProperty("pattern", _codec.Pattern))));
+                            new JProperty("type", _codecArguments.Type.ToString()),
+                            new JProperty("what", _codecArguments.What.ToString()),
+                            new JProperty("negate", _codecArguments.Negate),
+                            new JProperty("multilineTag", _codecArguments.MultilineTag),
+                            new JProperty("pattern", _codecArguments.Pattern))));
                json.Add(cp);              
            }

@@ -51,6 +61,13 @@ namespace TimberWinR.Inputs
        public override void Shutdown()
        {
            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);    
+            // This must come from another thread.
+            ThreadPool.QueueUserWorkItem((o) =>
+            {
+                Thread.Sleep(100);
+                var hWnd = System.Diagnostics.Process.GetCurrentProcess().MainWindowHandle;
+                PostMessage(hWnd, WM_KEYDOWN, VK_RETURN, 0);
+            });
            base.Shutdown();
        }

@@ -65,8 +82,8 @@ namespace TimberWinR.Inputs
                {
                    string msg = ToPrintable(line);

-                    if (_codec != null && _codec.Type == Codec.CodecType.multiline)
-                        applyMultilineCodec(msg);
+                    if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)                   
+                        _codec.Apply(msg, this);                   
                    else
                    {
                        JObject jo = new JObject();
@@ -78,73 +95,5 @@ namespace TimberWinR.Inputs
            }
            Finished();
        }
-
-        // return true to cancel codec
-        private void applyMultilineCodec(string msg)
-        {
-            if (_codec.Re == null)
-                _codec.Re = new Regex(_codec.Pattern);
-
-            Match match = _codec.Re.Match(msg);
-
-            bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
-
-            switch (_codec.What)
-            {
-                case Codec.WhatType.previous:
-                    if (isMatch)
-                    {
-                        if (_multiline == null)
-                            _multiline = new List<string>();
-
-                        _multiline.Add(msg);
-                    }
-                    else // No Match
-                    {
-                        if (_multiline != null)
-                        {
-                            string single = string.Join("\n", _multiline.ToArray());
-                            _multiline = null;
-                            JObject jo = new JObject();
-                            jo["message"] = single;
-                            jo.Add("tags", new JArray(_codec.MultilineTag));
-                            AddDefaultFields(jo);
-                            ProcessJson(jo);                          
-                        }
-                        _multiline = new List<string>();
-                        _multiline.Add(msg);
-                    }
-                    break;
-                case Codec.WhatType.next:
-                    if (isMatch)
-                    {
-                        if (_multiline == null)
-                            _multiline = new List<string>();
-                        _multiline.Add(msg);
-                    }
-                    else // No match
-                    {
-                        if (_multiline != null)
-                        {
-                            _multiline.Add(msg);
-                            string single = string.Join("\n", _multiline.ToArray());
-                            _multiline = null;
-                            JObject jo = new JObject();
-                            jo["message"] = single;
-                            jo.Add("tags", new JArray(_codec.MultilineTag));
-                            AddDefaultFields(jo);
-                            ProcessJson(jo);                          
-                        }
-                        else
-                        {
-                            JObject jo = new JObject();
-                            jo["message"] = msg;
-                            AddDefaultFields(jo);
-                            ProcessJson(jo);                          
-                        }
-                    }
-                    break;
-            }
-        }
    }
 }
--- a/TimberWinR/Inputs/TailFileListener.cs
+++ b/TimberWinR/Inputs/TailFileListener.cs
@@ -0,0 +1,321 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Net.Configuration;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using System.IO;
+using Interop.MSUtil;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+
+using NLog;
+using NLog.LayoutRenderers;
+using TimberWinR.Codecs;
+using TimberWinR.Parser;
+
+namespace TimberWinR.Inputs
+{
+    /// <summary>
+    /// Tail a file.
+    /// </summary>
+    public class TailFileListener : InputListener
+    {
+        private object _locker = new object();
+        private int _pollingIntervalInSeconds;
+        private TimberWinR.Parser.TailFileArguments _arguments;
+        private long _receivedMessages;
+        private long _errorCount;      
+        private CodecArguments _codecArguments;
+        private ICodec _codec;
+
+        public bool Stop { get; set; }
+
+        public TailFileListener(TimberWinR.Parser.TailFileArguments arguments,
+            CancellationToken cancelToken)
+            : base(cancelToken, "Win32-TailLog")
+        {         
+            Stop = false;
+
+            EnsureRollingCaught();
+
+            _codecArguments = arguments.CodecArguments;
+            if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
+                _codec = new Multiline(_codecArguments);
+
+            if (!string.IsNullOrEmpty(arguments.Type))
+                SetTypeName(arguments.Type);
+
+            _receivedMessages = 0;
+            _arguments = arguments;
+            _pollingIntervalInSeconds = arguments.Interval;
+
+            foreach (string srcFile in _arguments.Location.Split(','))
+            {
+                string file = srcFile.Trim();
+                Task.Factory.StartNew(() => TailFileWatcher(file));
+            }
+        }
+
+        public override void Shutdown()
+        {
+            LogManager.GetCurrentClassLogger()
+                .Info("{0}: Shutting Down {1} for {2}", Thread.CurrentThread.ManagedThreadId, InputType,
+                    _arguments.Location);
+            Stop = true;
+            base.Shutdown();
+        }
+
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("taillog",
+                    new JObject(
+                        new JProperty("messages", _receivedMessages),
+                        new JProperty("errors", _errorCount),
+                        new JProperty("type", InputType),
+                        new JProperty("location", _arguments.Location),
+                        new JProperty("logSource", _arguments.LogSource),
+                        new JProperty("recurse", _arguments.Recurse),
+                        new JProperty("files",
+                            new JArray(from f in Files
+                                       select new JValue(f))),
+                        new JProperty("filedb",
+                            new JArray(from f in Files
+                                       select JObject.FromObject(LogsFileDatabase.LookupLogFile(f))))
+                        )));
+
+
+            if (_codecArguments != null)
+            {
+                var cp = new JProperty("codec",
+                    new JArray(
+                        new JObject(
+                            new JProperty("type", _codecArguments.Type.ToString()),
+                            new JProperty("what", _codecArguments.What.ToString()),
+                            new JProperty("negate", _codecArguments.Negate),
+                            new JProperty("multilineTag", _codecArguments.MultilineTag),
+                            new JProperty("pattern", _codecArguments.Pattern))));
+                json.Add(cp);
+            }
+
+
+            return json;
+        }
+
+        private void TailFileContents(string fileName, long offset, LogsFileDatabaseEntry dbe)
+        {
+            const int bufSize = 16535;
+            long prevLen = offset;
+
+            FileInfo fi = new FileInfo(fileName);
+            if (!fi.Exists)
+                return;
+
+            LogManager.GetCurrentClassLogger().Trace(":{0} Tailing File: {1} as Pos: {2}", Thread.CurrentThread.ManagedThreadId, fileName, prevLen);
+
+            using (var stream = new FileStream(fi.FullName, FileMode.Open, FileAccess.Read, FileShare.Delete | FileShare.ReadWrite))
+            {
+                stream.Seek(prevLen, SeekOrigin.Begin);
+
+                char[] buffer = new char[bufSize];
+                StringBuilder current = new StringBuilder();
+                using (StreamReader sr = new StreamReader(stream))
+                {
+                    int nRead;
+                    do
+                    {
+                        // Read a buffered amount
+                        nRead = sr.ReadBlock(buffer, 0, bufSize);
+                        for (int i = 0; i < nRead; ++i)
+                        {
+                            // We need the terminator!
+                            if (buffer[i] == '\n' || buffer[i] == '\r')
+                            {
+                                if (current.Length > 0)
+                                {
+                                    string line = string.Concat(dbe.Previous, current);
+                                    var json = new JObject();
+
+                                    if (json["logSource"] == null)
+                                    {
+                                        if (string.IsNullOrEmpty(_arguments.LogSource))
+                                            json.Add(new JProperty("logSource", fileName));
+                                        else
+                                            json.Add(new JProperty("logSource", _arguments.LogSource));
+                                    }
+
+                                    //LogManager.GetCurrentClassLogger().Debug(":{0} File: {1}:{2}  {3}", Thread.CurrentThread.ManagedThreadId, fileName, dbe.LinesProcessed, line);
+
+                                    // We've processed the partial input
+                                    dbe.Previous = "";
+                                    json["Text"] = line;
+                                    json["Index"] = dbe.LinesProcessed;
+                                    json["LogFileName"] = fileName;
+                                    if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
+                                    {
+                                        try
+                                        {                                         
+                                            _codec.Apply(line, this);
+                                            Interlocked.Increment(ref _receivedMessages);
+                                            dbe.IncrementLineCount();
+                                        }
+                                        catch (Exception ex)
+                                        {
+                                            Interlocked.Increment(ref _errorCount);
+                                            LogManager.GetCurrentClassLogger().ErrorException("Filter Error", ex);
+                                        }
+                                    }
+                                    else
+                                    {
+                                        try
+                                        {
+                                            ProcessJson(json);
+                                            dbe.IncrementLineCount();                                                                                      
+                                            Interlocked.Increment(ref _receivedMessages);
+                                            LogsFileDatabase.Update(dbe, true, sr.BaseStream.Position);
+                                        }
+                                        catch (Exception ex)
+                                        {
+                                            Interlocked.Increment(ref _errorCount);
+                                            LogManager.GetCurrentClassLogger().ErrorException("Process Error", ex);
+                                        }
+                                    }
+
+                                }
+                                current = new StringBuilder();
+                            }
+                            else // Copy character into the buffer
+                            {
+                                current.Append(buffer[i]);
+                            }
+                        }
+                    } while (nRead > 0);
+
+                    // We didn't encounter the newline, so save it.
+                    if (current.Length > 0)
+                    {
+                        dbe.Previous = current.ToString();
+                    }
+                }
+            }         
+        }
+        // One thread for each kind of file to watch, i.e. "*.log,*.txt" would be two separate
+        // threads.
+        private void TailFileWatcher(string fileToWatch)
+        {
+            Dictionary<string, string> _fnfmap = new Dictionary<string, string>();
+
+            using (var syncHandle = new ManualResetEventSlim())
+            {
+                // Execute the query
+                while (!Stop && !CancelToken.IsCancellationRequested)
+                {
+                    try
+                    {
+                        if (!CancelToken.IsCancellationRequested)
+                        {
+                            var isWildcardPattern = fileToWatch.Contains('*');
+                            string path = Path.GetDirectoryName(fileToWatch);
+                            string name = Path.GetFileName(fileToWatch);
+                            if (string.IsNullOrEmpty(path))
+                                path = ".";
+
+                            LogManager.GetCurrentClassLogger().Trace(":{0} Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, Path.Combine(path, name));
+
+                            // Ok, we have a potential file filter here as 'fileToWatch' could be foo.log or *.log
+
+                            SearchOption so = SearchOption.TopDirectoryOnly;
+                            if (_arguments.Recurse == -1)
+                                so = SearchOption.AllDirectories;
+
+                            foreach (string fileName in Directory.GetFiles(path, name, so))
+                            {
+                                var dbe = LogsFileDatabase.LookupLogFile(fileName);
+
+                                // We only spin up 1 thread for a file we haven't yet seen.                               
+                                if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile)
+                                {
+                                    LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                    LogsFileDatabase.Update(dbe, false, dbe.LastPosition);
+
+                                    Task.Factory.StartNew(() => TailFileWatcher(fileName));
+                                }
+                                else if (!isWildcardPattern)
+                                {
+                                    FileInfo fi = new FileInfo(dbe.FileName);
+                                    SaveVisitedFileName(fileName);
+
+                                    //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);                                
+                                    long length = fi.Length;
+                                    bool logHasRolled = false;
+                                    if (fi.Length < dbe.LastPosition || fi.CreationTimeUtc != dbe.CreationTimeUtc)
+                                    {
+                                        LogManager.GetCurrentClassLogger().Info("{0}: Log has Rolled: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                        logHasRolled = true;
+                                        LogsFileDatabase.Roll(dbe);
+                                    }
+                                    // Log has rolled or this is a file we are seeing for the first time.
+                                    bool processWholeFile = logHasRolled || !dbe.ProcessedFile || dbe.NewFile;
+                                    if (processWholeFile)
+                                    {
+                                        LogsFileDatabase.Update(dbe, true, 0);
+                                        LogManager.GetCurrentClassLogger().Debug("{0}: Process Whole File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                        TailFileContents(dbe.FileName, 0, dbe);
+                                    }
+                                    else
+                                    {
+                                        TailFileContents(dbe.FileName, dbe.LastPosition, dbe);
+                                    }
+                                }
+                            }
+                        }
+                    }
+                    catch (FileNotFoundException fnfex)
+                    {
+                        string fn = fnfex.FileName;
+
+                        if (!_fnfmap.ContainsKey(fn))
+                            LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
+                        _fnfmap[fn] = fn;
+                    }
+                    catch (IOException ioex)
+                    {
+                        LogManager.GetCurrentClassLogger().Debug("Log has rolled: {0}", ioex.Message);
+                    }
+                    catch (OperationCanceledException)
+                    {
+                        break;
+                    }
+                    catch (Exception ex)
+                    {
+                        LogManager.GetCurrentClassLogger().Error(ex);
+                    }
+                    finally
+                    {
+                        try
+                        {
+                            if (!Stop)
+                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                        }
+                        catch (OperationCanceledException)
+                        {
+                            Stop = true;
+                        }
+                        catch (Exception ex1)
+                        {
+                            LogManager.GetCurrentClassLogger().Warn(ex1);
+                        }
+                    }
+                }
+            }
+            Finished();
+        }
+    }
+}
+
--- a/TimberWinR/Inputs/TcpInputListener.cs
+++ b/TimberWinR/Inputs/TcpInputListener.cs
@@ -16,7 +16,9 @@ namespace TimberWinR.Inputs
        private Thread _listenThreadV4;
        private Thread _listenThreadV6;
        private readonly int _port;
+        private TimberWinR.Parser.TcpParameters _arguments;
        private long _receivedMessages;
+        private long _errorCount;

        public override JObject ToJson()
        {
@@ -24,19 +26,23 @@ namespace TimberWinR.Inputs
                new JProperty("tcp",
                    new JObject(
                        new JProperty("port", _port),
+                        new JProperty("errors", _errorCount),
                        new JProperty("messages", _receivedMessages)
                        )));
-
            return json;
        }

-        public TcpInputListener(CancellationToken cancelToken, int port = 5140)
+        public TcpInputListener(TimberWinR.Parser.TcpParameters arguments, CancellationToken cancelToken, int port = 5140)
            : base(cancelToken, "Win32-Tcp")
        {
            _port = port;
+            _arguments = arguments;

            LogManager.GetCurrentClassLogger().Info("Tcp Input(v4/v6) on Port {0} Ready", _port);

+            if (!string.IsNullOrEmpty(arguments.Type))
+                SetTypeName(arguments.Type);
+
            _receivedMessages = 0;
           
            _tcpListenerV6 = new System.Net.Sockets.TcpListener(IPAddress.IPv6Any, port);
@@ -52,7 +58,7 @@ namespace TimberWinR.Inputs

        public override void Shutdown()
        {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1}", Thread.CurrentThread.ManagedThreadId, InputType);

            this._tcpListenerV4.Stop();
            this._tcpListenerV6.Stop();
@@ -68,7 +74,6 @@ namespace TimberWinR.Inputs

            listener.Start();

-
            while (!CancelToken.IsCancellationRequested)
            {
                try
@@ -90,6 +95,33 @@ namespace TimberWinR.Inputs
            }
        }

+        //
+        // Renames, and AddFields
+        //
+        private void ApplyFilters(JObject json)
+        {
+            if (_arguments.Renames != null)
+            {
+                for (int i = 0; i < _arguments.Renames.Length; i += 2)
+                {
+                    var oldName = ExpandField(_arguments.Renames[i], json);
+                    var newName = ExpandField(_arguments.Renames[i + 1], json);
+                    RenameProperty(json, oldName, newName);
+                }
+            }
+
+            if (_arguments.AddFields != null)
+            {
+                for (int i = 0; i < _arguments.AddFields.Length; i += 2)
+                {
+                    var fieldName = ExpandField(_arguments.AddFields[i], json);
+                    var fieldValue = ExpandField(_arguments.AddFields[i + 1], json);
+                    AddOrModify(json, fieldName, fieldValue);
+                }
+            }
+        }
+
+
        private void HandleNewClient(object client)
        {
            var tcpClient = (TcpClient)client;
@@ -108,18 +140,27 @@ namespace TimberWinR.Inputs
                            try
                            {
                                JObject json = JObject.Load(reader);
+                                ApplyFilters(json);
                                ProcessJson(json);
-                                _receivedMessages++;
+                                Interlocked.Increment(ref _receivedMessages);
                            }
                            catch (Exception ex)
                            {
+                                var jex1 = LogErrors.LogException("Bad Json", ex);
+                                if (jex1 != null)
+                                    ProcessJson(jex1);
+
                                LogManager.GetCurrentClassLogger().Warn(ex);
+                                Interlocked.Increment(ref _errorCount);
                            }

                        }
                    }
                }
            }
+            catch(OperationCanceledException)
+            {
+            }
            catch (Exception ex)
            {
                LogManager.GetCurrentClassLogger().Error(ex);
--- a/TimberWinR/Inputs/UdpInputListener.cs
+++ b/TimberWinR/Inputs/UdpInputListener.cs
@@ -1,10 +1,9 @@
 using System;
-using System.IO;
+using System.Collections.Concurrent;
 using System.Text;
 using System.Threading;
 using System.Net;
 using System.Net.Sockets;
-using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;

@@ -12,92 +11,191 @@ namespace TimberWinR.Inputs
 {
    public class UdpInputListener : InputListener
    {
-        private readonly System.Net.Sockets.UdpClient _udpListener;
-        private IPEndPoint groupV4;
-        private IPEndPoint groupV6;
-
-        private Thread _listenThreadV4;
-        private Thread _listenThreadV6;
-
+        private UdpClient _udpListenerV4;
+        private IPEndPoint _udpEndpointV4;
+        private readonly BlockingCollection<byte[]> _unprocessedRawData;
+        private readonly Thread _rawDataProcessingThread;
        private readonly int _port;
        private long _receivedMessages;
-
-        private struct listenProfile
-        {
-            public IPEndPoint endPoint;
-            public UdpClient client;
-        }
+        private long _parseErrors;
+        private long _receiveErrors;
+        private long _parsedMessages;
+        private TimberWinR.Parser.UdpParameters _arguments;

        public override JObject ToJson()
        {
-            JObject json = new JObject(
-                new JProperty("udp",
-                    new JObject(
-                        new JProperty("port", _port),
-                        new JProperty("messages", _receivedMessages)
-                        )));
+            var json =
+                new JObject(new JProperty("udp",
+                    new JObject(new JProperty("port", _port),
+                        new JProperty("receive_errors", _receiveErrors),
+                        new JProperty("parse_errors", _parseErrors),
+                        new JProperty("messages", _receivedMessages),
+                        new JProperty("parsed_messages", _parsedMessages),
+                        new JProperty("unprocessed_messages", _unprocessedRawData.Count))));

            return json;
        }

-        public UdpInputListener(CancellationToken cancelToken, int port = 5140)
-            : base(cancelToken, "Win32-Udp")
+        public UdpInputListener(TimberWinR.Parser.UdpParameters arguments, CancellationToken cancelToken, int port = 5140) : base(cancelToken, "Win32-Udp")
        {
            _port = port;
-
-            LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port);
-
            _receivedMessages = 0;
+            _arguments = arguments;

-            _udpListener = new System.Net.Sockets.UdpClient(port);
+            if (!string.IsNullOrEmpty(arguments.Type))
+                SetTypeName(arguments.Type);

-            _listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener));
-            _listenThreadV4.Start(new listenProfile() { endPoint = groupV4, client = _udpListener });
+            // setup raw data processor
+            _unprocessedRawData = new BlockingCollection<byte[]>();
+            _rawDataProcessingThread = new Thread(ProcessDataLoop) { Name = "Win32-Udp-DataProcessor"};
+            _rawDataProcessingThread.Start();

-            _listenThreadV6 = new Thread(new ParameterizedThreadStart(StartListener));
-            _listenThreadV6.Start(new listenProfile() { endPoint = groupV6, client = _udpListener });
+            // start listing to udp port
+            StartListener();
        }

-
        public override void Shutdown()
        {
            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
-            _udpListener.Close();
-            Finished();
+
+            // close UDP listeners, which will end the listener threads          
+            _udpListenerV4.Close();
+
            base.Shutdown();
        }

-
-        private void StartListener(object useProfile)
+        private void StartListener()
        {
-            var profile = (listenProfile)useProfile;
+            _udpEndpointV4 = new IPEndPoint(IPAddress.Any, _port);
+
+            // setup listener
+            _udpListenerV4 = new UdpClient(_port);
+
+            // start listening on UDP port
+            StartReceiving();
+
+            // all started; log details
+            LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _udpEndpointV4);
+        }
+
+        private void StartReceiving()
+        {
+            if (!CancelToken.IsCancellationRequested)
+                _udpListenerV4.BeginReceive(DataReceived, null);
+        }
+
+        private void DataReceived(IAsyncResult result)
+        {
+            if (CancelToken.IsCancellationRequested)
+            {
+                _unprocessedRawData.CompleteAdding();
+                return;
+            }

            try
            {
-                while (!CancelToken.IsCancellationRequested)
-                {
-                    byte[] bytes = profile.client.Receive(ref profile.endPoint);
-                    try
-                    {
-                        var data = Encoding.ASCII.GetString(bytes, 0, bytes.Length);
-                        JObject json = JObject.Parse(data);
-                        ProcessJson(json);
-                        _receivedMessages++;
+                byte[] bytes = _udpListenerV4.EndReceive(result, ref _udpEndpointV4);
+                Interlocked.Increment(ref _receivedMessages);
+                StartReceiving();
+                _unprocessedRawData.Add(bytes);
            }
-                    catch (Exception ex1)
+            catch (SocketException)
            {
-                        LogManager.GetCurrentClassLogger().Warn(ex1);
+                LogManager.GetCurrentClassLogger().Info("Socked exception. Ending UDP Listener.");
+                _unprocessedRawData.CompleteAdding();
            }
-                }
-                _udpListener.Close();
+            catch (ObjectDisposedException)
+            {
+                LogManager.GetCurrentClassLogger().Info("Object disposed. Ending UDP Listener");
+                _unprocessedRawData.CompleteAdding();
            }
            catch (Exception ex)
            {
-                if (!CancelToken.IsCancellationRequested)
-                    LogManager.GetCurrentClassLogger().Error(ex);
+                LogManager.GetCurrentClassLogger().Warn("Error while receiving data.", ex);
+
+                Interlocked.Increment(ref _receiveErrors);
+                StartReceiving();
+            }
+        }
+
+        private void ProcessDataLoop()
+        {           
+            while (!_unprocessedRawData.IsCompleted)
+            {
+                try
+                {
+                    ProcessData(_unprocessedRawData.Take());
+                }
+                catch (OperationCanceledException)
+                {
+                    // we are shutting down.
+                    break;
+                }
+                catch (InvalidOperationException)
+                {
+                    // when the collection is marked as completed
+                    break;
+                }
+                catch (Exception ex)
+                {
+                    LogManager.GetCurrentClassLogger().ErrorException("Error while processing data", ex);
+                    Thread.Sleep(100);
+                }
            }

            Finished();
        }
+
+        //
+        // Renames, and AddFields
+        //
+        private void ApplyFilters(JObject json)
+        {      
+            if (_arguments.Renames != null)
+            {
+                for (int i=0; i<_arguments.Renames.Length; i += 2)
+                {
+                    var oldName = ExpandField(_arguments.Renames[i], json);
+                    var newName = ExpandField(_arguments.Renames[i + 1], json);
+                    RenameProperty(json, oldName, newName);
+                }
+            }
+
+            if (_arguments.AddFields != null)
+            {
+                for (int i = 0; i < _arguments.AddFields.Length; i += 2)
+                {
+                    var fieldName = ExpandField(_arguments.AddFields[i], json);
+                    var fieldValue = ExpandField(_arguments.AddFields[i + 1], json);
+                    AddOrModify(json, fieldName, fieldValue);
+                }
+            }
+        }
+
+        private void ProcessData(byte[] bytes)
+        {
+            var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
+
+            try
+            {
+                var json = JObject.Parse(data);
+                ApplyFilters(json);
+                ProcessJson(json);
+                _parsedMessages++;
+            }
+            catch (Exception ex)
+            {
+                var jex1 = LogErrors.LogException(string.Format("Invalid JSON: {0}", data), ex);
+                if (jex1 != null)
+                {
+                    ProcessJson(jex1);
+                }
+
+                var msg = string.Format("Bad JSON: {0}", data);
+                LogManager.GetCurrentClassLogger().Warn(msg, ex);
+
+                _parseErrors++;
+            }
+        }
    }
 }
--- a/TimberWinR/Inputs/W3CInputListener.cs
+++ b/TimberWinR/Inputs/W3CInputListener.cs
@@ -23,11 +23,11 @@ namespace TimberWinR.Inputs
    public class W3CInputListener : InputListener
    {
        private readonly int _pollingIntervalInSeconds;
-        private readonly TimberWinR.Parser.W3CLog _arguments;
+        private readonly TimberWinR.Parser.W3CLogParameters _arguments;
        private long _receivedMessages;
        public bool Stop { get; set; }

-        public W3CInputListener(TimberWinR.Parser.W3CLog arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
+        public W3CInputListener(TimberWinR.Parser.W3CLogParameters arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 5)
            : base(cancelToken, "Win32-W3CLog")
        {
            _arguments = arguments;
@@ -153,7 +153,7 @@ namespace TimberWinR.Inputs
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                        }
-                        catch (OperationCanceledException oce)
+                        catch (OperationCanceledException)
                        {
                            break;
                        }
--- a/TimberWinR/Inputs/WindowsEvtInputListener.cs
+++ b/TimberWinR/Inputs/WindowsEvtInputListener.cs
@@ -1,17 +1,10 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Runtime.InteropServices;
-using System.Security.AccessControl;
-using System.Text;
 using System.Threading;
-using System.Threading.Tasks;
-using System.IO;
-using Interop.MSUtil;
-
 using Newtonsoft.Json.Linq;
-using Newtonsoft.Json.Serialization;
 using NLog;
+using TimberWinR.Parser;
 using LogQuery = Interop.MSUtil.LogQueryClassClass;
 using EventLogInputFormat = Interop.MSUtil.COMEventLogInputContextClassClass;
 using LogRecordSet = Interop.MSUtil.ILogRecordset;
@@ -23,13 +16,13 @@ namespace TimberWinR.Inputs
    /// </summary>
    public class WindowsEvtInputListener : InputListener
    {
-        private int _pollingIntervalInSeconds = 1;
-        private TimberWinR.Parser.WindowsEvent _arguments;
+        private readonly int _pollingIntervalInSeconds = 1;
+        private readonly WindowsEvent _arguments;
        private long _receivedMessages;
-        private List<Thread> _tasks { get; set; }
+        private readonly List<Thread> _tasks;
        public bool Stop { get; set; }

-        public WindowsEvtInputListener(TimberWinR.Parser.WindowsEvent arguments, CancellationToken cancelToken)
+        public WindowsEvtInputListener(WindowsEvent arguments, CancellationToken cancelToken)
            : base(cancelToken, "Win32-Eventlog")
        {
            _arguments = arguments;
@@ -38,8 +31,7 @@ namespace TimberWinR.Inputs

            foreach (string eventHive in _arguments.Source.Split(','))
            {
-                string hive = eventHive.Trim();
-                var thread = new Thread(new ParameterizedThreadStart(EventWatcher));
+                var thread = new Thread(EventWatcher) {Name = "Win32-Eventlog-" + eventHive};
                _tasks.Add(thread);
                thread.Start(eventHive);
            }
@@ -49,6 +41,10 @@ namespace TimberWinR.Inputs
        {
            Stop = true;
            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            foreach (var thread in _tasks)
+            {
+                thread.Join();
+            }
            base.Shutdown();
        }

@@ -76,8 +72,6 @@ namespace TimberWinR.Inputs
        {
            string location = ploc.ToString();

-            LogQuery oLogQuery = new LogQuery();
-
            LogManager.GetCurrentClassLogger().Info("WindowsEvent Input Listener Ready");

            // Instantiate the Event Log Input Format object
@@ -93,9 +87,7 @@ namespace TimberWinR.Inputs
                resolveSIDs = _arguments.ResolveSIDS
            };

-            oLogQuery = null;
-
-            Dictionary<string, Int64> logFileMaxRecords = new Dictionary<string, Int64>();
+            var logFileMaxRecords = new Dictionary<string, Int64>();

            using (var syncHandle = new ManualResetEventSlim())
            {
@@ -107,7 +99,7 @@ namespace TimberWinR.Inputs
                    {
                        try
                        {                          
-                            oLogQuery = new LogQuery();
+                            var oLogQuery = new LogQuery();

                            var qfiles = string.Format("SELECT Distinct [EventLog] FROM {0}", location);
                            var rsfiles = oLogQuery.Execute(qfiles, iFmt);
@@ -145,27 +137,25 @@ namespace TimberWinR.Inputs
                                        object v = record.getValue(field.Name);
                                        if (field.Name == "Data")
                                            v = ToPrintable(v.ToString());
+                                        if ((field.Name == "TimeGenerated" || field.Name == "TimeWritten") && field.DataType == typeof (DateTime))
+                                            v = ((DateTime) v).ToUniversalTime();
                                        json.Add(new JProperty(field.Name, v));
                                    }

                                    var lrn = (Int64)record.getValueEx("RecordNumber");
                                    logFileMaxRecords[fileName] = lrn;

-                                    record = null;
                                    ProcessJson(json);
                                    _receivedMessages++;
-                                    json = null;
-
                                }
                                // Close the recordset
                                rs.close();
-                                rs = null;
                                GC.Collect();
                            }
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                        }
-                        catch (OperationCanceledException oce)
+                        catch (OperationCanceledException)
                        {
                            break;
                        }
--- a/TimberWinR/LogErrors.cs
+++ b/TimberWinR/LogErrors.cs
@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+using TimberWinR.Parser;
+
+namespace TimberWinR
+{
+    public class LogErrors
+    {
+        public static JObject LogException(Exception ex)
+        {
+            return LogException("Exception", ex);
+        }
+
+        public static JObject LogException(string errorMessage, Exception ex)
+        {
+            JObject result = new JObject();
+            result["type"] = "TimberWinR-Error";
+            result["ErrorMessage"] = errorMessage;
+            
+            try
+            {  
+                JsonConvert.DefaultSettings = () => new JsonSerializerSettings
+                {
+                    Formatting = Formatting.Indented,
+                    NullValueHandling = NullValueHandling.Ignore
+                };
+
+                var exJson = JObject.Parse(JsonConvert.SerializeObject(ex));
+
+                result.Merge(exJson, new JsonMergeSettings
+                {
+                    MergeArrayHandling = MergeArrayHandling.Replace
+                });
+                return result;
+            }
+            catch (Exception ex1)
+            {
+                result["ErrorMessage"] = ex1.ToString();
+                return result;
+            }           
+        }
+    }
+}
--- a/TimberWinR/Manager.cs
+++ b/TimberWinR/Manager.cs
@@ -23,9 +23,11 @@ namespace TimberWinR
    {
        public Configuration Config { get; set; }
        public List<OutputSender> Outputs { get; set; }       
-        public List<TcpInputListener> Tcps { get; set; }
-        public List<TcpInputListener> Udps { get; set; }
        public List<InputListener> Listeners { get; set; }
+        public bool LiveMonitor { get; set; }      
+
+        public event Action<Configuration> OnConfigurationProcessed;
+
        public DateTime StartedOn { get; set; }
        public string JsonConfig { get; set; }
        public string LogfileDir { get; set; }
@@ -60,11 +62,17 @@ namespace TimberWinR
            Interlocked.Add(ref numMessages, count);
        }

-        public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken)
+        public Manager()
+        {
+            LogsFileDatabase.Manager = this;
+        }
+
+        public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken, bool processConfiguration = true)
        {          
            LogsFileDatabase.Manager = this;

            StartedOn = DateTime.UtcNow;
+            LiveMonitor = liveMonitor;

            var vfi = new FileInfo(jsonConfigFile);

@@ -98,27 +106,20 @@ namespace TimberWinR

            LogManager.GlobalThreshold = LogLevel.FromString(logLevel);

-            LogManager.GetCurrentClassLogger()
-                .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());
-
+            //LogManager.GetCurrentClassLogger()
+            //    .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());

            LogManager.GetCurrentClassLogger()
-                .Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
+                .Info("TimberWinR Version {0}", Assembly.GetEntryAssembly().GetName().Version.ToString());

+            LogManager.GetCurrentClassLogger()
+                .Info("Database Filename: {0}", LogsFileDatabase.Instance.DatabaseFileName);

            try
-            {
-                // Is it a directory?
-                if (Directory.Exists(jsonConfigFile))
-                {
-                    DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
-                    LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
-                    Config = Configuration.FromDirectory(jsonConfigFile);
-                }
-                else
            {
                var fi = new FileInfo(jsonConfigFile);
-
+                if (fi.Exists)
+                {
                    LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From File: {0}", fi.FullName);

                    if (!fi.Exists)
@@ -127,6 +128,12 @@ namespace TimberWinR
                    LogManager.GetCurrentClassLogger().Info("Initialized, Reading Config: {0}", fi.FullName);
                    Config = Configuration.FromFile(jsonConfigFile);
                }
+                else if (Directory.Exists(jsonConfigFile))
+                {
+                    DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
+                    LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
+                    Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
+                }
            }
            catch (JsonSerializationException jse)
            {
@@ -139,36 +146,69 @@ namespace TimberWinR
            LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
            LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);

+            if (processConfiguration)
+            {
+                ProcessConfiguration(cancelToken, Config);
+            }         
+        }
+
+        public void Start(CancellationToken cancelToken)
+        {
+            ProcessConfiguration(cancelToken, Config);
+        }
+
+        public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
+        {
            // Read the Configuration file
-            if (Config != null)
+            if (config != null)
            {                
-                if (Config.RedisOutputs != null)
+                if (OnConfigurationProcessed != null)
+                    OnConfigurationProcessed(config);
+
+                if (config.StatsDOutputs != null)
                {
-                    foreach (var ro in Config.RedisOutputs)
+                    foreach (var ro in config.StatsDOutputs)
+                    {
+                        var output = new StatsDOutput(this, ro, cancelToken);
+                        Outputs.Add(output);
+                    }
+                }
+
+                if (config.RedisOutputs != null)
+                {
+                    foreach (var ro in config.RedisOutputs)
                    {
                        var redis = new RedisOutput(this, ro, cancelToken);
                        Outputs.Add(redis);
                    }
-
                }
-                if (Config.ElasticsearchOutputs != null)
+                if (config.ElasticsearchOutputs != null)
                {
-                    foreach (var ro in Config.ElasticsearchOutputs)
+                    foreach (var ro in config.ElasticsearchOutputs)
                    {
                        var els = new ElasticsearchOutput(this, ro, cancelToken);
                        Outputs.Add(els);
                    }
                }
-                if (Config.StdoutOutputs != null)
+                if (config.StdoutOutputs != null)
                {
-                    foreach (var ro in Config.StdoutOutputs)
+                    foreach (var ro in config.StdoutOutputs)
                    {
                        var stdout = new StdoutOutput(this, ro, cancelToken);
                        Outputs.Add(stdout);
                    }
                }

-                foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C)
+                if (config.FileOutputs != null)
+                {
+                    foreach (var ro in config.FileOutputs)
+                    {
+                        var output = new FileOutput(this, ro, cancelToken);
+                        Outputs.Add(output);
+                    }
+                }
+
+                foreach (Parser.IISW3CLogParameters iisw3cConfig in config.IISW3C)
                {
                    var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
                    Listeners.Add(elistner);
@@ -176,7 +216,7 @@ namespace TimberWinR
                        output.Connect(elistner);
                }

-                foreach (Parser.W3CLog iisw3cConfig in Config.W3C)
+                foreach (Parser.W3CLogParameters iisw3cConfig in config.W3C)
                {
                    var elistner = new W3CInputListener(iisw3cConfig, cancelToken);
                    Listeners.Add(elistner);
@@ -184,7 +224,7 @@ namespace TimberWinR
                        output.Connect(elistner);
                }

-                foreach (Parser.WindowsEvent eventConfig in Config.Events)
+                foreach (Parser.WindowsEvent eventConfig in config.Events)
                {
                    var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);
                    Listeners.Add(elistner);
@@ -192,7 +232,7 @@ namespace TimberWinR
                        output.Connect(elistner);
                }

-                foreach (var logConfig in Config.Logs)
+                foreach (var logConfig in config.Logs)
                {
                    var elistner = new LogsListener(logConfig, cancelToken);
                    Listeners.Add(elistner);
@@ -200,23 +240,31 @@ namespace TimberWinR
                        output.Connect(elistner);
                }

-                foreach (var tcp in Config.Tcps)
+                foreach (var logConfig in config.TailFiles)
                {
-                    var elistner = new TcpInputListener(cancelToken, tcp.Port);
+                    var elistner = new TailFileListener(logConfig, cancelToken);
                    Listeners.Add(elistner);
                    foreach (var output in Outputs)
                        output.Connect(elistner);
                }

-                foreach (var udp in Config.Udps)
+                foreach (var tcp in config.Tcps)
                {
-                    var elistner = new UdpInputListener(cancelToken, udp.Port);
+                    var elistner = new TcpInputListener(tcp, cancelToken, tcp.Port);
                    Listeners.Add(elistner);
                    foreach (var output in Outputs)
                        output.Connect(elistner);
                }

-                foreach (var stdin in Config.Stdins)
+                foreach (var udp in config.Udps)
+                {
+                    var elistner = new UdpInputListener(udp, cancelToken, udp.Port);
+                    Listeners.Add(elistner);
+                    foreach (var output in Outputs)
+                        output.Connect(elistner);
+                }
+
+                foreach (var stdin in config.Stdins)
                {
                    var elistner = new StdinListener(stdin, cancelToken);
                    Listeners.Add(elistner);
@@ -224,6 +272,15 @@ namespace TimberWinR
                        output.Connect(elistner);
                }

+                foreach (var stdin in config.Generators)
+                {
+                    var elistner = new GeneratorInput(stdin, cancelToken);
+                    Listeners.Add(elistner);
+                    foreach (var output in Outputs)
+                        output.Connect(elistner);
+                }
+
+
                var computerName = System.Environment.MachineName + "." +
                                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
                                       @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
@@ -236,7 +293,9 @@ namespace TimberWinR
                    JObject json = new JObject(
                        new JProperty("TimberWinR",
                            new JObject(
-                             new JProperty("version", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+                                new JProperty("version",
+                                    Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                        //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
                                new JProperty("host", computerName),
                                new JProperty("output", output.Name),
                                new JProperty("initialized", DateTime.UtcNow)
@@ -246,7 +305,6 @@ namespace TimberWinR
                    output.Startup(json);
                }               
            }
-
        }


--- a/TimberWinR/Outputs/Elasticsearch.cs
+++ b/TimberWinR/Outputs/Elasticsearch.cs
@@ -2,52 +2,113 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
+using System.Net.Sockets;
+using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
+using Elasticsearch.Net;
+using Elasticsearch.Net.ConnectionPool;
+using Nest;
 using Newtonsoft.Json.Linq;
 using NLog;
 using RapidRegex.Core;
 using RestSharp;
+using System.Text.RegularExpressions;
+using Elasticsearch.Net.Serialization;
+using Newtonsoft.Json;

 namespace TimberWinR.Outputs
 {
-    using System.Text.RegularExpressions;
+    public class Person
+    {
+        public string Firstname { get; set; }
+        public string Lastname { get; set; }
+    }

    public partial class ElasticsearchOutput : OutputSender
    {
        private TimberWinR.Manager _manager;
        private readonly int _port;
+        private readonly bool _ssl;
+        private readonly string _username;
+        private readonly string _password;
        private readonly int _interval;
-        private readonly string[] _host;
+        private readonly int _flushSize;
+        private readonly int _idleFlushTimeSeconds;
+        private readonly string[] _hosts;
        private readonly string _protocol;      
-        private int _hostIndex;
        private readonly int _timeout;
        private readonly object _locker = new object();
        private readonly List<JObject> _jsonQueue;
        private readonly int _numThreads;
        private long _sentMessages;
        private long _errorCount;
-        private Parser.ElasticsearchOutput eo;
+        private readonly int _maxQueueSize;
+        private readonly bool _queueOverflowDiscardOldest;
+        private readonly bool _disablePing;
+        private readonly int _pingTimeout;
+
+        private Parser.ElasticsearchOutputParameters _parameters;
        public bool Stop { get; set; }
      
-        public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken)
+        /// <summary>
+        /// Get the bulk connection pool of hosts
+        /// </summary>
+        /// <returns></returns>
+        private ElasticClient getClient()
+        {
+            var nodes = new List<Uri>();
+            foreach (var host in _hosts)
+            {
+                var uri = ComposeUri(host, _port, _ssl, _username, _password);
+                nodes.Add(uri);
+            }
+            var pool = new StaticConnectionPool(nodes.ToArray());
+            var settings = new ConnectionSettings(pool)
+                .ExposeRawResponse();
+
+            if (_disablePing)
+                settings.DisablePing();
+            else if (_pingTimeout != 0)
+                settings.SetPingTimeout(_pingTimeout);
+
+            var client = new ElasticClient(settings);
+            return client;
+        }
+
+        public static Uri ComposeUri(string host, int port, bool ssl, string username, string password)
+        {
+            return ssl
+                ? new Uri(string.Format("https://{0}:{1}@{2}:{3}", Uri.EscapeDataString(username), Uri.EscapeDataString(password), host, port))
+                : new Uri(string.Format("http://{0}:{1}", host, port));
+        }
+
+        public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutputParameters parameters, CancellationToken cancelToken)
            : base(cancelToken, "Elasticsearch")
        {
            _sentMessages = 0;
            _errorCount = 0;
           
-            this.eo = eo;
-            _protocol = eo.Protocol;
-            _timeout = eo.Timeout;
+            _parameters = parameters;
+            _flushSize = parameters.FlushSize;
+            _idleFlushTimeSeconds = parameters.IdleFlushTimeInSeconds;
+            _protocol = parameters.Protocol;
+            _timeout = parameters.Timeout;
            _manager = manager;
-            _port = eo.Port;
-            _interval = eo.Interval;
-            _host = eo.Host;
-            _hostIndex = 0;
+            _port = parameters.Port;
+            _ssl = parameters.Ssl;
+            _username = parameters.Username;
+            _password = parameters.Password;
+            _interval = parameters.Interval;
+            _hosts = parameters.Host;          
            _jsonQueue = new List<JObject>();
-            _numThreads = eo.NumThreads;
+            _numThreads = parameters.NumThreads;
+            _maxQueueSize = parameters.MaxQueueSize;
+            _queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
+            _disablePing = !parameters.EnablePing;
+            _pingTimeout = parameters.PingTimeout;

-            for (int i = 0; i < eo.NumThreads; i++)
+            for (int i = 0; i < parameters.NumThreads; i++)
            {
                Task.Factory.StartNew(ElasticsearchSender, cancelToken, TaskCreationOptions.LongRunning, TaskScheduler.Current);
            }
@@ -58,16 +119,23 @@ namespace TimberWinR.Outputs
            JObject json = new JObject(
                new JProperty("elasticsearch",
                    new JObject(
-                        new JProperty("host", string.Join(",", _host)),
+                        new JProperty("host", string.Join(",", _hosts)),
                        new JProperty("errors", _errorCount),
-                        new JProperty("sent_messages", _sentMessages),
-                        new JProperty("queued_messages", _jsonQueue.Count),
+                        new JProperty("messages", _sentMessages),
+                        new JProperty("queuedMessageCount", _jsonQueue.Count),
                        new JProperty("port", _port),
+                        new JProperty("ssl", _ssl),
+                        new JProperty("username", _username),
+                        new JProperty("password", _password),
+                        new JProperty("flushSize", _flushSize),                                       
+                        new JProperty("idleFlushTime", _idleFlushTimeSeconds),     
                        new JProperty("interval", _interval),
                        new JProperty("threads", _numThreads),
+                        new JProperty("maxQueueSize", _maxQueueSize),
+                        new JProperty("overflowDiscardOldest", _queueOverflowDiscardOldest),               
                        new JProperty("hosts",
                            new JArray(
-                                from h in _host
+                                from h in _hosts
                                select new JObject(
                                    new JProperty("host", h)))))));
            return json;
@@ -77,6 +145,13 @@ namespace TimberWinR.Outputs
        // 
        private void ElasticsearchSender()
        {
+            // Force an inital flush
+            DateTime lastFlushTime = DateTime.MinValue;
+
+            LogManager.GetCurrentClassLogger()
+                            .Info("{0}: Elasticsarch Output To {1} Ready", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));
+         
+
            using (var syncHandle = new ManualResetEventSlim())
            {               
                // Execute the query
@@ -86,82 +161,59 @@ namespace TimberWinR.Outputs
                    {
                        try
                        {                           
-                            JObject[] messages;
+                            int messageCount = 0;
+                            List<JObject> messages = new List<JObject>();
+                            
+                            // Lets get whats in the queue
                            lock (_locker)
                            {
-                                var count = _jsonQueue.Count;
-                                messages = _jsonQueue.Take(count).ToArray();
-                                _jsonQueue.RemoveRange(0, count);
-                                if (messages.Length > 0)
-                                    _manager.IncrementMessageCount(messages.Length);
+                                messageCount = _jsonQueue.Count;
+                                
+                                // Time to flush?                             
+                                if (messageCount >= _flushSize || (DateTime.UtcNow - lastFlushTime).Seconds >= _idleFlushTimeSeconds)
+                                {
+                                    messages = _jsonQueue.Take(messageCount).ToList();
+                                    _jsonQueue.RemoveRange(0, messageCount);
+                                    if (messages.Count > 0)
+                                        _manager.IncrementMessageCount(messages.Count);
+                                }
                            }

-                            if (messages.Length > 0)
+                            // We have some messages to work with
+                            if (messages.Count > 0)
                            {
-                                int numHosts = _host.Length;
-                                while (numHosts-- > 0)
+                                var client = getClient();
+
+                                LogManager.GetCurrentClassLogger()
+                                    .Debug("Sending {0} Messages to {1}", messages.Count, string.Join(",", _hosts));
+                                // This loop will process all messages we've taken from the queue
+                                // that have the same index and type (an elasticsearch requirement)
+                                do
                                {
                                    try
                                    {
-                                        // Get the next client
-                                        RestClient client = getClient();
-                                        if (client != null)
-                                        {
-                                            LogManager.GetCurrentClassLogger()
-                                                .Debug("Sending {0} Messages to {1}", messages.Length, client.BaseUrl);
+                                        // Grab all messages with same index and type (this is the whole point, group the same ones) 
+                                        var bulkTypeName = this._parameters.GetTypeName(messages[0]);
+                                        var bulkIndexName = this._parameters.GetIndexName(messages[0]);

-                                            foreach (JObject json in messages)
-                                            {
-                                                var typeName = this.eo.GetTypeName(json);
-                                                var indexName = this.eo.GetIndexName(json);
-                                                var req =
-                                                    new RestRequest(string.Format("/{0}/{1}/", indexName, typeName),
-                                                        Method.POST);
+                                        IEnumerable<JObject> bulkItems =
+                                            messages.TakeWhile(
+                                                message =>
+                                                    String.Compare(bulkTypeName, _parameters.GetTypeName(message), false) == 0 &&
+                                                    String.Compare(bulkIndexName, _parameters.GetIndexName(message), false) == 0);

-                                                req.AddParameter("text/json", json.ToString(), ParameterType.RequestBody);
+                                        // Send the message(s), if the are successfully sent, they
+                                        // are removed from the queue
+                                        lastFlushTime = transmitBulkData(bulkItems, bulkIndexName, bulkTypeName, client, lastFlushTime, messages);
                                        
-                                                req.RequestFormat = DataFormat.Json;
-
-                                                try
-                                                {
-                                                    client.ExecuteAsync(req, response =>
-                                                    {
-                                                        if (response.StatusCode != HttpStatusCode.Created)
-                                                        {
-                                                            LogManager.GetCurrentClassLogger()
-                                                                .Error("Failed to send: {0}", response.ErrorMessage);
-                                                            Interlocked.Increment(ref _errorCount);
-                                                        }
-                                                        else
-                                                        {
-                                                            _sentMessages++;
                                        GC.Collect();
                                    }
-                                                    });
-                                                }
-                                                catch (Exception error)
-                                                {
-                                                    LogManager.GetCurrentClassLogger().Error(error);
-                                                    Interlocked.Increment(ref _errorCount);
-                                                }
-                                            }
-                                            GC.Collect();
-                                        }
-                                        else
-                                        {
-                                            LogManager.GetCurrentClassLogger()
-                                                .Fatal("Unable to connect with any Elasticsearch hosts, {0}",
-                                                    String.Join(",", _host));
-                                            Interlocked.Increment(ref _errorCount);
-                                        }
-
-                                    }
                                    catch (Exception ex)
                                    {
                                        LogManager.GetCurrentClassLogger().Error(ex);
-                                        Interlocked.Increment(ref _errorCount);
-                                    }
+                                        break;
                                    }
+                                } while (messages.Count > 0);
                            }
                            GC.Collect();
                            if (!Stop)
@@ -169,45 +221,77 @@ namespace TimberWinR.Outputs
                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                            }
                        }
-                        catch (OperationCanceledException oce)
+                        catch (OperationCanceledException)
                        {
                            break;
                        }
-                        catch (Exception)
+                        catch (Exception ex)
                        {
-                            throw;
+                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                    }
                }
            }
-        }
-        private RestClient getClient()
-        {
-            if (_hostIndex >= _host.Length)
-                _hostIndex = 0;
-
-            int numTries = 0;
-            while (numTries < _host.Length)
-            {
-                try
-                {
-                    string url = string.Format("{0}://{1}:{2}", _protocol.Replace(":", ""), _host[_hostIndex], _port);
-                    var client = new RestClient(url);
-                    client.Timeout = _timeout;
-
-                    _hostIndex++;
-                    if (_hostIndex >= _host.Length)
-                        _hostIndex = 0;
-
-                    return client;
-                }
-                catch (Exception)
-                {
-                }
-                numTries++;
-            }

-            return null;
+            LogManager.GetCurrentClassLogger()
+                          .Info("{0}: Elasticsarch Output To {1} Terminated", Thread.CurrentThread.ManagedThreadId, string.Join(",", _hosts));         
+
+        }
+
+        //
+        // Send the messages to Elasticsearch (bulk)
+        //
+        private DateTime transmitBulkData(IEnumerable<JObject> bulkItems, string bulkIndexName, string bulkTypeName,
+            ElasticClient client, DateTime lastFlushTime, List<JObject> messages)
+        {
+            var bulkRequest = new BulkRequest() {Refresh = true};
+            bulkRequest.Operations = new List<IBulkOperation>();
+            foreach (var json in bulkItems)
+            {
+                // ES requires a timestamp, add one if not present
+                var ts = json["@timestamp"];
+                if (ts == null)
+                    json["@timestamp"] = DateTime.UtcNow;
+                var bi = new BulkIndexOperation<JObject>(json);
+                bi.Index = bulkIndexName;
+                bi.Type = bulkTypeName;
+                bulkRequest.Operations.Add(bi);
+            }
+
+            // The total messages processed for this operation.
+            int numMessages = bulkItems.Count();
+     
+            var response = client.Bulk(bulkRequest);
+            if (!response.IsValid)
+            {
+                LogManager.GetCurrentClassLogger().Error("Failed to send: {0}", response);
+                Interlocked.Increment(ref _errorCount);
+                interlockedInsert(messages);  // Put the messages back into the queue
+            }
+            else // Success!
+            {
+                lastFlushTime = DateTime.UtcNow;
+                LogManager.GetCurrentClassLogger()
+                    .Info("Successfully sent {0} messages in a single bulk request", numMessages);
+                Interlocked.Add(ref _sentMessages, numMessages);                              
+            }
+
+            // Remove them from the working list
+            messages.RemoveRange(0, numMessages);
+            return lastFlushTime;
+        }
+
+        // Places messages back into the queue (for a future attempt)
+        private void interlockedInsert(List<JObject> messages)
+        {
+            lock (_locker)
+            {
+                _jsonQueue.InsertRange(0, messages);
+                if (_jsonQueue.Count > _maxQueueSize)
+                {
+                    LogManager.GetCurrentClassLogger().Warn("Exceeded maximum queue depth");
+                }
+            }
        }


@@ -217,10 +301,30 @@ namespace TimberWinR.Outputs
                ApplyFilters(jsonMessage);

            var message = jsonMessage.ToString();
-            LogManager.GetCurrentClassLogger().Debug(message);
+            LogManager.GetCurrentClassLogger().Trace(message);

            lock (_locker)
            {
+                if (_jsonQueue.Count >= _maxQueueSize)
+                {
+                    // If we've exceeded our queue size, and we're supposed to throw out the oldest objects first,
+                    // then remove as many as necessary to get us under our limit
+                    if (_queueOverflowDiscardOldest)
+                    {
+                        LogManager.GetCurrentClassLogger()
+                            .Warn("Overflow discarding oldest {0} messages", _jsonQueue.Count - _maxQueueSize + 1);
+
+                        _jsonQueue.RemoveRange(0, (_jsonQueue.Count - _maxQueueSize) + 1);
+                    }
+                    // Otherwise we're in a "discard newest" mode, and this is the newest message, so just ignore it
+                    else
+                    {
+                        LogManager.GetCurrentClassLogger()
+                            .Warn("Overflow discarding newest message: {0}", message);
+
+                        return;
+                    }
+                }
                _jsonQueue.Add(jsonMessage);
            }
        }
--- a/TimberWinR/Outputs/File.cs
+++ b/TimberWinR/Outputs/File.cs
@@ -0,0 +1,140 @@
+using System.IO;
+using System.Text;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace TimberWinR.Outputs
+{
+    public class FileOutput : OutputSender
+    {
+        private TimberWinR.Manager _manager;
+        private readonly int _interval;
+        private readonly object _locker = new object();
+        private readonly List<JObject> _jsonQueue;
+        private long _sentMessages;     
+        private Parser.FileOutputParameters _arguments;
+        public bool Stop { get; set; }
+
+        public FileOutput(TimberWinR.Manager manager, Parser.FileOutputParameters arguments, CancellationToken cancelToken)
+            : base(cancelToken, "File")
+        {
+            _arguments = arguments;
+            _sentMessages = 0;
+            _manager = manager;
+            _interval = arguments.Interval;
+            _jsonQueue = new List<JObject>();
+
+            var elsThread = new Task(FileSender, cancelToken);
+            elsThread.Start();
+        }
+
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("file-output",
+                    new JObject(
+                        new JProperty("queuedMessageCount", _jsonQueue.Count),
+                        new JProperty("sentMessageCount", _sentMessages))));
+
+            return json;
+        }
+
+        // 
+        // Pull off messages from the Queue, batch them up and send them all across
+        // 
+        private void FileSender()
+        {
+           
+            using (var syncHandle = new ManualResetEventSlim())
+            {
+                var fi = new FileInfo(_arguments.FileName);
+                if (File.Exists(_arguments.FileName))
+                    File.Delete(_arguments.FileName);
+
+                LogManager.GetCurrentClassLogger().Info("File Output Sending To: {0}", fi.FullName);
+       
+                using (StreamWriter sw = File.AppendText(_arguments.FileName))
+                {
+                    // Execute the query
+                    while (!Stop)
+                    {
+                        if (!CancelToken.IsCancellationRequested)
+                        {
+                            try
+                            {
+                                JObject[] messages;
+                                lock (_locker)
+                                {
+                                    messages = _jsonQueue.Take(_jsonQueue.Count).ToArray();
+                                    _jsonQueue.RemoveRange(0, messages.Length);
+                                }
+
+                                if (messages.Length > 0)
+                                {
+                                    try
+                                    {
+                                        foreach (JObject obj in messages)
+                                        {                                           
+                                            sw.WriteLine(obj.ToString(_arguments.ToFormat()));                                        
+                                            _sentMessages++;
+                                        }
+                                    }
+                                    catch (Exception ex)
+                                    {
+                                        LogManager.GetCurrentClassLogger().Error(ex);
+                                    }
+                                }
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
+                            }
+                            catch (OperationCanceledException)
+                            {
+                                break;
+                            }
+                            catch (Exception)
+                            {
+                            }
+                        }
+                    }
+                }
+            }
+        }
+
+        protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage)
+        {
+            if (_manager.Config.Filters != null)
+            {
+                if (ApplyFilters(jsonMessage))
+                    return;
+            }
+
+            var message = jsonMessage.ToString();
+   
+            lock (_locker)
+            {
+                _jsonQueue.Add(jsonMessage);
+            }
+        }
+
+        private bool ApplyFilters(JObject json)
+        {
+            bool drop = false;
+
+            foreach (var filter in _manager.Config.Filters)
+            {
+                if (!filter.Apply(json))
+                    drop = true;
+            }
+
+            return drop;
+        }
+
+    }
+}
+
--- a/TimberWinR/Outputs/Redis.cs
+++ b/TimberWinR/Outputs/Redis.cs
@@ -1,11 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Net.Sockets;
 using System.Text;
 using System.Threading;
-using ctstone.Redis;
+using CSRedis;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
@@ -17,32 +18,128 @@ using TimberWinR.Parser;

 namespace TimberWinR.Outputs
 {
+    internal class BatchCounter
+    {
+        // Total number of times reached max batch count (indicates we are under pressure)
+        public int ReachedMaxBatchCountTimes { get; set; }
+
+        private readonly int[] _sampleQueueDepths;
+        private int _sampleCountIndex;
+        private const int QUEUE_SAMPLE_SIZE = 30; // 30 samples over 2.5 minutes (default)
+        private object _locker = new object();
+        private bool _warnedReachedMax;
+
+        private readonly int _maxBatchCount;
+        private readonly int _batchCount;
+        private int _totalSamples;
+
+        public int[] Samples()
+        {
+            return _sampleQueueDepths;
+        }
+
+        public BatchCounter(int batchCount, int maxBatchCount)
+        {
+            _batchCount = batchCount;
+            _maxBatchCount = maxBatchCount;
+            _sampleQueueDepths = new int[QUEUE_SAMPLE_SIZE];
+            _sampleCountIndex = 0;
+            _totalSamples = 0;
+            ReachedMaxBatchCountTimes = 0;
+        }
+        public void SampleQueueDepth(int queueDepth)
+        {
+            lock (_locker)
+            {
+                if (_totalSamples < QUEUE_SAMPLE_SIZE)
+                    _totalSamples++;
+
+                // Take a sample of the queue depth
+                if (_sampleCountIndex >= QUEUE_SAMPLE_SIZE)
+                    _sampleCountIndex = 0;
+
+                _sampleQueueDepths[_sampleCountIndex++] = queueDepth;
+            }
+        }
+
+        public int AverageQueueDepth()
+        {
+            lock (_locker)
+            {
+                if (_totalSamples > 0)
+                {
+                    var samples = _sampleQueueDepths.Take(_totalSamples);
+                    int avg = (int)samples.Average();
+                    return avg;
+                }
+                return 0;
+            }
+        }
+
+        // Sample the queue and adjust the batch count if needed (ramp up slowly)
+        public int UpdateCurrentBatchCount(int queueSize, int currentBatchCount)
+        {
+            var avgQueueDepth = AverageQueueDepth();
+        
+            if (currentBatchCount < _maxBatchCount && currentBatchCount < queueSize && avgQueueDepth > currentBatchCount)
+            {
+                currentBatchCount += Math.Max(avgQueueDepth / _batchCount, _batchCount / 5);
+                if (currentBatchCount >= _maxBatchCount && !_warnedReachedMax)
+                {
+                    LogManager.GetCurrentClassLogger().Warn("Maximum Batch Count of {0} reached.", currentBatchCount);
+                    _warnedReachedMax = true; // Only complain when it's reached (1 time, unless reset)
+                    ReachedMaxBatchCountTimes++;
+                    currentBatchCount = _maxBatchCount;
+                }
+            }
+            else // Reset to default
+            {
+                currentBatchCount = _batchCount;
+                _warnedReachedMax = false;
+            }
+
+            return currentBatchCount;
+        }
+    }
+
+
    public class RedisOutput : OutputSender
    {
+        public int QueueDepth
+        {
+            get { return _jsonQueue.Count; }
+        }
+
+        public long SentMessages
+        {
+            get { return _sentMessages; }
+        }
+
        private readonly string _logstashIndexName;
        private readonly int _port;
        private readonly int _timeout;
        private readonly object _locker = new object();
        private readonly List<string> _jsonQueue;
-        // readonly Task _consumerTask;
        private readonly string[] _redisHosts;
        private int _redisHostIndex;
        private TimberWinR.Manager _manager;
        private readonly int _batchCount;
+        private int _currentBatchCount;
+        private readonly int _maxBatchCount;
        private readonly int _interval;
        private readonly int _numThreads;
-
        private long _sentMessages;
        private long _errorCount;
        private long _redisDepth;
-
-        private int _maxQueueSize;
-        private bool _queueOverflowDiscardOldest;
+        private DateTime? _lastErrorTimeUTC;
+        private readonly int _maxQueueSize;
+        private readonly bool _queueOverflowDiscardOldest;
+        private BatchCounter _batchCounter;

        public bool Stop { get; set; }

        /// <summary>
-        /// Get the next client
+        /// Get the next client from the list of hosts.
        /// </summary>
        /// <returns></returns>
        private RedisClient getClient()
@@ -55,16 +152,19 @@ namespace TimberWinR.Outputs
            {
                try
                {
-                    RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port, _timeout);
+                    RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port);                   
+                    return client;
+                }
+                catch (Exception ex)
+                {
+                    LogManager.GetCurrentClassLogger().Error(ex);

+                }
+                finally
+                {
                    _redisHostIndex++;
                    if (_redisHostIndex >= _redisHosts.Length)
                        _redisHostIndex = 0;
-
-                    return client;
-                }
-                catch (Exception)
-                {
                }
                numTries++;
            }
@@ -79,15 +179,21 @@ namespace TimberWinR.Outputs
                    new JObject(
                        new JProperty("host", string.Join(",", _redisHosts)),
                        new JProperty("errors", _errorCount),
-                        new JProperty("redis_depth", _redisDepth),
-                        new JProperty("sent_messages", _sentMessages),
-                        new JProperty("queued_messages", _jsonQueue.Count),
+                        new JProperty("lastErrorTimeUTC", _lastErrorTimeUTC),
+                        new JProperty("redisQueueDepth", _redisDepth),
+                        new JProperty("sentMessageCount", _sentMessages),
+                        new JProperty("queuedMessageCount", _jsonQueue.Count),
                        new JProperty("port", _port),
                        new JProperty("maxQueueSize", _maxQueueSize),
                        new JProperty("overflowDiscardOldest", _queueOverflowDiscardOldest),
                        new JProperty("interval", _interval),
                        new JProperty("threads", _numThreads),
                        new JProperty("batchcount", _batchCount),
+                        new JProperty("currentBatchCount", _currentBatchCount),
+                        new JProperty("reachedMaxBatchCountTimes", _batchCounter.ReachedMaxBatchCountTimes),
+                        new JProperty("maxBatchCount", _maxBatchCount),
+                        new JProperty("averageQueueDepth", _batchCounter.AverageQueueDepth()),
+                        new JProperty("queueSamples", new JArray(_batchCounter.Samples())),
                        new JProperty("index", _logstashIndexName),
                        new JProperty("hosts",
                            new JArray(
@@ -97,25 +203,33 @@ namespace TimberWinR.Outputs
            return json;
        }

-        public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken)
+        public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutputParameters parameters, CancellationToken cancelToken)
            : base(cancelToken, "Redis")
        {
            _redisDepth = 0;
-            _batchCount = ro.BatchCount;
+            _batchCount = parameters.BatchCount;
+            _maxBatchCount = parameters.MaxBatchCount;
+            // Make sure maxBatchCount is larger than batchCount
+            if (_maxBatchCount <= _batchCount)
+                _maxBatchCount = _batchCount * 10;
+
            _manager = manager;
            _redisHostIndex = 0;
-            _redisHosts = ro.Host;
+            _redisHosts = parameters.Host;
            _jsonQueue = new List<string>();
-            _port = ro.Port;
-            _timeout = ro.Timeout;
-            _logstashIndexName = ro.Index;
-            _interval = ro.Interval;
-            _numThreads = ro.NumThreads;
+            _port = parameters.Port;
+            _timeout = parameters.Timeout;
+            _logstashIndexName = parameters.Index;
+            _interval = parameters.Interval;
+            _numThreads = parameters.NumThreads;
            _errorCount = 0;
-            _maxQueueSize = ro.MaxQueueSize;
-            _queueOverflowDiscardOldest = ro.QueueOverflowDiscardOldest;
+            _lastErrorTimeUTC = null;
+            _maxQueueSize = parameters.MaxQueueSize;
+            _queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
+            _batchCounter = new BatchCounter(_batchCount, _maxBatchCount);
+            _currentBatchCount = _batchCount;

-            for (int i = 0; i < ro.NumThreads; i++)
+            for (int i = 0; i < parameters.NumThreads; i++)
            {
                var redisThread = new Task(RedisSender, cancelToken);
                redisThread.Start();
@@ -140,7 +254,7 @@ namespace TimberWinR.Outputs
            }

            var message = jsonMessage.ToString();
-            LogManager.GetCurrentClassLogger().Debug(message);
+            LogManager.GetCurrentClassLogger().Trace(message);

            lock (_locker)
            {
@@ -176,13 +290,12 @@ namespace TimberWinR.Outputs
            {
                if (!filter.Apply(json))
                {
-                    LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
+                    LogManager.GetCurrentClassLogger().Debug("{0}: Dropping: {1}", Thread.CurrentThread.ManagedThreadId, json.ToString());
                    drop = true;
                }
            }
            return drop;
        }
-
        // 
        // Pull off messages from the Queue, batch them up and send them all across
        // 
@@ -198,17 +311,24 @@ namespace TimberWinR.Outputs
                        try
                        {
                            string[] messages;
+                            // Exclusively
                            lock (_locker)
                            {
-                                messages = _jsonQueue.Take(_batchCount).ToArray();
+                                _batchCounter.SampleQueueDepth(_jsonQueue.Count);
+                                // Re-compute current batch size
+
+                                LogManager.GetCurrentClassLogger().Trace("{0}: Average Queue Depth: {1}, Current Length: {2}", Thread.CurrentThread.ManagedThreadId, _batchCounter.AverageQueueDepth(), _jsonQueue.Count); 
+       
+                                _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count, _currentBatchCount);
+
+                                messages = _jsonQueue.Take(_currentBatchCount).ToArray();
                                _jsonQueue.RemoveRange(0, messages.Length);
-                                if (messages.Length > 0)
-                                    _manager.IncrementMessageCount(messages.Length);
                            }

                            if (messages.Length > 0)
                            {
                                int numHosts = _redisHosts.Length;
+                                bool sentSuccessfully = false;
                                while (numHosts-- > 0)
                                {
                                    try
@@ -220,21 +340,28 @@ namespace TimberWinR.Outputs
                                            {
                                                client.StartPipe();
                                                LogManager.GetCurrentClassLogger()
-                                                    .Debug("Sending {0} Messages to {1}", messages.Length, client.Host);
+                                                    .Debug("{0}: Sending {1} Messages to {2}", Thread.CurrentThread.ManagedThreadId, messages.Length, client.Host);

                                                try
                                                {
                                                    _redisDepth = client.RPush(_logstashIndexName, messages);
-                                                    _sentMessages += messages.Length;
+                                                    Interlocked.Add(ref _sentMessages, messages.Length);
+                                                    client.EndPipe();
+                                                    sentSuccessfully = true;
+                                                    if (messages.Length > 0)
+                                                        _manager.IncrementMessageCount(messages.Length);
                                                }
                                                catch (SocketException ex)
                                                {
                                                    LogManager.GetCurrentClassLogger().Warn(ex);
                                                    Interlocked.Increment(ref _errorCount);
+                                                    _lastErrorTimeUTC = DateTime.UtcNow;
                                                }
-                                                finally
+                                                catch (Exception ex)
                                                {
-                                                    client.EndPipe();
+                                                    LogManager.GetCurrentClassLogger().Error(ex);
+                                                    Interlocked.Increment(ref _errorCount);
+                                                    _lastErrorTimeUTC = DateTime.UtcNow;
                                                }
                                                break;
                                            }
@@ -244,6 +371,7 @@ namespace TimberWinR.Outputs
                                                LogManager.GetCurrentClassLogger()
                                                    .Fatal("Unable to connect with any Redis hosts, {0}",
                                                        String.Join(",", _redisHosts));
+                                                _lastErrorTimeUTC = DateTime.UtcNow;
                                            }
                                        }
                                    }
@@ -251,21 +379,35 @@ namespace TimberWinR.Outputs
                                    {
                                        LogManager.GetCurrentClassLogger().Error(ex);
                                        Interlocked.Increment(ref _errorCount);
+                                        _lastErrorTimeUTC = DateTime.UtcNow;
+                                    }
+                                } // No more hosts to try.
+
+                                // Couldn't send, put it back into the queue.
+                                if (!sentSuccessfully)
+                                {
+                                    lock (_locker)
+                                    {
+                                        _jsonQueue.InsertRange(0, messages);
                                    }
                                }
                            }                          
-                            GC.Collect();
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                        }
-                        catch (OperationCanceledException oce)
+                        catch (OperationCanceledException)
+                        {
+                            break;
+                        }
+                        catch (ThreadAbortException)
                        {
                            break;
                        }
                        catch (Exception ex)
                        {
-
-                            throw;
+                            _lastErrorTimeUTC = DateTime.UtcNow;
+                            Interlocked.Increment(ref _errorCount);
+                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                    }
                }
--- a/TimberWinR/Outputs/StatsD.cs
+++ b/TimberWinR/Outputs/StatsD.cs
@@ -0,0 +1,368 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Net.Sockets;
+using System.Text;
+using System.Threading;
+using CSRedis;
+using Nest;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using System.Threading.Tasks;
+using RapidRegex.Core;
+using System.Text.RegularExpressions;
+using System.Globalization;
+using StatsdClient;
+using TimberWinR.Parser;
+
+namespace TimberWinR.Outputs
+{   
+    public class StatsDOutput : OutputSender
+    {
+        public int QueueDepth
+        {
+            get { return _jsonQueue.Count; }
+        }
+
+        public long SentMessages
+        {
+            get { return _sentMessages; }
+        }
+
+       
+        private readonly int _port;
+        public string _host { get; set; }
+
+        private readonly int _interval;
+        private readonly object _locker = new object();
+        private readonly List<JObject> _jsonQueue;
+        private TimberWinR.Manager _manager;        
+        private long _sentMessages;
+        private long _errorCount;
+        private readonly int _maxQueueSize;
+        private readonly bool _queueOverflowDiscardOldest;
+        private readonly int _flushSize;
+        private readonly int _idleFlushTimeSeconds;
+        private readonly int _numThreads;
+        private Parser.StatsDOutputParameters _params;
+
+        public bool Stop { get; set; }
+       
+        public override JObject ToJson()
+        {
+            var json = new JObject(
+                new JProperty("statsd",
+                    new JObject(                       
+                        new JProperty("errors", _errorCount),                       
+                        new JProperty("sentMessageCount", _sentMessages),
+                        new JProperty("queuedMessageCount", _jsonQueue.Count),
+                        new JProperty("port", _port),
+                        new JProperty("threads", _numThreads),
+                        new JProperty("flushSize", _flushSize),
+                        new JProperty("idleFlushTime", _idleFlushTimeSeconds),  
+                        new JProperty("maxQueueSize", _maxQueueSize),
+                        new JProperty("overflowDiscardOldest", _queueOverflowDiscardOldest),
+                        new JProperty("interval", _interval),
+                        new JProperty("host", _host)
+                        )));
+                       
+            return json;
+        }
+
+        public StatsDOutput(TimberWinR.Manager manager, Parser.StatsDOutputParameters parameters, CancellationToken cancelToken)
+            : base(cancelToken, "StatsD")
+        {
+            _params = parameters;
+            _manager = manager;                     
+            _port = parameters.Port;
+            _host = parameters.Host;
+            _interval = parameters.Interval;
+            _flushSize = parameters.FlushSize;
+            _idleFlushTimeSeconds = parameters.IdleFlushTimeInSeconds;
+            _maxQueueSize = parameters.MaxQueueSize;
+            _queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
+            _numThreads = parameters.NumThreads;
+            _jsonQueue = new List<JObject>();
+
+            var metricsConfig = new MetricsConfig
+            {
+                StatsdServerName = _host,
+                Prefix = parameters.Namespace,
+            };
+
+            StatsdClient.Metrics.Configure(metricsConfig);
+
+            for (int i = 0; i < _numThreads; i++)
+            {
+                Task.Factory.StartNew(StatsDSender, cancelToken, TaskCreationOptions.LongRunning, TaskScheduler.Current);
+            }
+
+        }
+
+        public override string ToString()
+        {
+            return string.Format("StatsD Host: {0} Port: {1}", _host, _port);
+        }
+
+        /// <summary>
+        /// Forward on Json message to Redis Logstash queue
+        /// </summary>
+        /// <param name="jsonMessage"></param>
+        protected override void MessageReceivedHandler(JObject jsonMessage)
+        {
+            if (_manager.Config.Filters != null)
+            {
+                if (ApplyFilters(jsonMessage))
+                    return;
+            }
+           
+
+            var message = jsonMessage.ToString();
+            LogManager.GetCurrentClassLogger().Trace(message);
+
+            lock (_locker)
+            {
+                if (_jsonQueue.Count >= _maxQueueSize)
+                {
+                    // If we've exceeded our queue size, and we're supposed to throw out the oldest objects first,
+                    // then remove as many as necessary to get us under our limit
+                    if (_queueOverflowDiscardOldest)
+                    {
+                        LogManager.GetCurrentClassLogger()
+                            .Warn("Overflow discarding oldest {0} messages", _jsonQueue.Count - _maxQueueSize + 1);
+
+                        _jsonQueue.RemoveRange(0, (_jsonQueue.Count - _maxQueueSize) + 1);
+                    }
+                    // Otherwise we're in a "discard newest" mode, and this is the newest message, so just ignore it
+                    else
+                    {
+                        LogManager.GetCurrentClassLogger()
+                            .Warn("Overflow discarding newest message: {0}", message);
+
+                        return;
+                    }
+                }
+
+                _jsonQueue.Add(jsonMessage);
+            }
+        }
+
+        private bool ApplyFilters(JObject json)
+        {
+            bool drop = false;
+            foreach (var filter in _manager.Config.Filters)
+            {
+                if (!filter.Apply(json))
+                {
+                    LogManager.GetCurrentClassLogger().Debug("{0}: Dropping: {1}", Thread.CurrentThread.ManagedThreadId, json.ToString());
+                    drop = true;
+                }
+            }
+            // Check for matching type (if defined).
+            if (!drop && !string.IsNullOrEmpty(_params.InputType) && json["type"] != null)
+            {
+                string msgType = json["type"].ToString();
+                if (!string.IsNullOrEmpty(msgType) && msgType != _params.InputType)
+                    return true;
+            }
+
+            return drop;
+        }
+
+        // Places messages back into the queue (for a future attempt)
+        private void interlockedInsert(List<JObject> messages)
+        {
+            lock (_locker)
+            {
+                Interlocked.Increment(ref _errorCount);
+                _jsonQueue.InsertRange(0, messages);
+                if (_jsonQueue.Count > _maxQueueSize)
+                {
+                    LogManager.GetCurrentClassLogger().Warn("Exceeded maximum queue depth");
+                }
+            }
+        }
+
+        // 
+        // Pull off messages from the Queue, batch them up and send them all across
+        // 
+        private void StatsDSender()
+        {
+            DateTime lastFlushTime = DateTime.MinValue;
+
+            using (var syncHandle = new ManualResetEventSlim())
+            {
+                // Execute the query
+                while (!Stop)
+                {
+                    if (!CancelToken.IsCancellationRequested)
+                    {
+                        try
+                        {
+                            int messageCount = 0;
+                            List<JObject> messages = new List<JObject>();
+
+                            // Lets get whats in the queue
+                            lock (_locker)
+                            {
+                                messageCount = _jsonQueue.Count;
+
+                                // Time to flush?                             
+                                if (messageCount >= _flushSize || (DateTime.UtcNow - lastFlushTime).Seconds >= _idleFlushTimeSeconds)
+                                {
+                                    messages = _jsonQueue.Take(messageCount).ToList();
+                                    _jsonQueue.RemoveRange(0, messageCount);
+                                    if (messages.Count > 0)
+                                        _manager.IncrementMessageCount(messages.Count);
+                                }
+                            }
+
+                            TransmitStats(messages);
+
+                            if (!Stop)
+                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
+                        }
+                        catch (OperationCanceledException)
+                        {
+                            break;
+                        }
+                        catch (ThreadAbortException)
+                        {
+                            break;
+                        }
+                        catch (Exception ex)
+                        {                           
+                            Interlocked.Increment(ref _errorCount);
+                            LogManager.GetCurrentClassLogger().Error(ex);
+                        }
+                    }
+                }
+            }
+        }
+
+        protected string ExpandField(string fieldName, JObject json)
+        {
+            foreach (var token in json.Children())
+            {
+                string replaceString = "%{" + token.Path + "}";
+                fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
+            }
+            return fieldName;
+        }
+
+        private string BuildMetricPath(string metric, JObject json)
+        {
+            return string.Format("{0}.{1}", ExpandField(_params.Sender, json), ExpandField(metric, json));
+        }
+       
+        private void TransmitStats(List<JObject> messages)
+        {
+            // We've got some to send.
+            if (messages.Count > 0)
+            {
+                do
+                {
+                    try
+                    {
+                        int numMessages = messages.Count;
+                        foreach (var m in messages)
+                        {
+                            SendMetrics(m);
+                        }
+                        messages.RemoveRange(0, numMessages);
+                        Interlocked.Add(ref _sentMessages, numMessages);  
+                    }
+                    catch (Exception ex)
+                    {
+                        LogManager.GetCurrentClassLogger().Error(ex);
+                       
+                        interlockedInsert(messages);  // Put the messages back into the queue
+                        break;
+                    }
+                } while (messages.Count > 0);
+            }
+        }
+
+        // Process all the metrics for this json
+        private void SendMetrics(JObject m)
+        {
+            if (_params.Gauges != null && _params.Gauges.Length > 0)
+                DoGauges(m);
+            if (_params.Counts != null && _params.Counts.Length > 0)
+                DoCounts(m);
+            if (_params.Timings != null && _params.Timings.Length > 0)
+                DoTimings(m);
+            if (_params.Increments != null && _params.Increments.Length > 0)
+                DoIncrements(m);
+            if (_params.Decrements != null && _params.Decrements.Length > 0)
+                DoDecrements(m);
+        }
+
+        // Process the Gauges
+        private void DoGauges(JObject json)
+        {
+            for (int i=0; i<_params.Gauges.Length; i += 2)
+            {
+                string metricPath = BuildMetricPath(_params.Gauges[i], json);
+                string gaugeName = ExpandField(_params.Gauges[i + 1], json);
+                int value;
+                if (int.TryParse(gaugeName, out value))
+                {
+                    Metrics.Gauge(metricPath, value);
+                }
+            }
+        }
+
+        // Process the Gauges
+        private void DoTimings(JObject json)
+        {
+            for (int i = 0; i < _params.Timings.Length; i += 2)
+            {
+                string metricPath = BuildMetricPath(_params.Timings[i], json);
+                string timingName = ExpandField(_params.Timings[i + 1], json);
+                int value;
+                if (int.TryParse(timingName, out value))
+                {
+                    Metrics.Timer(metricPath, value, _params.SampleRate);
+                }
+            }
+        }
+
+        // Process the Counts
+        private void DoCounts(JObject json)
+        {
+            for (int i = 0; i < _params.Counts.Length; i += 2)
+            {
+                string metricPath = BuildMetricPath(_params.Counts[i], json);
+                string countName = ExpandField(_params.Counts[i + 1], json);
+                int value;
+                if (int.TryParse(countName, out value))
+                {
+                   Metrics.Counter(metricPath, value, _params.SampleRate);
+                }
+            }
+        }
+        // Process the Increments
+        private void DoIncrements(JObject json)
+        {
+            foreach (var metric in _params.Increments)
+            {
+                string metricPath = BuildMetricPath(metric, json);
+                Metrics.Counter(metricPath, 1,_params.SampleRate);
+            }
+        }
+
+        // Process the Increments
+        private void DoDecrements(JObject json)
+        {
+            foreach (var metric in _params.Increments)
+            {
+                string metricPath = BuildMetricPath(metric, json);
+                Metrics.Counter(metricPath, -1, _params.SampleRate);
+            }
+        }
+    }
+}
--- a/TimberWinR/Outputs/Stdout.cs
+++ b/TimberWinR/Outputs/Stdout.cs
@@ -17,7 +17,7 @@ namespace TimberWinR.Outputs
        private long _sentMessages;
        public bool Stop { get; set; }

-        public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutput eo, CancellationToken cancelToken)
+        public StdoutOutput(TimberWinR.Manager manager, Parser.StdoutOutputParameters eo, CancellationToken cancelToken)
            : base(cancelToken, "Stdout")
        {
            _sentMessages = 0;
@@ -34,7 +34,8 @@ namespace TimberWinR.Outputs
            JObject json = new JObject(
                new JProperty("stdout",
                    new JObject(
-                        new JProperty("sent_messages", _sentMessages))));
+                        new JProperty("queuedMessageCount", _jsonQueue.Count),
+                        new JProperty("sentMessageCount", _sentMessages))));

            return json;
        }
@@ -67,7 +68,7 @@ namespace TimberWinR.Outputs
                                    foreach (JObject obj in messages)
                                    {
                                        Console.WriteLine(obj.ToString());
-                                        _sentMessages++;
+                                        Interlocked.Increment(ref _sentMessages);
                                    }
                                }
                                catch (Exception ex)
@@ -78,7 +79,7 @@ namespace TimberWinR.Outputs
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                        }
-                        catch (OperationCanceledException oce)
+                        catch (OperationCanceledException)
                        {
                            break;
                        }
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -12,6 +12,7 @@ using Microsoft.SqlServer.Server;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
+using NLog.Config;
 using TimberWinR.Outputs;
 using System.CodeDom.Compiler;

@@ -43,10 +44,8 @@ namespace TimberWinR.Parser
            JToken token = json[oldName];
            if (token != null)
            {
+                json.Add(newName, token.DeepClone());
                json.Remove(oldName);
-                JToken newToken = json[newName];
-                if (newToken == null)
-                    json.Add(newName, token);
            }
        }

@@ -255,7 +254,7 @@ namespace TimberWinR.Parser
    public class Stdin : IValidateSchema
    {
        [JsonProperty(PropertyName = "codec")]
-        public Codec Codec { get; set; }
+        public CodecArguments CodecArguments { get; set; }

        public void Validate()
        {
@@ -263,12 +262,46 @@ namespace TimberWinR.Parser
        }
    }

-    public class Codec
+    public class GeneratorParameters : IValidateSchema
+    {
+        [JsonProperty(PropertyName = "type")]
+        public string Type { get; set; }
+
+        [JsonProperty(PropertyName = "codec")]
+        public CodecArguments CodecArguments { get; set; }
+
+        [JsonProperty(PropertyName = "message")]
+        public string Message { get; set; }
+
+        [JsonProperty(PropertyName = "count")]
+        public int Count { get; set; }
+
+        [JsonProperty(PropertyName = "rate")]
+        public int Rate { get; set; }
+
+        public void Validate()
+        {
+        }
+
+        public GeneratorParameters()
+        {
+            Count = 0; // Infinity messages
+            Rate = 10; // Milliseconds
+            Message = "Hello, world!";
+            CodecArguments = new CodecArguments();
+            CodecArguments.Type = CodecArguments.CodecType.plain;
+        }
+    }
+
+
+    public class CodecArguments
    {
        public enum CodecType
        {
            singleline,
-            multiline
+            multiline,
+            json,
+            plain
        };

        public enum WhatType
@@ -290,15 +323,49 @@ namespace TimberWinR.Parser

        public Regex Re { get; set; }

-        public Codec()
+        public CodecArguments()
        {
            Negate = false;
            MultilineTag = "multiline";
        }
    }

-    public class Log : IValidateSchema
+    public class TailFileArguments : IValidateSchema
    {
+        [JsonProperty(PropertyName = "type")]
+        public string Type { get; set; }
+        [JsonProperty(PropertyName = "location")]
+        public string Location { get; set; }
+        [JsonProperty(PropertyName = "recurse")]
+        public int Recurse { get; set; }
+        [JsonProperty(PropertyName = "fields")]
+        public List<Field> Fields { get; set; }
+        [JsonProperty(PropertyName = "interval")]
+        public int Interval { get; set; }
+        [JsonProperty(PropertyName = "logSource")]
+        public string LogSource { get; set; }
+        [JsonProperty(PropertyName = "codec")]
+        public CodecArguments CodecArguments { get; set; }
+
+        public TailFileArguments()
+        {
+            Fields = new List<Field>();
+            Fields.Add(new Field("LogFilename", "string"));
+            Fields.Add(new Field("Index", "integer"));
+            Fields.Add(new Field("Text", "string"));
+            Interval = 30;
+        }
+
+        public void Validate()
+        {
+
+        }
+    }
+
+    public class LogParameters : IValidateSchema
+    {
+        [JsonProperty(PropertyName = "type")]
+        public string Type { get; set; }
        [JsonProperty(PropertyName = "location")]
        public string Location { get; set; }
        [JsonProperty(PropertyName = "iCodepage")]
@@ -314,9 +381,9 @@ namespace TimberWinR.Parser
        [JsonProperty(PropertyName = "logSource")]
        public string LogSource { get; set; }
        [JsonProperty(PropertyName = "codec")]
-        public Codec Codec { get; set; }
+        public CodecArguments CodecArguments { get; set; }

-        public Log()
+        public LogParameters()
        {
            Fields = new List<Field>();
            Fields.Add(new Field("LogFilename", "string"));
@@ -331,39 +398,51 @@ namespace TimberWinR.Parser
        }
    }

-    public class Tcp : IValidateSchema
+    public class TcpParameters : IValidateSchema
    {
        [JsonProperty(PropertyName = "port")]
        public int Port { get; set; }
+        [JsonProperty(PropertyName = "type")]
+        public string Type { get; set; }
+        [JsonProperty("add_field")]
+        public string[] AddFields { get; set; }
+        [JsonProperty("rename")]
+        public string[] Renames { get; set; }

-        public Tcp()
+        public TcpParameters()
        {
            Port = 5140;
+            Type = "Win32-Tcp";
        }

        public void Validate()
        {
-
        }
    }


-    public class Udp : IValidateSchema
+    public class UdpParameters : IValidateSchema
    {
        [JsonProperty(PropertyName = "port")]
        public int Port { get; set; }
+        [JsonProperty(PropertyName = "type")]
+        public string Type { get; set; }
+        [JsonProperty("add_field")]
+        public string[] AddFields { get; set; }
+        [JsonProperty("rename")]
+        public string[] Renames { get; set; }

-        public Udp()
+        public UdpParameters()
        {
            Port = 5142;
+            Type = "Win32-Udp";
        }

        public void Validate()
        {
-
        }
    }
-    public class W3CLog : IValidateSchema
+    public class W3CLogParameters : IValidateSchema
    {
        [JsonProperty(PropertyName = "location")]
        public string Location { get; set; }
@@ -380,7 +459,7 @@ namespace TimberWinR.Parser
        [JsonProperty(PropertyName = "fields")]
        public List<Field> Fields { get; set; }

-        public W3CLog()
+        public W3CLogParameters()
        {
            CodePage = 0;
            DtLines = 10;
@@ -398,7 +477,7 @@ namespace TimberWinR.Parser
    }


-    public class IISW3CLog : IValidateSchema
+    public class IISW3CLogParameters : IValidateSchema
    {
        [JsonProperty(PropertyName = "location")]
        public string Location { get; set; }
@@ -418,7 +497,7 @@ namespace TimberWinR.Parser
        [JsonProperty(PropertyName = "fields")]
        public List<Field> Fields { get; set; }

-        public IISW3CLog()
+        public IISW3CLogParameters()
        {
            CodePage = -2;
            Recurse = 0;
@@ -464,8 +543,98 @@ namespace TimberWinR.Parser
        }
    }

-    public class ElasticsearchOutput
+    public class StatsDOutputParameters : IValidateSchema
    {
+        public class StatsDGaugeHashException : Exception
+        {
+            public StatsDGaugeHashException()
+                : base("StatsD output 'gauge' must be an array of pairs.")
+            {
+            }
+        }
+
+        public class StatsDCountHashException : Exception
+        {
+            public StatsDCountHashException()
+                : base("StatsD output 'count' must be an array of pairs.")
+            {
+            }
+        }
+        [JsonProperty(PropertyName = "type")]
+        public string InputType { get; set; }
+        [JsonProperty(PropertyName = "sender")]
+        public string Sender { get; set; }
+        [JsonProperty(PropertyName = "namespace")]
+        public string Namespace { get; set; }    
+        [JsonProperty(PropertyName = "host")]
+        public string Host { get; set; }
+        [JsonProperty(PropertyName = "port")]
+        public int Port { get; set; }
+        [JsonProperty(PropertyName = "interval")]
+        public int Interval { get; set; }
+        [JsonProperty(PropertyName = "flush_size")]
+        public int FlushSize { get; set; }
+        [JsonProperty(PropertyName = "idle_flush_time")]
+        public int IdleFlushTimeInSeconds { get; set; }
+        [JsonProperty(PropertyName = "max_queue_size")]
+        public int MaxQueueSize { get; set; }
+        [JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
+        public bool QueueOverflowDiscardOldest { get; set; }
+        [JsonProperty(PropertyName = "threads")]
+        public int NumThreads { get; set; }
+        [JsonProperty(PropertyName = "sample_rate")]
+        public double SampleRate { get; set; }
+        [JsonProperty(PropertyName = "increment")] // Array: metric names
+        public string[] Increments { get; set; }
+        [JsonProperty(PropertyName = "decrement")] // Array: metric names
+        public string[] Decrements { get; set; }
+        [JsonProperty(PropertyName = "gauge")] // Hash: metric_name => gauge
+        public string[] Gauges { get; set; }
+        [JsonProperty(PropertyName = "count")] // Hash: metric_name => count
+        public string[] Counts { get; set; }
+        [JsonProperty(PropertyName = "timing")] // Hash: metric_name => count
+        public string[] Timings { get; set; }
+
+        public StatsDOutputParameters()
+        {
+            SampleRate = 1;
+            Port = 8125;
+            Host = "localhost";
+            Interval = 5000;
+            FlushSize = 5000;
+            IdleFlushTimeInSeconds = 10;
+            QueueOverflowDiscardOldest = true;
+            MaxQueueSize = 50000;
+            NumThreads = 1;
+            Namespace = "timberwinr";
+            Sender = System.Environment.MachineName.ToLower() + "." +
+                     Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                         @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
+                         .GetValue("Domain", "")
+                         .ToString().ToLower();
+        }
+
+        public void Validate()
+        {
+            if (Gauges != null && Gauges.Length % 2 != 0)
+                throw new StatsDGaugeHashException();
+
+            if (Counts != null && Counts.Length % 2 != 0)
+                throw new StatsDCountHashException();
+        }
+    }
+
+
+    public class ElasticsearchOutputParameters : IValidateSchema
+    {
+        public class ElasticsearchBasicAuthException : Exception
+        {
+            public ElasticsearchBasicAuthException()
+                : base("Elasticsearch 'username' and 'password' properties must be set when SSL is enabled.")
+            {
+            }
+        }
+
        const string IndexDatePattern = "(%\\{(?<format>[^\\}]+)\\})";

        [JsonProperty(PropertyName = "host")]
@@ -474,6 +643,12 @@ namespace TimberWinR.Parser
        public string Index { get; set; }
        [JsonProperty(PropertyName = "port")]
        public int Port { get; set; }
+        [JsonProperty(PropertyName = "ssl")]
+        public bool Ssl { get; set; }
+        [JsonProperty(PropertyName = "username")]
+        public string Username { get; set; }
+        [JsonProperty(PropertyName = "password")]
+        public string Password { get; set; }
        [JsonProperty(PropertyName = "timeout")]
        public int Timeout { get; set; }
        [JsonProperty(PropertyName = "threads")]
@@ -482,16 +657,37 @@ namespace TimberWinR.Parser
        public string Protocol { get; set; }
        [JsonProperty(PropertyName = "interval")]
        public int Interval { get; set; }
+        [JsonProperty(PropertyName = "flush_size")]
+        public int FlushSize { get; set; }
+        [JsonProperty(PropertyName = "idle_flush_time")]
+        public int IdleFlushTimeInSeconds { get; set; }
+        [JsonProperty(PropertyName = "max_queue_size")]
+        public int MaxQueueSize { get; set; }
+        [JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
+        public bool QueueOverflowDiscardOldest { get; set; }
+        [JsonProperty(PropertyName = "enable_ping")]
+        public bool EnablePing { get; set; }
+        [JsonProperty(PropertyName = "ping_timeout")]
+        public int PingTimeout { get; set; }

-        public ElasticsearchOutput()
+        public ElasticsearchOutputParameters()
        {
+            FlushSize = 5000;
+            IdleFlushTimeInSeconds = 10;
            Protocol = "http";
            Port = 9200;
+            Ssl = false;
+            Username = string.Empty;
+            Password = string.Empty;
            Index = "";
            Host = new string[] { "localhost" };
            Timeout = 10000;
            NumThreads = 1;
            Interval = 1000;
+            QueueOverflowDiscardOldest = true;
+            MaxQueueSize = 50000;
+            EnablePing = false;
+            PingTimeout = 0;
        }

        public string GetIndexName(JObject json)
@@ -532,9 +728,14 @@ namespace TimberWinR.Parser
            return typeName;
        }

+        public void Validate()
+        {
+            if (Ssl && (string.IsNullOrWhiteSpace(Username) || string.IsNullOrWhiteSpace(Password)))
+                throw new ElasticsearchBasicAuthException();
+        }
    }

-    public class RedisOutput
+    public class RedisOutputParameters
    {
        [JsonProperty(PropertyName = "host")]
        public string[] Host { get; set; }
@@ -546,6 +747,8 @@ namespace TimberWinR.Parser
        public int Timeout { get; set; }
        [JsonProperty(PropertyName = "batch_count")]
        public int BatchCount { get; set; }
+        [JsonProperty(PropertyName = "max_batch_count")]
+        public int MaxBatchCount { get; set; }
        [JsonProperty(PropertyName = "threads")]
        public int NumThreads { get; set; }
        [JsonProperty(PropertyName = "interval")]
@@ -555,13 +758,13 @@ namespace TimberWinR.Parser
        [JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
        public bool QueueOverflowDiscardOldest { get; set; }

-        public RedisOutput()
+        public RedisOutputParameters()
        {
            Port = 6379;
            Index = "logstash";
            Host = new string[] { "localhost" };
            Timeout = 10000;
-            BatchCount = 10;
+            BatchCount = 200;
            NumThreads = 1;
            Interval = 5000;
            QueueOverflowDiscardOldest = true;
@@ -569,27 +772,72 @@ namespace TimberWinR.Parser
        }
    }

-    public class StdoutOutput
+
+
+    public class StdoutOutputParameters
    {
        [JsonProperty(PropertyName = "interval")]
        public int Interval { get; set; }

-        public StdoutOutput()
+        public StdoutOutputParameters()
        {
            Interval = 1000;
        }
    }

+    public class FileOutputParameters
+    {
+        public enum FormatKind
+        {
+            none, indented
+        };
+
+        [JsonProperty(PropertyName = "interval")]
+        public int Interval { get; set; }
+
+        [JsonProperty(PropertyName = "file_name")]
+        public string FileName { get; set; }
+
+        [JsonProperty(PropertyName = "format")]
+        public FormatKind Format { get; set; }
+
+        public FileOutputParameters()
+        {
+            Format = FormatKind.none;
+            Interval = 1000;
+            FileName = "timberwinr.out";
+        }
+
+        public Newtonsoft.Json.Formatting ToFormat()
+        {
+            switch (Format)
+            {
+                case FormatKind.indented:
+                    return Newtonsoft.Json.Formatting.Indented;
+
+                case FormatKind.none:
+                default:
+                    return Newtonsoft.Json.Formatting.None;
+            }
+        }
+    }
+
    public class OutputTargets
    {
        [JsonProperty("Redis")]
-        public RedisOutput[] Redis { get; set; }
+        public RedisOutputParameters[] Redis { get; set; }

        [JsonProperty("Elasticsearch")]
-        public ElasticsearchOutput[] Elasticsearch { get; set; }
+        public ElasticsearchOutputParameters[] Elasticsearch { get; set; }

        [JsonProperty("Stdout")]
-        public StdoutOutput[] Stdout { get; set; }
+        public StdoutOutputParameters[] Stdout { get; set; }
+
+        [JsonProperty("File")]
+        public FileOutputParameters[] File { get; set; }
+
+        [JsonProperty("StatsD")]
+        public StatsDOutputParameters[] StatsD { get; set; }
    }

    public class InputSources
@@ -598,22 +846,28 @@ namespace TimberWinR.Parser
        public WindowsEvent[] WindowsEvents { get; set; }

        [JsonProperty("Logs")]
-        public Log[] Logs { get; set; }
+        public LogParameters[] Logs { get; set; }
+
+        [JsonProperty("TailFiles")]
+        public TailFileArguments[] TailFilesArguments { get; set; }

        [JsonProperty("Tcp")]
-        public Tcp[] Tcps { get; set; }
+        public TcpParameters[] Tcps { get; set; }

        [JsonProperty("Udp")]
-        public Udp[] Udps { get; set; }
+        public UdpParameters[] Udps { get; set; }

        [JsonProperty("IISW3CLogs")]
-        public IISW3CLog[] IISW3CLogs { get; set; }
+        public IISW3CLogParameters[] IISW3CLogs { get; set; }

        [JsonProperty("W3CLogs")]
-        public W3CLog[] W3CLogs { get; set; }
+        public W3CLogParameters[] W3CLogs { get; set; }

        [JsonProperty("Stdin")]
        public Stdin[] Stdins { get; set; }
+
+        [JsonProperty("Generator")]
+        public GeneratorParameters[] Generators { get; set; }
    }

    public partial class Grok : LogstashFilter, IValidateSchema
@@ -660,7 +914,7 @@ namespace TimberWinR.Parser

        public override void Validate()
        {
-            if (Match == null || Match.Length != 2)
+            if (Match == null || Match.Length % 2 != 0)
                throw new GrokFilterException();

            if (AddTag != null && AddTag.Length % 2 != 0)
--- a/TimberWinR/ReleaseNotes.md
+++ b/TimberWinR/ReleaseNotes.md
@@ -2,67 +2,114 @@
 ==================================
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.

-Version History
+Version / Date
+### 1.3.26.0 - 2015-05-15
+1. Added StatsD outputter
+2. Fixed shutdown hang if shutdown was received before service was fully started up.
+3. Closed issue [#36](https://github.com/Cimpress-MCP/TimberWinR/issues/36)

-### 1.3.19.0 - 01/12/2015
+### 1.3.25.0 - 2015-04-30
+1. Fixed Issue [#49](https://github.com/Cimpress-MCP/TimberWinR/issues/49)
+2. Fixed potential non-thread safe when renaming properties
+3. Added add_field, rename support to Udp/Tcp Input Listeners
+4. Fixed issue with multiple renames (was previously only renaming the first one)
+5. Added File outputter for testing.

-1. Added support for Multiline codecs for Stdin and Logs listeners, addresses issue #23
+### 1.3.24.0 - 2015-04-29
+1. Fixed potential bug in TailFiles when tailing log files which are partially flushed
+   to disk, it now will not process the line until the \r\n has been seen.
+2. Added Generator input.

-### 1.3.18.0 - 12/22/2014
+### 1.3.23.0 - 2015-04-23
+1. Fixed bug with parsing a single json config file, rather than reading
+   JSON files from a directory.
+2. Diabled elasticsearch outputter ping by default and parameterized the ping capability.
+
+### 1.3.22.0 - 2015-04-14
+1. Fixed minor bug with TailFiles and service re-starts not picking up
+   rolled files right away.
+
+### 1.3.21.0 - 2015-04-13
+1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets
+   and hosts file.  If we want to add this back, I will add a Udpv6 input.
+
+### 1.3.20.0 - 2015-04-03
+
+1. A re-factoring of Logs and TailLogs to be more efficient and detect log rolling correctly,
+   this requires http://support.microsoft.com/en-us/kb/172190 which will be detected and
+   set by TimberWinR, however, requires a reboot.
+2. Fixed issue [#38](https://github.com/Cimpress-MCP/TimberWinR/issues/38) diagnostic output not showing drop flag for Grok filter.
+3. Created TimberWinR.TestGenerator for complete testing of TimberWinR
+4. Fixed ipv4/ipv6 thread-safe issue with UdpInputListener which might lead to corrupted input data.
+
+### 1.3.19.1 - 2015-03-03
+
+1. Added new Redis parameter _max\_batch\_count_ which increases the _batch\_count_ dynamically over time 
+   to handle input flooding.   Default is _batch\_count_ * 10 
+
+### 1.3.19.0 - 2015-02-26
+
+1. Added support for Multiline codecs for Stdin and Logs listeners, closes issue [#23](https://github.com/Cimpress-MCP/TimberWinR/issues/23)
+2. Added new TailFiles input type which uses a native implementation (more-efficient) than using LogParser's Log
+3. Updated Udp input listner to use UTF8 Encoding rather than ASCII
+4. Reduced noisy complaint about missing log files for Logs listener
+5. Fixed bug when tailing non-existent log files which resulted in high cpu-usage.
+6. Added feature to watch the configuration directory
+
+### 1.3.18.0 - 2014-12-22

 1. Fixed bug introduced in 1.3.17.0 which changed the meaning of the delay for Elasticsearch, Redis and Stdout 
 intervals to be interpreted as seconds instead of milliseconds.   1.3.17.0 should not be used.
 2. Removed ability for installer to downgrade which was leading to leaving previous versions laying around (i.e. reverts 1.3.13.0 change)

-### 1.3.17.0 - 12/19/2014
+### 1.3.17.0 - 2014-12-19

 1. Continued work improving shutdown time by using syncHandle.Wait instead of Thread.Sleep

-### 1.3.16.0 - 12/19/2014
+### 1.3.16.0 - 2014-12-19

 1. Added logSource property to the Log input to facility the steering of log messages to different indices.

-### 1.3.15.0 - 12/12/2014
+### 1.3.15.0 - 2014-12-12

 1. Fixed bug whereby if the Udp or Tcp inputs receive an impropery formatted Json it caused the thread to terminate, and ignore
 future messages.

-### 1.3.14.0 - 12/11/2014
+### 1.3.14.0 - 2014-12-11

 1. Fixed bug with the Grok filter to match properly the value of the Text field against non-blank entries.

-### 1.3.13.0 - 12/02/2014
+### 1.3.13.0 - 2014-12-02

 1. Fixed MSI installer to allow downgrades.

-### 1.3.12.0 - 11/25/2014
+### 1.3.12.0 - 2014-11-25

 1. Fixed all remaining memory leaks due to the COM Weak Surrogate which requires an explicit GC.Collect

-### 1.3.11.0 - 11/21/2014
+### 1.3.11.0 - 2014-11-21

 1. Re-worked WindowsEvent listener to enable shutting down in a quicker fashion.

-### 1.3.10.0 - 11/18/2014
+### 1.3.10.0 - 2014-11-18

 1. Refactored Conditions handler to use non-leaking evaluator.

-### 1.3.9.0 - 11/11/2014
+### 1.3.9.0 - 2014-11-11

 1. Merged in pull request #9
 2. Updated chocolately uninstall to preserve GUID

-### 1.3.8.0 - 11/06/2014
+### 1.3.8.0 - 2014-11-06
+
 1. Added interval parameter to WindowsEvent input listener
 2. Increased default value for interval to 60 seconds for polling WindowsEvents

-### 1.3.7.0 - 10/21/2014
+### 1.3.7.0 - 2014-10-21
+
 1. Added additional information for diagnostics port
 2. Completed minor handling of Log rolling detection

-### 1.3.6.0 - 10/16/2014
+### 1.3.6.0 - 2014-10-16
+
 1. Handle rolling of logs whereby the logfile remains the same, but the content resets back to 0 bytes.
-
-
-
-
--- a/TimberWinR/TimberWinR.csproj
+++ b/TimberWinR/TimberWinR.csproj
@@ -12,6 +12,8 @@
    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <DebugSymbols>true</DebugSymbols>
@@ -32,7 +34,10 @@
  </PropertyGroup>
  <ItemGroup>
    <Reference Include="csredis">
-      <HintPath>..\packages\csredis.1.4.7.1\lib\net40\csredis.dll</HintPath>
+      <HintPath>..\packages\csredis.3.2.1\lib\net40\csredis.dll</HintPath>
+    </Reference>
+    <Reference Include="Elasticsearch.Net">
+      <HintPath>..\packages\Elasticsearch.Net.1.3.1\lib\Elasticsearch.Net.dll</HintPath>
    </Reference>
    <Reference Include="Interop.MSUtil, Version=1.0.0.0, Culture=neutral, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
@@ -40,42 +45,54 @@
      <HintPath>lib\com-logparser\Interop.MSUtil.dll</HintPath>
    </Reference>
    <Reference Include="MaxMind.Db">
-      <HintPath>..\packages\MaxMind.Db.0.2.3.0\lib\net40\MaxMind.Db.dll</HintPath>
-      <Private>True</Private>
+      <HintPath>..\packages\MaxMind.Db.1.0.0.0\lib\net40\MaxMind.Db.dll</HintPath>
    </Reference>
    <Reference Include="MaxMind.GeoIP2">
-      <HintPath>..\packages\MaxMind.GeoIP2.0.4.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
-      <Private>True</Private>
+      <HintPath>..\packages\MaxMind.GeoIP2.2.1.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
+    </Reference>
+    <Reference Include="Nest">
+      <HintPath>..\packages\NEST.1.3.1\lib\Nest.dll</HintPath>
    </Reference>
    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net40\Newtonsoft.Json.dll</HintPath>
    </Reference>
    <Reference Include="NLog">
-      <HintPath>..\packages\NLog.3.1.0.0\lib\net40\NLog.dll</HintPath>
+      <HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
    </Reference>
-    <Reference Include="RapidRegex.Core">
-      <HintPath>..\packages\RapidRegex.Core.1.0.0.2\lib\net40\RapidRegex.Core.dll</HintPath>
+    <Reference Include="RapidRegex.Core, Version=1.0.0.4, Culture=neutral, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\RapidRegex.Core.1.0.0.4\lib\net40\RapidRegex.Core.dll</HintPath>
    </Reference>
    <Reference Include="RestSharp">
-      <HintPath>..\packages\RestSharp.104.4.0\lib\net4\RestSharp.dll</HintPath>
+      <HintPath>..\packages\RestSharp.105.0.0\lib\net4\RestSharp.dll</HintPath>
+    </Reference>
+    <Reference Include="StatsdClient">
+      <HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.dll</HintPath>
+    </Reference>
+    <Reference Include="StatsdClient.Configuration">
+      <HintPath>..\packages\StatsdClient.1.0.0.19\lib\net35\StatsdClient.Configuration.dll</HintPath>
    </Reference>
    <Reference Include="System" />
+    <Reference Include="System.configuration" />
    <Reference Include="System.Core" />
    <Reference Include="System.Linq.Dynamic">
-      <HintPath>..\packages\System.Linq.Dynamic.1.0.3\lib\net40\System.Linq.Dynamic.dll</HintPath>
+      <HintPath>..\packages\System.Linq.Dynamic.1.0.4\lib\net40\System.Linq.Dynamic.dll</HintPath>
    </Reference>
    <Reference Include="System.Xml.Linq" />
    <Reference Include="System.Data.DataSetExtensions" />
    <Reference Include="Microsoft.CSharp" />
    <Reference Include="System.Data" />
    <Reference Include="System.Xml" />
-    <Reference Include="Topshelf, Version=3.1.122.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
+    <Reference Include="Topshelf, Version=3.1.135.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll</HintPath>
+      <HintPath>..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll</HintPath>
    </Reference>
  </ItemGroup>
  <ItemGroup>
+    <Compile Include="Codecs\JsonCodec.cs" />
+    <Compile Include="Codecs\Multiline.cs" />
+    <Compile Include="Codecs\PlainCodec.cs" />
    <Compile Include="Configuration.cs" />
    <Compile Include="ConfigurationErrors.cs" />
    <Compile Include="Diagnostics\Diagnostics.cs" />
@@ -85,8 +102,12 @@
    <Compile Include="Filters\GeoIPFilter.cs" />
    <Compile Include="Filters\JsonFilter.cs" />
    <Compile Include="Filters\MutateFilter.cs" />
+    <Compile Include="ICodec.cs" />
    <Compile Include="Inputs\FieldDefinitions.cs" />
+    <Compile Include="Inputs\GeneratorInput.cs" />
    <Compile Include="Inputs\IISW3CRowReader.cs" />
+    <Compile Include="Inputs\LogsFileDatabase.cs" />
+    <Compile Include="Inputs\TailFileListener.cs" />
    <Compile Include="Inputs\UdpInputListener.cs" />
    <Compile Include="Inputs\W3CInputListener.cs" />
    <Compile Include="Inputs\IISW3CInputListener.cs" />
@@ -97,10 +118,13 @@
    <Compile Include="Inputs\TcpInputListener.cs" />
    <Compile Include="Inputs\LogsListener.cs" />
    <Compile Include="Inputs\WindowsEvtInputListener.cs" />
+    <Compile Include="LogErrors.cs" />
    <Compile Include="Manager.cs" />
    <Compile Include="Outputs\Elasticsearch.cs" />
    <Compile Include="Outputs\OutputSender.cs" />
+    <Compile Include="Outputs\StatsD.cs" />
    <Compile Include="Outputs\Redis.cs" />
+    <Compile Include="Outputs\File.cs" />
    <Compile Include="Outputs\Stdout.cs" />
    <Compile Include="Parser.cs" />
    <Compile Include="Properties\AssemblyInfo.cs" />
@@ -124,8 +148,12 @@
    </Content>
    <None Include="mdocs\Codec.md" />
    <None Include="mdocs\DateFilter.md" />
+    <None Include="mdocs\StatsD.md" />
    <None Include="mdocs\Filters.md" />
    <None Include="mdocs\GeoIPFilter.md" />
+    <None Include="mdocs\Generator.md" />
+    <None Include="mdocs\FileOutput.md" />
+    <None Include="mdocs\TailFiles.md" />
    <None Include="mdocs\UdpInput.md" />
    <None Include="mdocs\W3CInput.md" />
    <None Include="mdocs\JsonFilter.md" />
@@ -138,7 +166,9 @@
    <None Include="mdocs\Logs.md" />
    <None Include="mdocs\IISW3CInput.md" />
    <None Include="mdocs\WindowsEvents.md" />
-    <None Include="packages.config" />
+    <None Include="packages.config">
+      <SubType>Designer</SubType>
+    </None>
    <None Include="ReleaseNotes.md" />
  </ItemGroup>
  <ItemGroup>
@@ -149,6 +179,13 @@
  </ItemGroup>
  <ItemGroup />
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
--- a/TimberWinR/mdocs/Codec.md
+++ b/TimberWinR/mdocs/Codec.md
@@ -8,6 +8,10 @@ The following parameters are allowed when configuring the Codec.
 | *type*            | enum           |Codec type 'multiline'                                                    | Must be 'multiline'           |     |
 | *pattern*         | regex          |Regular expression to be matched                                          | Must be legal .NET Regex      |     |
 | *what*            | enum           |Value can be previous or next                                             | If the pattern matched, does event belong to the next or previous event? |     |
+| *negate*          | bool           |Inverts the pattern sense                                                 | If true, a message not matching the pattern will constitute a match of the multiline filter and the what will be applied. (vice-versa is also true) |  false   |
+| *multiline_tag*   | string         |Tag to be added when multiline conversion is applied                      |                               |  multiline | 
+
+This codec applies to [Logs](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) and [Stdin](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) only.

 Example Input: Mutliline input log file

@@ -20,6 +24,7 @@ Example Input: Mutliline input log file
                "location": "C:\\Logs1\\multiline.log",
                "recurse": -1,
                "codec": {
+                  "negate": false,
                  "type": "multiline",
                  "pattern": "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)",
                  "what": "previous"
--- a/TimberWinR/mdocs/ElasticsearchOutput.md
+++ b/TimberWinR/mdocs/ElasticsearchOutput.md
@@ -3,15 +3,24 @@
 The Elasticsearch output passes on data directly to Elasticsearch.

 ## Parameters
-The following parameters are allowed when configuring the Redis output.
+The following parameters are allowed when configuring the Elasticsearch output.

 | Parameter     |   Type   |  Description                                                | Details               |  Default |
 | :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
-| *threads*     | string   | Location of log files(s) to monitor                         | Number of worker theads to send messages | 1 |
+| *flush_size*                    | integer  | Maximum number of messages before flushing           |   | 50000  |
+| *host*                          | [string] | Array of hostname(s) of your Elasticsearch server(s) | IP or DNS name |  |
+| *idle_flush_time*               | integer  | Maximum number of seconds elapsed before triggering a flush           |   | 10  |
+| *index*                         | [string]    | The index name to use                                       | index used/created | logstash-yyyy.dd.mm |
 | *interval*                      | integer  | Interval in milliseconds to sleep during batch sends        | Interval       | 5000 |
-| *index*       | string   | The index name to use                                       | index used/created | logstash-yyyy.dd.mm |
-| *host*        | [string] | The hostname(s) of your Elasticsearch server(s) | IP or DNS name |  |
-| *port*        | integer  | Redis port number                                           | This port must be open  | 9200  |
+| *max_queue_size*                | integer  | Maximum Elasticsearch queue depth       |  | 50000 |
+| *port*                          | integer  | Elasticsearch port number                                   | This port must be open  | 9200  |
+| *ssl*                           | bool  | If true, use an HTTPS connection to Elasticsearch. See [this page] (https://www.elastic.co/guide/en/found/current/elk-and-found.html#_using_logstash) for a configuration example. | *username* and *password* are also required for HTTPS connections. | false |
+| *username*                      | string | Username for Elasticsearch credentials. | Required for HTTPS connection. | |
+| *password*                      | string | Password for Elasticsearch credentials. | Required for HTTPS connection. | |
+| *queue_overflow_discard_oldest* | bool  | If true, discard oldest messages when max_queue_size reached otherwise discard newest |  | true |
+| *threads*                       | [string]    | Number of Threads                         | Number of worker threads processing messages | 1 |
+| *enable_ping* | bool  | If true, pings the server to test for keep alive |  | false |
+| *ping_timeout* | integer  | Default ping timeout when enable_ping is true | milliseconds | 200 |

 ### Index parameter
 If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need.
@@ -26,7 +35,7 @@ Example Input:
                    "threads":  1,   
                    "interval": 5000,                             
                    "host": [
-                        "tstlexiceapp006.vistaprint.svc"
+                        "tstlexiceapp006.mycompany.svc"
                    ]
                }
            ]
--- a/TimberWinR/mdocs/FileOutput.md
+++ b/TimberWinR/mdocs/FileOutput.md
@@ -0,0 +1,27 @@
+# Output: File
+
+The File output passes on data into a text file.
+
+## Parameters
+The following parameters are allowed when configuring the File output.
+
+| Parameter     |   Type   |  Description                                                | Details               |  Default |
+| :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
+| *interval*    | integer  | Interval in milliseconds to sleep before appending data       | Interval      | 1000 |
+| *file_name*   | string   | Name of the file to be created                              |               | timberwinr.out |
+
+Example Input:
+```json
+{
+    "TimberWinR": {
+        "Outputs": {
+            "File": [
+               { 
+                    "file_name":  "foo.out",   
+                    "interval": 1000                   
+                }
+            ]
+		}
+	}
+}
+```
--- a/TimberWinR/mdocs/Generator.md
+++ b/TimberWinR/mdocs/Generator.md
@@ -0,0 +1,37 @@
+# Input: Generator
+
+The Generator input can be used to Generate log files for test purposes.
+
+## Parameters
+The following parameters are allowed when configuring the test log Generator.
+
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *type*            | string  |Message type                                |  |  Win32-InputGen   |
+| *message*         | string  |Message format to send                                  |  |  Hello, World!   |
+| *count*           | integer |Number of messages to generate                                       | 0 - Infinite, otherwise that number | 0 |
+| *rate*            | integer |Sleep time between generated messages                                    | Milliseconds | 10 |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
+
+Example: Generate 100000 "Hello Win32-InputGen" messages
+
+```json
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Generator": [
+                {
+                   "message":  "Hello %{type}",
+                   "count": 100000
+                }
+            ]
+		}
+	}
+}
+```
+## Fields
+After a successful parse of the generated line, the following fields are added:
+
+| Name | Type | Description |
+| ---- |:-----| :-----------|
+| Message | STRING | Text line content  |
--- a/TimberWinR/mdocs/GrokFilter.md
+++ b/TimberWinR/mdocs/GrokFilter.md
@@ -26,14 +26,14 @@ The following operations are allowed when mutating a field.

 | Operation       |     Type        | Description                                                            
 | :---------------|:----------------|:-----------------------------------------------------------------------|
-| *type*          | property:string |Type to which this filter applyes, if empty, applies to all types.
-| *condition*     | property:string |C# expression
-| *rename*        | property:array  |Rename one or more fields                                       
-| *match*         | property:string |Required field must match before any subsequent grok operations are executed.
 | *add_field*     | property:array  |If the filter is successful, add an arbitrary field to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.                                    
-| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
 | *add_tag*       | property:array  |If the filter is successful, add an arbitrary tag to this event.  Tag names can be dynamic and include parts of the event using the %{field} syntax.                                  
+| *condition*     | property:string |C# expression
+| *match*         | property:array  |Required field must match (any) before any subsequent grok operations are executed.
+| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
 | *remove_tag*    | property:array  |If the filter is successful, remove arbitrary tags from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                          
+| *rename*        | property:array  |Rename one or more fields                                       
+| *type*          | property:string |Type to which this filter applies, if empty, applies to all types.

 ## Operation Details
 ### match 
@@ -67,6 +67,28 @@ Given this configuration
    }     
  ]
 ```
+
+Given this configuration
+```json
+  "Filters": [     
+    {
+	   "grok": {
+           "matches": [
+               "message",
+               "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}"
+           ],
+           "add_tag": [               
+               "http_log"
+           ],
+           "add_field": [
+               "verb", "%{method}"
+           ]
+        }           
+    }     
+  ]
+```
+
+
 And if the message matches, then 6 fields would be added to the event:
 1. client=55.3.244.1
 2. method=GET
--- a/TimberWinR/mdocs/JsonFilter.md
+++ b/TimberWinR/mdocs/JsonFilter.md
@@ -7,25 +7,27 @@ The following operations are allowed when mutating a field.

 | Operation       |     Type        | Description                                                            
 | :---------------|:----------------|:-----------------------------------------------------------------------|
-| *type*          | property:string |Type to which this filter applies, if empty, applies to all types.
-| *condition*     | property:string |C# expression, if the expression is true, continue, otherwise, ignore
-| *remove_source* | property:bool   |If true, the source property is removed, default: true
-| *source*        | property:string |Required field indicates which field contains the Json to be parsed
-| *promote*       | property:string |If supplied any properties named *promote* will be promoted to top-level
-| *target*        | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
 | *add_field*     | property:array  |If the filter is successful, add an arbitrary field to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.                                    
-| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
 | *add_tag*       | property:array  |If the filter is successful, add an arbitrary tag to this event.  Tag names can be dynamic and include parts of the event using the %{field} syntax.                                  
+| *condition*     | property:string |C# expression, if the expression is true, continue, otherwise, ignore
+| *promote*       | property:string |If supplied any properties named *promote* will be promoted to top-level
+| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
 | *remove_tag*    | property:array  |If the filter is successful, remove arbitrary tags from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                          
+| *remove_source* | property:bool   |If true, the source property is removed, default: true
+| *rename*        | property:array  |Rename one or more fields                                       
+| *source*        | property:string |Required field indicates which field contains the Json to be parsed
+| *target*        | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
+| *type*          | property:string |Type to which this filter applies, if empty, applies to all types.

 ## Operation Details
 ### source 
-The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second
-argument.  In the below example, the message is spected to be something like this from a fictional sample log:
+The source field is required, and indicates what Field contains the target Json, In the
+below example, the [Logs](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) input produces a Text field, 
+which contains a line to be parsed as Json.

 Given this input configuration:

-Lets assume that a newline such as the following is appended to foo.jlog:
+Lets assume that a new line such as the following is appended to foo.jlog:, this would end up being the Text field
 ```
   {"Email":"james@example.com","Active":true,"CreatedDate":"2013-01-20T00:00:00Z","Roles":["User","Admin"]}
 ```
@@ -54,14 +56,14 @@ Lets assume that a newline such as the following is appended to foo.jlog:
        }       
 ```

-In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed
+In the above example, the file foo.jlog is being tailed, and when a new line is appended, it is assumed
 to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff*

 The resulting output would be:
 ```
  {
    "type": "Win32-FileLog",
-    "ComputerName": "dev.vistaprint.net",
+    "ComputerName": "dev.mycompany.net",
    "Text": "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}",
    "stuff": {
      "Email": "james@example.com",
@@ -84,8 +86,8 @@ The fields must be in pairs with oldname first and newname second.
        "target": "stuff",
        "source": "Text",
        "rename": [
-            "Text",
-            "Data"
+            "level",
+            "Level"
        ]                   
     } 
   }     
@@ -97,6 +99,8 @@ The fields must be in pairs with fieldName first and value second.
  "Filters": [     
    {
 		"json": {  
+          "type":  "Win32-FileLog",       
+          "source": "Text",   			
 			"add_field": [
              "ComputerName", "Host",
              "Username", "%{SID}"				         
@@ -112,7 +116,9 @@ Remove the fields.  More than one field can be specified at a time.
  "Filters": [     
    {
 		"json": {      			
-			"remove_tag": [             
+          "type":  "Win32-FileLog",       
+          "source": "Text",   			
+			"remove_field": [             
             "static_tag1",
             "Computer_%{Host}"
 			]
@@ -128,6 +134,8 @@ Adds the tag(s) to the tag array.
  "Filters": [     
    {
 		"json": {      			
+          "type":  "Win32-FileLog",       
+          "source": "Text",   			
 			"add_tag": [
               "foo_%{Host}",
 			   "static_tag1"      
@@ -143,6 +151,8 @@ Remove the tag(s) to the tag array.  More than one tag can be specified at a tim
  "Filters": [     
    {
 		"json": {      			
+          "type":  "Win32-FileLog",       
+          "source": "Text",   			
 			"remove_tag": [             
             "static_tag1",
             "Username"
--- a/TimberWinR/mdocs/Logs.md
+++ b/TimberWinR/mdocs/Logs.md
@@ -7,10 +7,12 @@ The following parameters are allowed when configuring WindowsEvents.

 | Parameter         |     Type       |  Description                                                             | Details               |  Default |
 | :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *iCodepage*       | integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
 | *location*        | string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
+| *logSource*       | string  |Source name                                  | Used for conditions |     |
 | *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
 | *splitLongLines*  | boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false |
-| *iCodepage*       | integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
+| *type*            | string  |Typename for this Input                                         |  |  Win32-FileLog  |
 | [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |

 Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern.  I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
@@ -21,6 +23,7 @@ Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.
        "Inputs": {
            "Logs": [
                {
+                    "logSource": "log files",
                    "location": "C:\\Logs1\\*.log",
                    "recurse": -1
                }
@@ -37,3 +40,4 @@ After a successful parse of an event, the following fields are added:
 | LogFilename | STRING |Full path of the file containing this line | 
 | Index | INTEGER | Line number |
 | Text | STRING | Text line content  |
+| type | STRING | Win32-FileLog |
--- a/TimberWinR/mdocs/RedisOutput.md
+++ b/TimberWinR/mdocs/RedisOutput.md
@@ -7,13 +7,15 @@ The following parameters are allowed when configuring the Redis output.

 | Parameter     |   Type   |  Description                                                | Details               |  Default |
 | :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
-| *threads*     | string   | Location of log files(s) to monitor                         | Number of worker theads to send messages | 1 |
-| *interval*    | integer  | Interval in milliseconds to sleep during batch sends        | Interval      | 5000 |
+| *batch_count* | integer  | Sent as a single message                         | Number of messages to aggregate | 200 |
+| *host*        | string | The hostname(s) of your Redis server(s) | IP or DNS name |  |
 | *index*       | string   | The name of the redis list                                  | logstash index name | logstash |
-| *host*        | [string] | The hostname(s) of your Redis server(s) | IP or DNS name |  |
-| *port*        | integer  | Redis port number                                           | This port must be open  | 6379  |
+| *interval*    | integer  | Interval in milliseconds to sleep during batch sends        | Interval      | 5000 |
+| *max_batch_count* | integer  | Dynamically adjusted count maximum                      | Increases over time | batch_count * 10 |
 | *max_queue_size* | integer  | Maximum redis queue depth       |  | 50000 |
+| *port*        | integer  | Redis port number                                           | This port must be open  | 6379  |
 | *queue_overflow_discard_oldest* | bool  | If true, discard oldest messages when max_queue_size reached otherwise discard newest |  | true |
+| *threads*     | string   | Location of log files(s) to monitor                         | Number of worker theads to send messages | 1 |

 Example Input:
 ```json
@@ -26,7 +28,7 @@ Example Input:
                    "interval": 5000, 
                    "batch_count":  500,              
                    "host": [
-                        "tstlexiceapp006.vistaprint.svc"
+                        "tstlexiceapp006.mycompany.svc"
                    ]
                }
            ]
--- a/TimberWinR/mdocs/StatsD.md
+++ b/TimberWinR/mdocs/StatsD.md
@@ -0,0 +1,94 @@
+# Output: StatsD
+
+The StatsD output passes on data directly to StatsD. (https://github.com/etsy/statsd)
+
+## Parameters
+The following parameters are allowed when configuring the StatsD output.
+
+| Parameter     |   Type   |  Description                                                | Details               |  Default |
+| :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
+| *count*                         | string | Array of (metric_name, gauge name) pairs counted | Must come in pairs  |  |
+| *decrement*                     | string | Array of metrics to be decremented |  |  |
+| *flush_size*                    | integer  | Maximum number of messages before flushing           |   | 50000  |
+| *gauge*                         | string | Array of (metric_name, gauge name) pairs gauged | Must come in pairs  |  |
+| *host*                          | string | Hostname or IP of StatsD server | localhost |  |
+| *idle_flush_time*               | integer  | Maximum number of seconds elapsed before triggering a flush           |   | 10  |
+| *increment*                     | string | Array of metrics to be incremented |  |  |
+| *interval*                      | integer  | Interval in milliseconds to sleep between sends     | Interval       | 5000 |
+| *max_queue_size*                | integer  | Maximum StatsD queue depth       |  | 50000 |
+| *namespace*                     | string | Namespace for stats | timberwinr |  |
+| *port*                          | integer  | StatsD port number                                   | This port must be open  | 8125  |
+| *queue_overflow_discard_oldest* | bool  | If true, discard oldest messages when max_queue_size reached otherwise discard newest |  | true |
+| *sample_rate*                   | integer  | StatsD sample rate           |   | 1  |
+| *sender*                        | string | Sender name | FQDN |  |
+| *threads*                       | string    | Number of Threads processing messages          | | 1 |
+| *timing*                        | string | Array of (metric_name, timing_name) pairs timed | Must come in pairs  |  |
+| *type*                          | string |Type to which this filter applies, if empty, applies to all types.
+
+### Example Usage
+Example Input: Tail an apache log file, and record counts for bytes and increments for response codes.
+
+sample-apache.log (snip)
+```
+180.76.5.25 - - [13/May/2015:17:02:26 -0700] "GET /frameset.htm HTTP/1.1" 404 89 "-" "Mozilla/5.0 (compatible; Baiduspider/2.0; +http://www.baidu.com/search/spider.html)" "www.redlug.com"
+208.115.113.94 - - [13/May/2015:17:03:55 -0700] "GET /robots.txt HTTP/1.1" 200 37 "-" "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" "redlug.com"
+208.115.113.94 - - [13/May/2015:17:03:55 -0700] "GET /robots.txt HTTP/1.1" 200 37 "-" "Mozilla/5.0 (compatible; DotBot/1.1; http://www.opensiteexplorer.org/dotbot, help@moz.com)" "www.redlug.com"
+```
+
+Note: [COMBINEDAPACHELOG](https://github.com/elastic/logstash/blob/v1.4.2/patterns/grok-patterns) is a standard
+Grok Pattern.
+
+TimberWinR configuration
+
+```json
+{
+    "TimberWinR": {
+        "Inputs": {
+            "TailFiles": [
+                {
+                    "interval": 5,
+                    "logSource": "apache log files",
+                    "location": "..\\sample-apache.log",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "type": "Win32-TailLog",
+                    "match": [
+                        "Text",
+                        "%{COMBINEDAPACHELOG}"
+                    ]
+                }
+            }
+        ],
+        "Outputs": {            
+            "StatsD": [
+                {
+                    "type": "Win32-TailLog",
+                    "port": 8125,
+                    "host": "stats.mycompany.svc",
+                    "increment":  ["apache.response.%{response}"],
+                    "count":  ["apache.bytes", "%{bytes}"]
+                }
+            ]
+        }
+    }
+}
+
+```
+
+Assuming your FQDN is something like mymachine.mycompany.com, you should see the following in Graphite:
+
+```
+stats.counters.timberwinr.mymachine.mycompany.com.apache.bytes.count
+stats.counters.timberwinr.mymachine.mycompany.com.apache.bytes.rate
+stats.counters.timberwinr.mymachine.mycompany.com.apache.response.200.count
+stats.counters.timberwinr.mymachine.mycompany.com.apache.response.200.rate
+stats.counters.timberwinr.mymachine.mycompany.com.apache.response.404.count
+stats.counters.timberwinr.mymachine.mycompany.com.apache.response.404.rate
+...
+...
+```
--- a/TimberWinR/mdocs/StdinInput.md
+++ b/TimberWinR/mdocs/StdinInput.md
@@ -3,7 +3,12 @@
 The Stdin Input will read from the console (Console.ReadLine) and build a simple message for testing.

 ## Parameters
-There are no Parameters at this time.
+The following parameters are allowed when configuring WindowsEvents.
+
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
+

 ```json
 {
@@ -26,5 +31,3 @@ A field: "type": "Win32-Stdin" is automatically appended, and the entire JSON is
 | ---- |:-----| :-----------------------------------------------------------------------|
 | type | STRING |Win32-Stdin |
 | message | STRING | The message typed in  |
-| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
-
--- a/TimberWinR/mdocs/TailFiles.md
+++ b/TimberWinR/mdocs/TailFiles.md
@@ -0,0 +1,43 @@
+# Input: TailFiles
+
+The TailFiles input will monitor a log (text) file similar to how a Linux "tail -f" command works.  This uses
+a native implementation rather than uses LogParser
+
+## Parameters
+The following parameters are allowed when configuring WindowsEvents.
+
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *type*            | string  |Typename for this Input                                         |  |  Win32-TailLog  |
+| *location*        | string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
+| *logSource*       | string  |Source name                                  | Used for conditions |     |
+| *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
+| *interval*        | integer |Polling interval in seconds                                     | Defaults every 60 seconds | 60 |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
+
+Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern.  I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
+
+```json
+{
+    "TimberWinR": {
+        "Inputs": {
+            "TailFiles": [
+                {
+                    "logSource": "log files",
+                    "location": "C:\\Logs1\\*.log",
+                    "recurse": -1
+                }
+            ]
+		}
+	}
+}
+```
+## Fields
+After a successful parse of an event, the following fields are added:
+
+| Name | Type | Description |
+| ---- |:-----| :-----------|
+| LogFilename | STRING |Full path of the file containing this line | 
+| Index | INTEGER | Line number |
+| Text | STRING | Text line content  |
+| type | STRING | Win32-TailLog |
--- a/TimberWinR/mdocs/TcpInput.md
+++ b/TimberWinR/mdocs/TcpInput.md
@@ -6,8 +6,11 @@ The Tcp input will open a port and listen for properly formatted JSON and will f
 The following parameters are allowed when configuring the Tcp input.

 | Parameter         |     Type         |  Description                                                             | Details               |  Default |
-| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| :---------------- |:-----------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *add_field*       | property:array   |Add field(s) to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.  |     |
 | *port*            | integer          |Port number to open        | Must be an available port |     |
+| *rename*          | property:array   |Rename one or more fields  |  |  |                             
+| *type*            | string           |Typename for this Input    |  |  Win32-Tcp  |

 Example Input: Listen on Port 5140

--- a/TimberWinR/mdocs/UdpInput.md
+++ b/TimberWinR/mdocs/UdpInput.md
@@ -7,7 +7,10 @@ The following parameters are allowed when configuring the Udp input.

 | Parameter         |     Type       |  Description                                                             | Details               |  Default |
 | :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *add_field*       | property:array   |Add field(s) to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.  |     |
 | *port*            | integer  |Port number to open        | Must be an available port |     |
+| *rename*          | property:array   |Rename one or more fields  |  |  |                             
+| *type*            | string   |Typename for this Input                                         |  |  Win32-Udp  |

 Example Input: Listen on Port 5142

--- a/Show More
+++ b/Show More
Author	SHA1	Message	Date
Tommy Parnell	5165224c65	change DEPRECATION notice	2016-01-20 08:06:25 -05:00
Ryan Breen	fd9306e24e	Update README.md Add deprecation notice.	2015-11-05 14:53:57 -05:00
Greg Lutz	050ae7cf84	Merge pull request #59 from schmidt4brains/feature/AddHttpsBasicAuthSupport Added support for HTTPS authentication to Found clusters	2015-09-11 06:46:29 -04:00
Doug Schmidt	a99b04e1b1	Added support for HTTPS authentication to Found clusters Elasticsearch's newly acquired found.io hosted clusters require HTTPS basic authentication in order to feed the index. This feature branch adds optional support for HTTPS according to https://www.elastic.co/guide/en/found/current/elk-and-found.html#_using_logstash When SSL is true, a non-empty username and password are required to authenticate against an Elasticsearch cluster.	2015-08-31 17:15:36 -07:00
Greg Lutz	dddbf6ae0d	logo	2015-07-13 11:26:50 -04:00
Greg Lutz	147e243478	Update WindowsEvents.md	2015-06-19 11:47:45 -04:00
Eric Fontana	7cf1230d7a	Merge pull request #52 from Cimpress-MCP/add_statsd_output Added StatsD outputter and smart shutdown code.	2015-05-18 07:00:36 -04:00
Eric Fontana	4fe5cec9e2	More doc tweaks to explain where it will show up.	2015-05-15 11:17:20 -04:00
Eric Fontana	9308f91316	I just realized this closed issue #36	2015-05-15 11:06:21 -04:00
Eric Fontana	28d20199ab	Updated Release notes and bumped version number to 1.3.26.0	2015-05-15 11:05:06 -04:00
Eric Fontana	5692d2ec42	Switched to use Nuget component for StatsD instead of including the code.	2015-05-15 10:41:09 -04:00
Eric Fontana	4dbb926698	Remove internal names.	2015-05-15 09:57:11 -04:00
Eric Fontana	92f23c1117	Added StatsD outputter and smart shutdown code.	2015-05-15 09:53:24 -04:00
Eric Fontana	cd31cf47f5	Merge pull request #51 from Cimpress-MCP/issue49 Release candidate 1.3.25.0	2015-04-30 13:14:23 -04:00
Eric Fontana	8fe3b1815c	Incorporated review changes	2015-04-30 13:12:44 -04:00
Eric Fontana	58ae2db05c	Remove internal names.	2015-04-30 10:20:24 -04:00
Eric Fontana	f1c4343cf6	Fixed typo	2015-04-30 10:10:15 -04:00
Eric Fontana	3669acbb26	Release 1.3.25.0 candidate	2015-04-30 10:09:07 -04:00
Eric Fontana	6600bb2b85	Added type parameter for Log/Tail inputs	2015-04-29 10:58:13 -04:00
Eric Fontana	50473a1f5a	Merge pull request #50 from Cimpress-MCP/add_input_generator All updates for 1.3.24.0	2015-04-29 10:47:41 -04:00
Eric Fontana	dcbb079101	Doc tweaks from review and make infinite be infinite	2015-04-29 10:04:48 -04:00
Eric Fontana	091fe9e7e4	Removed all usage of ForceFlush	2015-04-29 09:55:47 -04:00
Eric Fontana	6db530b526	Remove dead code	2015-04-29 09:51:40 -04:00
Eric Fontana	4ebe539ea8	All updates for 1.3.24.0	2015-04-29 08:35:07 -04:00
Eric Fontana	75619a239a	Updated installer to include Nest and Elasticsearch.dlls which were missing for elasticsearch output	2015-04-23 11:01:23 -04:00
Eric Fontana	fd67c271b5	Fixed bug with elasticsearch output to disable ping by default, and parametrize ping timeot	2015-04-23 08:35:37 -04:00
Eric Fontana	1f3aaf90fd	Merge pull request #46 from thoean/WaitUntilAllMesssagesAreProcessed UdpListener shutdown simplification	2015-04-21 12:59:50 -04:00
Markus Thurner	dcfdf73842	UdpListener: Simplify shutdown procedure and wait until all messages are processed.	2015-04-20 10:39:50 +02:00
Eric Fontana	dcd104e4f4	Tweaked shutdown code for async Udp Listener	2015-04-14 12:45:31 -04:00
Eric Fontana	2377d4ebd2	Merge pull request #41 from thoean/MakeUdpClientAsync Make UDP listener async	2015-04-14 11:30:48 -04:00
Markus Thurner	64979df012	Use Begin/End Receive for data, and immediately start receiving data again. Processing of result is done outside of the UDP receive method in a separate thread, allowing to process more messages in peak situations.	2015-04-14 15:51:38 +02:00
Eric Fontana	5eb75ab143	Be more pessimistic upon restarts	2015-04-14 07:33:57 -04:00
Eric Fontana	3e9ef6ae88	Updated to reflect master branch build badge	2015-04-13 13:49:03 -04:00
Eric Fontana	7df4dede90	Final tweak for embedded chocolatey	2015-04-13 11:19:37 -04:00
Eric Fontana	d9509757e3	Fixed installer/uninstaller for chocolatey	2015-04-13 11:06:41 -04:00
Eric Fontana	edcac22ea0	Merge pull request #45 from Cimpress-MCP/embedded_chocolatey Embedded chocolatey	2015-04-13 09:25:40 -04:00
Eric Fontana	e98ef89fe2	Updated uninstaller to match new installer.	2015-04-13 08:41:40 -04:00
Eric Fontana	6461369d39	Embed msi	2015-04-13 08:05:30 -04:00
Eric Fontana	f5b2858c1c	Don't check in MSI files.	2015-04-13 07:44:55 -04:00
Eric Fontana	3d0bfc248d	Stage the MSI for chocolatey	2015-04-13 07:42:58 -04:00
Eric Fontana	dbaa52a12e	Embed the MSI into chocolatey package	2015-04-13 07:31:06 -04:00
Eric Fontana	7e69175f19	Updated to use new Timestamp	2015-04-13 06:47:43 -04:00
Eric Fontana	baa70eebbc	Merge pull request #43 from Cimpress-MCP/udp_v4 Rolled Udp input listener support to ipv4 only.	2015-04-13 06:46:17 -04:00
Eric Fontana	29308446a9	Merge pull request #44 from thoean/UseUtcForEventLog Use utc for event log	2015-04-13 06:46:01 -04:00
Eric Fontana	a0cccc0b7f	Merge pull request #42 from thoean/SortableDateInReleaseNotes Use sortable date format to avoid confusion between dd/MM and MM/dd	2015-04-13 06:45:16 -04:00
Markus Thurner	ec2ec66915	Convert TimeGenerated and TimeWritten to UTC.	2015-04-13 12:27:36 +02:00
Markus Thurner	98ef675f9c	Wait for threads to be completed before shutting down, and naming threads for easier debugging.	2015-04-13 12:27:19 +02:00
Eric Fontana	1468a6d0e6	Updated to new version	2015-04-13 06:19:47 -04:00
Eric Fontana	44104f1e59	Rolled UDP port support back to v4 only.	2015-04-13 06:16:04 -04:00
Markus Thurner	22baef9838	Minor cleanup	2015-04-13 12:08:20 +02:00
Markus Thurner	1b51fcd989	Use sortable date format to avoid confusion between dd/MM and MM/dd.	2015-04-13 11:56:04 +02:00
Eric Fontana	0b3204efe8	Fixed release note version typo	2015-04-11 06:46:38 -04:00
Eric Fontana	80f8f9ee0c	fix blank template	2015-04-11 06:41:59 -04:00
Eric Fontana	9d08fc2b28	Minor cleanup	2015-04-11 06:35:09 -04:00
Eric Fontana	349b0bf031	Fixed usage of timeout	2015-04-09 13:04:22 -04:00
Eric Fontana	8b431f92eb	Fixed res	2015-04-09 12:58:00 -04:00
Eric Fontana	770ac1b7b1	Fixed doc and default for batch_count for Redis	2015-04-09 12:57:02 -04:00
Eric Fontana	5d07acad5b	Merge pull request #40 from Cimpress-MCP/test_fixture Bug fix for UdpInputListener, re-factored TailFile and LogsListener to u...	2015-04-09 12:38:12 -04:00
Eric Fontana	065aaa3ec4	Removed test which caused conflict with Newtonsoft	2015-04-09 11:12:15 -04:00
Eric Fontana	835f4f6b61	Fixed topshelf reference	2015-04-09 10:50:19 -04:00
Eric Fontana	2f84f46c39	Re-based NLog package	2015-04-09 10:25:35 -04:00
Eric Fontana	77a2258aff	Updated to latest versions of Packages.	2015-04-09 10:20:13 -04:00
Eric Fontana	8983916f6c	Updated nuget package restore	2015-04-09 09:22:15 -04:00
Eric Fontana	775f33c843	Removed branched packages/Elasticsearch.Net.1.3.1/Elasticsearch.Net.1.3.1.nupkg	2015-04-09 09:20:55 -04:00
Eric Fontana	3b58f768f4	Trimed project	2015-04-09 09:09:57 -04:00
Eric Fontana	c0dbe8f6e9	updates	2015-04-08 13:27:40 -04:00
Eric Fontana	e4bd5be8b1	Remove checked in packages, use restore instead.	2015-04-08 12:45:17 -04:00
Eric Fontana	786b6b4777	remove packages	2015-04-08 12:43:12 -04:00
Eric Fontana	51dc9ee54c	Put back nuget.e	2015-04-08 12:39:38 -04:00
Eric Fontana	796ca51f31	Removed checked in packages.	2015-04-08 12:38:37 -04:00
Eric Fontana	ef7e50b1b9	Added NuGet.exe	2015-04-08 12:25:24 -04:00
Eric Fontana	b43e26fc01	Cleaned up checked in packages.	2015-04-08 12:16:15 -04:00
Eric Fontana	7f6e563238	Enhanced to log JSON errors to Redis/Elasticsearch	2015-04-08 10:58:27 -04:00
Eric Fontana	e8a54782e4	Remove extra Start call for standalone operation.	2015-04-08 08:49:55 -04:00
Eric Fontana	4e5af689f6	Removed redundant Finished() call per Markus Review comments.	2015-04-08 08:15:17 -04:00
Eric Fontana	77f8c0d303	Bug fix for UdpInputListener, re-factored TailFile and LogsListener to use database.	2015-04-08 07:54:41 -04:00
Eric Fontana	9aeee16499	Merge pull request #34 from Cimpress-MCP/bulk_elasticsearch No functional changes - refactored to common code for Multiline codec	2015-03-16 08:35:32 -04:00
Eric Fontana	9ef79978a6	Undo bad commit	2015-03-16 08:34:07 -04:00
Eric Fontana	47f233f863	No functional changes - refactored to common code for Multiline codec	2015-03-16 08:32:54 -04:00
Chris Baldauf	a6affcb3c9	Update broken link based on girl report. Just updating from efontana -> cimpress-mcp organization. http://derecho.elijah.cs.cmu.edu:8585/cimpress-mcp	2015-03-13 12:44:36 -04:00
Eric Fontana	fa1009af51	Merge pull request #33 from Cimpress-MCP/bulk_elasticsearch Re-factored to use Elasticsearch Bulk API	2015-03-11 11:58:28 -04:00
Eric Fontana	b937d6ef45	Fixed doc to match default maxQueueSize	2015-03-11 10:29:31 -04:00
Eric Fontana	c16331ac10	Incorporated changes from reviews	2015-03-11 10:28:13 -04:00
Eric Fontana	ac10640edf	Disable test for AppVeyor builds	2015-03-06 11:57:00 -05:00
Eric Fontana	e9c27c8c19	Doc tweaks	2015-03-06 10:39:20 -05:00
Eric Fontana	7be95a976e	Re-factored to use Elasticsearch NEST and Bulk API	2015-03-06 10:01:50 -05:00
Eric Fontana	f2b0f1a85d	Merge pull request #31 from Cimpress-MCP/mitigate_flood Added new max_batch_count parameter to mitigate flooding when large burs...	2015-03-05 06:40:20 -05:00
Eric Fontana	eaba99144e	Fixed test comment	2015-03-05 06:39:26 -05:00
Eric Fontana	3208da6488	Final changes from PR reviews	2015-03-05 06:38:58 -05:00
Eric Fontana	fb473909e7	Removed all warnings.	2015-03-04 09:50:34 -05:00
Eric Fontana	b7095471fb	Added Unit tests for batchCount and enhanced Elasticsearch error reporting a bit.	2015-03-04 09:36:01 -05:00
Eric Fontana	d7fa582191	Re-factored Batchcounter logic into separate class.	2015-03-03 14:07:53 -05:00
Eric Fontana	6ea3e581fd	Merge branch 'mitigate_flood' of https://github.com/Cimpress-MCP/TimberWinR.git	2015-03-03 13:25:02 -05:00
Eric Fontana	99b51d240d	Updated per PR feedback	2015-03-03 13:24:55 -05:00
Eric Fontana	fd123e3a86	Merge pull request #32 from Cimpress-MCP/fix_redisHostIndex_inc Move redisHostIndex inc to a finally	2015-03-03 13:01:02 -05:00
Ryan Breen	42741fbe1e	Move redisHostIndex inc to a finally It looks like this will just retry connections to the same host if new RedisClient throws. Move _redisHostIndex++ to a finally. Should we also be logging when these happen?	2015-03-03 12:54:48 -05:00
Eric Fontana	cdc2d09150	Added new max_batch_count parameter to mitigate flooding when large bursts occur. Essentially the incoming rate could exceed the outgoing rate and this will mitigate this condition.	2015-03-03 09:55:08 -05:00
Eric Fontana	2e28a50222	Merge branch 'master' of https://github.com/Cimpress-MCP/TimberWinR.git	2015-03-03 08:23:31 -05:00
Eric Fontana	92a9adeca8	Handle quit condition gracefully.	2015-03-03 08:23:25 -05:00
Eric Fontana	91cf59612c	Merge pull request #30 from Cimpress-MCP/chrisbaldauf-default-diag-port-readme Fix default diagnostic port	2015-03-02 12:17:37 -05:00
Chris Baldauf	e0aac878ff	Fix default diagnostic port I noticed that the documented default diagnostics port and the actual were different. From what I could see, 5141 was the default diag port and 5142 was the default UDP port. Feel free to correct me if I'm mistaken.	2015-02-27 13:01:37 -05:00
Eric Fontana	dc89ac996a	Added missing batch_count param to the docs.	2015-02-27 11:23:37 -05:00
Eric Fontana	57b29a5425	Fixed high CPU usage problem for non-existent log files.	2015-02-27 07:37:09 -05:00
Eric Fontana	e5237e8e59	Default to current directory	2015-02-16 06:45:11 -05:00
Eric Fontana	fb61a49fe5	Bumped up TailFile interval to 60 seconds and updated docs.	2015-01-29 10:49:02 -05:00
Eric Fontana	775935683f	Fixed guid	2015-01-29 10:18:38 -05:00
Eric Fontana	024fa68e34	Removed Vistaprint references	2015-01-29 10:17:46 -05:00
Eric Fontana	4654d7dbc1	Added new TailFiles listener.	2015-01-29 09:36:22 -05:00
Eric Fontana	d1e5224ba3	Added missing logSource to the docs.	2015-01-20 09:57:28 -05:00
Eric Fontana	5a34b687bb	Split database into separate file	2015-01-20 09:10:18 -05:00
Eric Fontana	2982482f25	More doc updates	2015-01-20 09:06:36 -05:00
Eric Fontana	4d9aa4fd54	Formatting changes	2015-01-20 08:03:52 -05:00
Eric Fontana	5b0d28ce16	Fixed typo	2015-01-20 06:59:28 -05:00
Eric Fontana	4b255bfd27	More doc updates	2015-01-20 06:59:05 -05:00
Eric Fontana	e7a8ff3eb7	Merge branch 'master' of https://github.com/Cimpress-MCP/TimberWinR.git	2015-01-20 06:56:20 -05:00
Eric Fontana	3f227e0914	Fixed codec doc (was missing negate)	2015-01-20 06:56:15 -05:00
Eric Fontana	42301b5c9f	Merge pull request #26 from Cimpress-MCP/mutate_example_json_fix Update MutateFilter.md	2015-01-20 06:51:23 -05:00