Merge pull request #40 from Cimpress-MCP/test_fixture

Bug fix for UdpInputListener, re-factored TailFile and LogsListener to u...
2015-04-09 12:38:12 -04:00
parent e4bd5be8b1 065aaa3ec4
commit 5d07acad5b
55 changed files with 4779 additions and 430 deletions
--- a/.nuget/NuGet.Config
+++ b/.nuget/NuGet.Config
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+</configuration>
--- a/.nuget/NuGet.targets
+++ b/.nuget/NuGet.targets
@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+    <PropertyGroup>
+        <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">$(MSBuildProjectDirectory)\..\</SolutionDir>
+
+        <!-- Enable the restore command to run before builds -->
+        <RestorePackages Condition="  '$(RestorePackages)' == '' ">false</RestorePackages>
+
+        <!-- Property that enables building a package from a project -->
+        <BuildPackage Condition=" '$(BuildPackage)' == '' ">false</BuildPackage>
+
+        <!-- Determines if package restore consent is required to restore packages -->
+        <RequireRestoreConsent Condition=" '$(RequireRestoreConsent)' != 'false' ">true</RequireRestoreConsent>
+
+        <!-- Download NuGet.exe if it does not already exist -->
+        <DownloadNuGetExe Condition=" '$(DownloadNuGetExe)' == '' ">false</DownloadNuGetExe>
+    </PropertyGroup>
+
+    <ItemGroup Condition=" '$(PackageSources)' == '' ">
+        <!-- Package sources used to restore packages. By default, registered sources under %APPDATA%\NuGet\NuGet.Config will be used -->
+        <!-- The official NuGet package source (https://www.nuget.org/api/v2/) will be excluded if package sources are specified and it does not appear in the list -->
+        <!--
+            <PackageSource Include="https://www.nuget.org/api/v2/" />
+            <PackageSource Include="https://my-nuget-source/nuget/" />
+        -->
+    </ItemGroup>
+
+    <PropertyGroup Condition=" '$(OS)' == 'Windows_NT'">
+        <!-- Windows specific commands -->
+        <NuGetToolsPath>$([System.IO.Path]::Combine($(SolutionDir), ".nuget"))</NuGetToolsPath>
+    </PropertyGroup>
+
+    <PropertyGroup Condition=" '$(OS)' != 'Windows_NT'">
+        <!-- We need to launch nuget.exe with the mono command if we're not on windows -->
+        <NuGetToolsPath>$(SolutionDir).nuget</NuGetToolsPath>
+    </PropertyGroup>
+
+    <PropertyGroup>
+        <PackagesProjectConfig Condition=" '$(OS)' == 'Windows_NT'">$(MSBuildProjectDirectory)\packages.$(MSBuildProjectName.Replace(' ', '_')).config</PackagesProjectConfig>
+        <PackagesProjectConfig Condition=" '$(OS)' != 'Windows_NT'">$(MSBuildProjectDirectory)\packages.$(MSBuildProjectName).config</PackagesProjectConfig>
+    </PropertyGroup>
+
+    <PropertyGroup>
+      <PackagesConfig Condition="Exists('$(MSBuildProjectDirectory)\packages.config')">$(MSBuildProjectDirectory)\packages.config</PackagesConfig>
+      <PackagesConfig Condition="Exists('$(PackagesProjectConfig)')">$(PackagesProjectConfig)</PackagesConfig>
+    </PropertyGroup>
+    
+    <PropertyGroup>
+        <!-- NuGet command -->
+        <NuGetExePath Condition=" '$(NuGetExePath)' == '' ">$(NuGetToolsPath)\NuGet.exe</NuGetExePath>
+        <PackageSources Condition=" $(PackageSources) == '' ">@(PackageSource)</PackageSources>
+
+        <NuGetCommand Condition=" '$(OS)' == 'Windows_NT'">"$(NuGetExePath)"</NuGetCommand>
+        <NuGetCommand Condition=" '$(OS)' != 'Windows_NT' ">mono --runtime=v4.0.30319 "$(NuGetExePath)"</NuGetCommand>
+
+        <PackageOutputDir Condition="$(PackageOutputDir) == ''">$(TargetDir.Trim('\\'))</PackageOutputDir>
+
+        <RequireConsentSwitch Condition=" $(RequireRestoreConsent) == 'true' ">-RequireConsent</RequireConsentSwitch>
+        <NonInteractiveSwitch Condition=" '$(VisualStudioVersion)' != '' AND '$(OS)' == 'Windows_NT' ">-NonInteractive</NonInteractiveSwitch>
+
+        <PaddedSolutionDir Condition=" '$(OS)' == 'Windows_NT'">"$(SolutionDir) "</PaddedSolutionDir>
+        <PaddedSolutionDir Condition=" '$(OS)' != 'Windows_NT' ">"$(SolutionDir)"</PaddedSolutionDir>
+
+        <!-- Commands -->
+        <RestoreCommand>$(NuGetCommand) install "$(PackagesConfig)" -source "$(PackageSources)"  $(NonInteractiveSwitch) $(RequireConsentSwitch) -solutionDir $(PaddedSolutionDir)</RestoreCommand>
+        <BuildCommand>$(NuGetCommand) pack "$(ProjectPath)" -Properties "Configuration=$(Configuration);Platform=$(Platform)" $(NonInteractiveSwitch) -OutputDirectory "$(PackageOutputDir)" -symbols</BuildCommand>
+
+        <!-- We need to ensure packages are restored prior to assembly resolve -->
+        <BuildDependsOn Condition="$(RestorePackages) == 'true'">
+            RestorePackages;
+            $(BuildDependsOn);
+        </BuildDependsOn>
+
+        <!-- Make the build depend on restore packages -->
+        <BuildDependsOn Condition="$(BuildPackage) == 'true'">
+            $(BuildDependsOn);
+            BuildPackage;
+        </BuildDependsOn>
+    </PropertyGroup>
+
+    <Target Name="CheckPrerequisites">
+        <!-- Raise an error if we're unable to locate nuget.exe  -->
+        <Error Condition="'$(DownloadNuGetExe)' != 'true' AND !Exists('$(NuGetExePath)')" Text="Unable to locate '$(NuGetExePath)'" />
+        <!--
+        Take advantage of MsBuild's build dependency tracking to make sure that we only ever download nuget.exe once.
+        This effectively acts as a lock that makes sure that the download operation will only happen once and all
+        parallel builds will have to wait for it to complete.
+        -->
+        <MsBuild Targets="_DownloadNuGet" Projects="$(MSBuildThisFileFullPath)" Properties="Configuration=NOT_IMPORTANT;DownloadNuGetExe=$(DownloadNuGetExe)" />
+    </Target>
+
+    <Target Name="_DownloadNuGet">
+        <DownloadNuGet OutputFilename="$(NuGetExePath)" Condition=" '$(DownloadNuGetExe)' == 'true' AND !Exists('$(NuGetExePath)')" />
+    </Target>
+
+    <Target Name="RestorePackages" DependsOnTargets="CheckPrerequisites">        
+        <Exec Command="$(RestoreCommand)"
+              Condition="'$(OS)' != 'Windows_NT' And Exists('$(PackagesConfig)')" />
+
+        <Exec Command="$(RestoreCommand)"
+              LogStandardErrorAsError="true"
+              Condition="'$(OS)' == 'Windows_NT' And Exists('$(PackagesConfig)')" />
+    </Target>
+
+    <Target Name="BuildPackage" DependsOnTargets="CheckPrerequisites">
+        <Exec Command="$(BuildCommand)"
+              Condition=" '$(OS)' != 'Windows_NT' " />
+
+        <Exec Command="$(BuildCommand)"
+              LogStandardErrorAsError="true"
+              Condition=" '$(OS)' == 'Windows_NT' " />
+    </Target>
+
+    <UsingTask TaskName="DownloadNuGet" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll">
+        <ParameterGroup>
+            <OutputFilename ParameterType="System.String" Required="true" />
+        </ParameterGroup>
+        <Task>
+            <Reference Include="System.Core" />
+            <Using Namespace="System" />
+            <Using Namespace="System.IO" />
+            <Using Namespace="System.Net" />
+            <Using Namespace="Microsoft.Build.Framework" />
+            <Using Namespace="Microsoft.Build.Utilities" />
+            <Code Type="Fragment" Language="cs">
+                <![CDATA[
+                try {
+                    OutputFilename = Path.GetFullPath(OutputFilename);
+
+                    Log.LogMessage("Downloading latest version of NuGet.exe...");
+                    WebClient webClient = new WebClient();
+                    webClient.DownloadFile("https://www.nuget.org/nuget.exe", OutputFilename);
+
+                    return true;
+                }
+                catch (Exception ex) {
+                    Log.LogErrorFromException(ex);
+                    return false;
+                }
+            ]]>
+            </Code>
+        </Task>
+    </UsingTask>
+</Project>
--- a/.nuget/packages.config
+++ b/.nuget/packages.config
@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="NUnit.Runners" version="2.6.3" />
+  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
+  <package id="NUnit.Runners" version="2.6.4" />
+  <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
+  <package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
 </packages>
--- a/README.md
+++ b/README.md
@@ -69,7 +69,6 @@ A single Json filter using the single tag (this is only provided as a convienien
    }
 ]
 ```
-
 Multiple Json filters must use the jsonFilters and array syntax, also mutateFilters, grokFilters, dateFilters, geoipFilters.
 ```json
 "Filters": [
--- a/TimberWinR.ServiceHost/Program.cs
+++ b/TimberWinR.ServiceHost/Program.cs
@@ -7,7 +7,9 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Xml;
 using Microsoft.Win32;
+
 using TimberWinR.Outputs;
 using TimberWinR.ServiceHost;
 using TimberWinR.Inputs;
@@ -26,6 +28,9 @@ namespace TimberWinR.ServiceHost
        
        private static void Main(string[] args)
        {
+
+           
+
            Arguments arguments = new Arguments();
           
            HostFactory.Run(hostConfigurator =>
--- a/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
+++ b/TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj
@@ -12,6 +12,8 @@
    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <PlatformTarget>AnyCPU</PlatformTarget>
@@ -49,9 +51,8 @@
    <Reference Include="Microsoft.CSharp" />
    <Reference Include="System.Data" />
    <Reference Include="System.Xml" />
-    <Reference Include="Topshelf, Version=3.1.122.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll</HintPath>
+    <Reference Include="Topshelf">
+      <HintPath>..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll</HintPath>
    </Reference>
  </ItemGroup>
  <ItemGroup>
@@ -87,6 +88,13 @@
    </ProjectReference>
  </ItemGroup>
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
--- a/TimberWinR.ServiceHost/default.json
+++ b/TimberWinR.ServiceHost/default.json
@@ -18,7 +18,7 @@
        "Filters": [
            {
                "grok": {
-                    "condition": "[EventTypeName] == \"Information Event\"",
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
                    "match": [
                        "Text",
                        ""
--- a/TimberWinR.ServiceHost/packages.config
+++ b/TimberWinR.ServiceHost/packages.config
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
  <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
-  <package id="Topshelf" version="3.1.3" targetFramework="net40" />
+  <package id="Topshelf" version="3.1.4" targetFramework="net40" />
 </packages>
--- a/TimberWinR.TestGenerator/CommandLineOptions.cs
+++ b/TimberWinR.TestGenerator/CommandLineOptions.cs
@@ -0,0 +1,82 @@
+using System.ComponentModel;
+using CommandLine;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using CommandLine.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class CommandLineOptions
+    {
+       // [Option('r', "read", Required = true, HelpText = "Input file to be processed.")]
+      //  public string InputFile { get; set; }
+        [Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")]
+        public string TimberWinRConfigFile { get; set; }
+
+        [Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")]
+        public string TestDir { get; set; }
+
+        [Option("testFile", DefaultValue = "", HelpText = "Config file/directory to use")]
+        public string TestFile { get; set; }
+
+        [Option("resultsFile", HelpText = "Expected results Results json file")]
+        public string ExpectedResultsFile { get; set; }
+
+        [Option('n', "numMessages", DefaultValue = 1000, HelpText = "The number of messages to send to the output(s)")]
+        public int NumMessages { get; set; }
+
+        [Option('l', "logLevel", DefaultValue = "debug", HelpText = "Logging Level Debug|Error|Fatal|Info|Off|Trace|Warn")]
+        public string LogLevel { get; set; }
+
+        [Option('v', "verbose", DefaultValue = true, HelpText = "Prints all messages to standard output.")]
+        public bool Verbose { get; set; }
+
+        [Option("jsonLogDir", DefaultValue = ".", HelpText = "Json LogGenerator Log directory")]
+        public string JsonLogDir { get; set; }
+
+        [OptionArray('j', "json", DefaultValue = new string[] {})]
+        public string[] JsonLogFiles { get; set; }
+
+        [OptionArray("jroll", DefaultValue = new string[] { })]
+        public string[] JsonRollingLogFiles { get; set; }
+        
+        [Option("jsonRate", DefaultValue = 30, HelpText = "Json Rate in Milliseconds between generation of log lines")]
+        public int JsonRate { get; set; }
+
+        [Option('u', "udp", DefaultValue = 0, HelpText = "Enable UDP generator on this Port")]
+        public int Udp { get; set; }
+
+        [Option("udp-host", DefaultValue = "localhost", HelpText = "Host to send Udp data to")]
+        public string UdpHost { get; set; }
+
+        [Option("udp-rate", DefaultValue = 10, HelpText = "Udp Rate in Milliseconds between generation of log lines")]
+        public int UdpRate { get; set; }
+
+        [Option('t', "tcp", DefaultValue = 0, HelpText = "Enable Tcp generator on this Port")]
+        public int Tcp { get; set; }
+
+        [Option("tcp-host", DefaultValue = "localhost", HelpText = "Host to send Tcp data to")]
+        public string TcpHost { get; set; }
+
+        [Option("tcp-rate", DefaultValue = 10, HelpText = "Tcp Rate in Milliseconds between generation of log lines")]
+        public int TcpRate { get; set; }
+
+        [Option('r', "redis", DefaultValue = 0, HelpText = "Enable Redis generator on this Port")]
+        public int Redis { get; set; }
+
+        [Option("redis-host", DefaultValue = "", HelpText = "Host to send Redis data to")]
+        public string RedisHost { get; set; }
+
+        [ParserState]
+        public IParserState LastParserState { get; set; }
+
+        [HelpOption]
+        public string GetUsage()
+        {
+            return HelpText.AutoBuild(this,
+              (HelpText current) => HelpText.DefaultParsingErrorsHandler(this, current));
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/Dynamic.cs
+++ b/TimberWinR.TestGenerator/Dynamic.cs
--- a/TimberWinR.TestGenerator/JsonLogFileGenerator.cs
+++ b/TimberWinR.TestGenerator/JsonLogFileGenerator.cs
@@ -0,0 +1,252 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.IO;
+
+namespace TimberWinR.TestGenerator
+{
+    class JsonLogFileTestParameters
+    {
+        public int NumMessages { get; set; }
+        public string LogFileDir { get; set; }
+        public string LogFileName { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public JsonLogFileTestParameters()
+        {
+            SleepTimeMilliseconds = 30;
+            LogFileDir = ".";
+            NumMessages = 10;
+        }
+    }
+
+    class JsonLogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start JSON LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                {
+                    LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath);
+                    File.Delete(logFilePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var watch = Stopwatch.StartNew();
+
+            // This text is always added, making the file longer over time 
+            // if it is not deleted. 
+            using (StreamWriter sw = File.AppendText(logFilePath))
+            {
+                sw.AutoFlush = true;
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"LineNumber", i+1},
+                        {"Application", "jsonlogfile-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "jsonlog"},
+                        {"Message", string.Format("{0}: Testgenerator jsonlogfile message {1}", i+1, DateTime.UtcNow.ToString("o"))},
+                        {"Index", "logstash"}
+                    };
+                    sw.WriteLine(o.ToString(Formatting.None));
+
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+                LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+                watch.Reset();
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished JSON Log File Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+    }
+
+    class JsonRollingLogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start JSON RollingLogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                    File.Delete(logFilePath);
+
+                if (File.Exists(logFilePath + ".rolled"))
+                    File.Delete(logFilePath + ".rolled");
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+         
+            int quarters = parms.NumMessages/4;
+
+            int[] segments = new int[] {quarters, quarters, quarters, quarters + parms.NumMessages%4};
+            var watch = Stopwatch.StartNew();
+
+
+            int recordNumber = 0;
+            int currentTotal = 0;
+            for (int segment = 0; segment < 4; segment++)
+            {
+                currentTotal += segments[segment];
+             
+                // This text is always added, making the file longer over time 
+                // if it is not deleted. 
+                using (StreamWriter sw = File.AppendText(logFilePath))
+                {
+                    sw.AutoFlush = true;
+   
+                    var lwatch = Stopwatch.StartNew();
+
+                    // The Rolling Generator will roll 1/2 way through the log             
+                    for (int i = 0; i < segments[segment]; i++)
+                    {
+                        JObject o = new JObject
+                        {
+                            {"LineNumber", recordNumber + 1},
+                            {"Application", "jsonrollinglogfile-generator"},
+                            {"Host", hostName},
+                            {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                            {"Type", "jsonrollinglog"},
+                            {
+                                "Message",
+                                string.Format("{0}: Testgenerator jsonrollinglogfile message {1}", recordNumber + 1,
+                                    DateTime.UtcNow.ToString("o"))
+                            },
+                            {"Index", "logstash"}
+                        };
+                        sw.WriteLine(o.ToString(Formatting.None));
+                        recordNumber++;
+                        Thread.Sleep(parms.SleepTimeMilliseconds);
+                    }
+                    LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds for {2} logs", Path.GetFullPath(parms.LogFileName), lwatch.Elapsed, segments[segment]);
+                   
+                }
+               
+                // 
+                // We might not have yet processed all the lines from the first file, so wait till
+                // we catch up before rolling the log file.
+                //
+                LogManager.GetCurrentClassLogger().Info("{0}: Waiting for output to catch up: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, currentTotal);
+                WaitOutputToCatchUp(logFilePath, currentTotal);
+
+                //
+                // Roll the log + wait for the reader to catch up.
+                //  
+                
+                LogManager.GetCurrentClassLogger().Info("{0}: Rolling Log File: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, File.GetCreationTimeUtc(logFilePath));
+
+                RollLogFile(logFilePath); 
+                
+                LogManager.GetCurrentClassLogger().Info("{0}: Finished Rolling Log File: {1}", Thread.CurrentThread.ManagedThreadId, logFilePath);           
+            }
+            
+            watch.Stop();
+           
+            LogManager.GetCurrentClassLogger().Info("Finished JSON RollingLogFile File Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+
+        private static void WaitOutputToCatchUp(string logFilePath, int firstPart)
+        {
+            bool caughtUp = false;
+            do
+            {
+                var json = Program.Diagnostics.DiagnosticsOutput();
+
+                IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+                foreach (JToken t in inputs)
+                {
+                    JProperty inputProp = t.First as JProperty;
+                    if (inputProp.Name == "taillog" || inputProp.Name == "log")
+                    {
+                        var files = inputProp.Value["filedb"].Children().ToList();
+                        foreach (var file in files)
+                        {
+                            var fileName = file["FileName"].ToString();
+                            FileInfo fi1 = new FileInfo(fileName);
+                            FileInfo fi2 = new FileInfo(logFilePath);
+                            if (fi1.FullName == fi2.FullName)
+                            {
+                                var linesProcessed = file["LinesProcessed"].Value<int>();
+                                if (linesProcessed >= firstPart)
+                                {
+                                    caughtUp = true;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                Thread.Sleep(300);
+            } while (!caughtUp);
+
+            LogManager.GetCurrentClassLogger().Info("{0}:  Finished Waiting for output to catch up: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, firstPart);            
+
+        }
+
+        private static void RollLogFile(string logFilePath)
+        {
+            bool moved = false;           
+            do
+            {
+                try
+                {
+                    if (File.Exists(logFilePath + ".rolled"))
+                        File.Delete(logFilePath + ".rolled");
+
+                    File.Move(logFilePath, logFilePath + ".rolled");
+                    moved = true;                   
+                }
+                catch (Exception)
+                {
+                    Thread.Sleep(100);
+                }
+            } while (!moved);
+            Thread.Sleep(1000);
+        }
+    }
+
+}
--- a/TimberWinR.TestGenerator/LogFileGenerator.cs
+++ b/TimberWinR.TestGenerator/LogFileGenerator.cs
@@ -0,0 +1,94 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.IO;
+
+
+namespace TimberWinR.TestGenerator
+{
+    class LogFileTestParameters
+    {
+        public int NumMessages { get; set; }
+        public string LogFileDir { get; set; }
+        public string LogFileName { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public LogFileTestParameters()
+        {
+            SleepTimeMilliseconds = 30;
+            LogFileDir = ".";
+            NumMessages = 10;
+        }
+    }
+
+    class LogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                {
+                    LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath);
+                    File.Delete(logFilePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var watch = Stopwatch.StartNew();
+
+            // This text is always added, making the file longer over time 
+            // if it is not deleted. 
+            using (StreamWriter sw = File.AppendText(logFilePath))
+            {
+                sw.AutoFlush = true;
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"LineNumber", i+1},
+                        {"Application", "logfile-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "log"},
+                        {"Message", string.Format("{0}: Testgenerator logfile message {1}", i+1, DateTime.UtcNow.ToString("o"))},
+                        {"Index", "logstash"}
+                    };
+                    sw.WriteLine(o.ToString(Formatting.None));
+
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+                LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+                watch.Reset();
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished LogFile Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/Program.cs
+++ b/TimberWinR.TestGenerator/Program.cs
@@ -0,0 +1,579 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.CodeDom.Compiler;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+using ServiceStack.Text.Jsv;
+using TimberWinR.Parser;
+
+
+namespace TimberWinR.TestGenerator
+{
+    public class Program
+    {
+        private static List<Task> _tasks = new List<Task>();
+        private static CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
+        private static Manager _timberWinR;
+
+        public static Diagnostics.Diagnostics Diagnostics { get; set; }
+
+        private static PerformanceCounter cpuCounter = new PerformanceCounter();
+        private static PerformanceCounter ramCounter = new PerformanceCounter();
+        private static Task _monitorTask;
+
+        private static int _totalMessagesToSend;
+        private static int _cpuSampleCount;
+        private static double _avgCpuUsage;
+        private static double _totalCpuUsage;
+        private static double _maxCpuUsage;
+
+        private static int _memSampleCount;
+        private static double _avgMemUsage;
+        private static double _totalMemUsage;
+        private static double _maxMemUsage;
+
+        private static CommandLineOptions Options;
+
+        static int Main(string[] args)
+        {
+            _totalMessagesToSend = 0;
+
+            cpuCounter.CategoryName = "Processor";
+            cpuCounter.CounterName = "% Processor Time";
+            cpuCounter.InstanceName = "_Total";
+
+            ramCounter.CategoryName = "Memory";
+            ramCounter.CounterName = "% Committed Bytes In Use";
+           
+            Options = new CommandLineOptions();
+
+            if (CommandLine.Parser.Default.ParseArguments(args, Options))
+            {
+                var testFile = Options.TestFile;
+                if (!string.IsNullOrEmpty(testFile))
+                {
+                    if (!File.Exists(Options.TestFile))
+                        throw new Exception(string.Format("No such test file: {0} found", Options.TestFile));
+
+                    var fargs = ParseTestArguments(testFile, ref Options);
+                    if (!CommandLine.Parser.Default.ParseArguments(fargs, Options))
+                        return 2;
+                }
+
+                SetupTestDirectory(Options);
+
+                var swOverall = Stopwatch.StartNew();
+                swOverall.Start();
+
+                InitializeLogging(Options.LogLevel);
+
+                LogManager.GetCurrentClassLogger().Info("Starting CPU Usage: {0}, RAM Usage: {1}", getCurrentCpuUsage(), getAvailableRAM());
+
+                // Reset the tests.
+                ResetTests(Options);
+
+                var sw = Stopwatch.StartNew();
+
+                // Startup TimberWinR
+                StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
+
+                // Run the Generators
+                var arrayOfTasks = RunGenerators(Options);
+
+                // Wait for all Generators to finish
+                try
+                {
+                    Task.WaitAll(arrayOfTasks);
+                }
+                catch (AggregateException aex)
+                {
+                    LogManager.GetCurrentClassLogger().Error(aex);
+                }
+
+
+                LogManager.GetCurrentClassLogger().Info("Generation Finished: " + sw.Elapsed);
+                sw.Reset();
+                sw.Start();
+
+                // All generators are finished, wait till senders are done.
+                WaitForOutputTransmission();
+
+                LogManager.GetCurrentClassLogger().Info("Finished Transmission: " + sw.Elapsed);
+                sw.Reset();
+                sw.Start();
+
+                // Get all the stats
+                var jsonTimberWinr = ShutdownTimberWinR();
+
+                LogManager.GetCurrentClassLogger().Info("Finished Shutdown: " + sw.Elapsed);
+                sw.Stop();
+
+                swOverall.Stop();
+                LogManager.GetCurrentClassLogger().Info("Total Elapsed Time: {0}", swOverall.Elapsed);
+
+                int results = VerifyResults(Options, jsonTimberWinr);
+
+                Console.ReadKey();
+                return results;
+            }
+
+            return 1;
+        }
+
+        private static void CopySourceFile(string fileName, string outputDir)
+        {
+            FileInfo fi = new FileInfo(fileName);
+            if (fi.Exists)
+                File.Copy(fileName, Path.Combine(outputDir, fi.Name));
+        }
+
+        private static void SetupTestDirectory(CommandLineOptions options)
+        {
+            if (options.TestDir != "." && Directory.Exists(options.TestDir))
+                Directory.Delete(options.TestDir, true);
+
+            if (!Directory.Exists(options.TestDir))
+                Directory.CreateDirectory(options.TestDir);
+
+            CopySourceFile(options.TestFile, options.TestDir);
+            CopySourceFile(options.TimberWinRConfigFile, options.TestDir);
+            CopySourceFile(options.ExpectedResultsFile, options.TestDir);
+
+            Directory.SetCurrentDirectory(options.TestDir);
+        }
+
+        private static string[] ParseTestArguments(string testFile, ref CommandLineOptions options)
+        {
+            options = new CommandLineOptions();
+            JObject jtest = JObject.Parse(File.ReadAllText(testFile));
+            IList<JToken> inputs = jtest["arguments"].Children().ToList();
+            List<string> testargs = new List<string>();
+            foreach (JProperty it in inputs)
+            {
+                testargs.Add(it.Name);
+
+                var cc = it.Value.Children().Count();
+                if (cc > 0)
+                {
+                    for (int i = 0; i < cc; i++)
+                    {
+                        testargs.Add(it.Value[i].ToString());
+                    }
+                }
+                else
+                {
+                    testargs.Add(it.Value.ToString());
+                }
+            }
+            var fargs = testargs.ToArray();
+            return fargs;
+        }
+
+        private static int VerifyResults(CommandLineOptions options, JObject json)
+        {
+            var jresult = JObject.Parse(File.ReadAllText(options.ExpectedResultsFile));
+
+            json["maxCpuUsage"] = _maxCpuUsage;
+            json["avgCpuUsage"] = _avgCpuUsage;
+
+            json["maxMemUsage"] = _maxMemUsage;
+            json["avgMemUsage"] = _avgMemUsage;
+
+            // TailLogs
+
+            IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+            foreach (JToken t in inputs)
+            {
+                JProperty inputProp = t.First as JProperty;
+                switch (inputProp.Name)
+                {
+                    case "udp":
+                        return VerifyConditions(json, new string[] { "udp" }, inputProp, jresult);
+
+                    case "log":
+                    case "taillog":
+                        return VerifyConditions(json, new string[] { "log", "taillog" }, inputProp, jresult);
+                }
+            }
+
+            return 0;
+        }
+
+        private static int VerifyConditions(JObject json, string[] logTypes, JProperty inputProp, JObject jresult)
+        {
+            var ttail = inputProp.Value as JObject;
+            foreach (var resultInput in jresult["Results"]["Inputs"].Children().ToList())
+            {
+                JProperty rinputProp = resultInput.First as JProperty;
+                if (logTypes.Contains(rinputProp.Name))
+                {
+                    foreach (JProperty testProp in rinputProp.Value)
+                    {
+                        try
+                        {
+                            var cond1 = testProp.Value.ToString();
+                            IList<string> tkeys = ttail.Properties().Select(pn => pn.Name).ToList();
+                            foreach (string tkey in tkeys)
+                                cond1 = cond1.Replace(string.Format("[{0}]", tkey), string.Format("{0}", ttail[tkey].ToString()));
+
+                            // Add builtins
+                            cond1 = cond1.Replace("[avgCpuUsage]", json["avgCpuUsage"].ToString());
+                            cond1 = cond1.Replace("[maxCpuUsage]", json["maxCpuUsage"].ToString());
+                            cond1 = cond1.Replace("[avgMemUsage]", json["avgMemUsage"].ToString());
+                            cond1 = cond1.Replace("[maxMemUsage]", json["maxMemUsage"].ToString());
+
+                            var p1 = Expression.Parameter(typeof(JObject), "json");
+                            var e1 = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] { p1 },
+                                typeof(bool), cond1);
+                            bool r1 = (bool)e1.Compile().DynamicInvoke(ttail);
+                            if (!r1)
+                            {
+                                LogManager.GetCurrentClassLogger().Error("Test Failed: '{0}: ({1})'", testProp.Name, cond1);
+                                return 1;
+
+                            }
+                            else
+                            {
+                                LogManager.GetCurrentClassLogger()
+                                    .Info("PASSED({0}): '{1}: ({2})'", inputProp.Name, testProp.Name, cond1);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            LogManager.GetCurrentClassLogger()
+                                .Error("Error parsing expression '{0}': {1}", testProp.Value.ToString(),
+                                    ex.Message);
+                            return 2;
+                        }
+                    }
+                }
+            }
+            return 0;
+        }
+
+        // Wait till all output has been transmitted.
+        private static void WaitForOutputTransmission()
+        {
+            bool completed = false;
+            do
+            {
+                var json = Diagnostics.DiagnosticsOutput();
+
+                //Console.WriteLine(json.ToString(Formatting.Indented));
+
+                IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+                foreach (var so in inputs.Children())
+                {
+                    var token = so.First;
+                    var messages = token["messages"].Value<int>();
+                    //  Console.WriteLine("{0} messages", messages);
+                }
+
+
+                IList<JToken> outputs = json["timberwinr"]["outputs"].Children().ToList();
+                foreach (var so in outputs.Children())
+                {
+                    var outputToken = so.First;
+
+                    var mbc = outputToken["queuedMessageCount"].Value<int>();
+                    var smc = outputToken["sentMessageCount"].Value<int>();
+
+                    // LogManager.GetCurrentClassLogger().Info("Queued: {0}, Sent: {1}", mbc, smc);
+
+                    completed = mbc == 0 && smc >= _totalMessagesToSend;
+                }
+                Thread.Sleep(250);
+            } while (!completed);
+        }
+
+        private static void sampleUsages()
+        {
+            getCurrentCpuUsage();
+            getAvailableRAM();
+        }
+
+        private static string getCurrentCpuUsage()
+        {
+            _cpuSampleCount++;
+            var v = cpuCounter.NextValue();
+            if (v > _maxCpuUsage)
+                _maxCpuUsage = v;
+
+            _totalCpuUsage += v;
+            _avgCpuUsage = _totalCpuUsage / _cpuSampleCount;
+
+            return v + "%";
+        }
+
+        private static string getAvailableRAM()
+        {
+            _memSampleCount++;
+            var v = ramCounter.NextValue();
+            if (v > _maxMemUsage)
+                _maxMemUsage = v;
+
+            _totalMemUsage += v;
+            _avgMemUsage = _totalMemUsage / _memSampleCount;
+            return v + "MB";
+        }
+
+        private static JObject ShutdownTimberWinR()
+        {
+            _timberWinR.Shutdown();
+
+            // Cancel any/all other threads
+            _cancellationTokenSource.Cancel();
+
+            var json = Diagnostics.DiagnosticsOutput();
+
+            LogManager.GetCurrentClassLogger()
+                .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB", _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
+
+            LogManager.GetCurrentClassLogger().Info(json.ToString());
+
+            Diagnostics.Shutdown();
+
+            return json;
+        }
+
+        static void StartTimberWinR(string configFile, string logLevel, string logFileDir, bool enableLiveMonitor)
+        {
+            _timberWinR = new TimberWinR.Manager(configFile, logLevel, logFileDir, enableLiveMonitor, _cancellationTokenSource.Token, false);
+            _timberWinR.OnConfigurationProcessed += TimberWinROnOnConfigurationProcessed;
+            _timberWinR.Start(_cancellationTokenSource.Token);
+            Diagnostics = new Diagnostics.Diagnostics(_timberWinR, _cancellationTokenSource.Token, 5141);
+        }
+
+        private static void TimberWinROnOnConfigurationProcessed(Configuration configuration)
+        {
+            Console.WriteLine("Processed Config: {0}", configuration.GetHashCode());
+
+            if (!string.IsNullOrEmpty(Options.RedisHost) && configuration.RedisOutputs != null && configuration.RedisOutputs.Count() > 0)
+            {
+                foreach (var ro in configuration.RedisOutputs)
+                {
+                    ro.Host = new string[] { Options.RedisHost };
+                }
+            }
+
+        }
+
+        static void InitializeLogging(string logLevel)
+        {
+            var loggingConfiguration = new LoggingConfiguration();
+
+            // Create our default targets
+            var coloredConsoleTarget = new ColoredConsoleTarget();
+
+            var logFileDir = ".";
+
+            Target fileTarget = CreateDefaultFileTarget(logFileDir);
+
+            loggingConfiguration.AddTarget("Console", coloredConsoleTarget);
+            loggingConfiguration.AddTarget("DailyFile", fileTarget);
+
+            // The LogLevel.Trace means has to be at least Trace to show up on console
+            loggingConfiguration.LoggingRules.Add(new LoggingRule("*", LogLevel.Trace, coloredConsoleTarget));
+            // LogLevel.Debug means has to be at least Debug to show up in logfile
+            loggingConfiguration.LoggingRules.Add(new LoggingRule("*", LogLevel.Debug, fileTarget));
+
+            LogManager.Configuration = loggingConfiguration;
+            LogManager.EnableLogging();
+
+            LogManager.GlobalThreshold = LogLevel.FromString(logLevel);
+        }
+
+        static FileTarget CreateDefaultFileTarget(string logPath)
+        {
+            return new FileTarget
+            {
+                ArchiveEvery = FileArchivePeriod.None,
+                ArchiveAboveSize = 5 * 1024 * 1024,
+                MaxArchiveFiles = 5,
+                BufferSize = 10,
+                FileName = Path.Combine(logPath, "TimberWinR.TestGenerator", "TimberWinRTestGen.log"),
+                ArchiveFileName = Path.Combine(logPath, "TimberWinR-TestGenerator_log-{#######}.log"),
+            };
+        }
+
+        static void ResetTests(CommandLineOptions options)
+        {
+            if (File.Exists(".timberwinrdb"))
+                File.Delete(".timberwinrdb");
+
+            if (File.Exists("TimberWinR.TestGenerator\\TimberWinRTestGen.log"))
+                File.Delete("TimberWinR.TestGenerator\\TimberWinRTestGen.log");
+
+            if (File.Exists("TimberWinR\\TimberWinR.log"))
+                File.Delete("TimberWinR\\TimberWinR.log");
+
+            if (options.JsonLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonLogFiles)
+                {
+                    if (File.Exists(logFile))
+                        File.Delete(logFile);
+                }
+            }
+
+            if (options.JsonRollingLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonRollingLogFiles)
+                {
+                    if (File.Exists(logFile))
+                        File.Delete(logFile);
+                }
+            }
+        }
+
+        static Task[] RunGenerators(CommandLineOptions options)
+        {
+            _monitorTask = Task.Factory.StartNew(() =>
+                {
+                    using (var syncHandle = new ManualResetEventSlim())
+                    {
+                        try
+                        {
+                            // Execute the query
+                            while (!_cancellationTokenSource.Token.IsCancellationRequested)
+                            {
+                                sampleUsages();
+                                // LogManager.GetCurrentClassLogger().Info("Starting CPU Usage: {0}, RAM Usage: {1}", getCurrentCpuUsage(), getAvailableRAM());
+                                syncHandle.Wait(TimeSpan.FromMilliseconds(options.JsonRate), _cancellationTokenSource.Token);
+                            }
+                        }
+                        catch (OperationCanceledException)
+                        {
+                        }
+                        catch (Exception ex)
+                        {
+                            LogManager.GetCurrentClassLogger().Error(ex);
+                        }
+                    }
+                }, _cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Current);
+
+            StartJson(options);
+            StartJsonRolling(options);
+            StartUdp(options);
+            StartTcp(options);
+
+            return _tasks.ToArray();
+        }
+
+        static void StartJson(CommandLineOptions options)
+        {
+            if (options.JsonLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonLogFiles)
+                {
+                    _totalMessagesToSend += options.NumMessages;
+
+                    if (options.Verbose)
+                        LogManager.GetCurrentClassLogger()
+                            .Info("Starting LogFile Generator for {0}",
+                                Path.GetFullPath(Path.Combine(options.JsonLogDir, logFile)));
+                    _tasks.Add(Task.Factory.StartNew(() =>
+                    {
+                        var p = new JsonLogFileTestParameters()
+                        {
+                            NumMessages = options.NumMessages,
+                            LogFileDir = options.JsonLogDir,
+                            LogFileName = logFile,
+                            SleepTimeMilliseconds = options.JsonRate
+                        };
+                        JsonLogFileGenerator.Generate(p);
+                        Thread.Sleep(250);
+                    }));
+
+                }
+            }
+        }
+
+        private static void StartJsonRolling(CommandLineOptions options)
+        {
+            if (options.JsonRollingLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonRollingLogFiles)
+                {
+                    _totalMessagesToSend += options.NumMessages;
+
+                    if (options.Verbose)
+                        LogManager.GetCurrentClassLogger()
+                            .Info("Starting RollingLogFile Generator for {0}",
+                                Path.GetFullPath(Path.Combine(options.JsonLogDir, logFile)));
+                    _tasks.Add(Task.Factory.StartNew(() =>
+                    {
+                        var p = new JsonLogFileTestParameters()
+                        {
+                            NumMessages = options.NumMessages,
+                            LogFileDir = options.JsonLogDir,
+                            LogFileName = logFile,
+                            SleepTimeMilliseconds = options.JsonRate
+                        };
+                        JsonRollingLogFileGenerator.Generate(p);
+                        Thread.Sleep(250);
+                    }));
+
+                }
+            }
+        }
+
+        static void StartUdp(CommandLineOptions options)
+        {
+            if (options.Udp > 0)
+            {
+                if (options.Verbose)
+                    LogManager.GetCurrentClassLogger()
+                        .Info("Starting UDP Generator for {0}:{1}", options.UdpHost, options.Udp);
+
+                _tasks.Add(Task.Factory.StartNew(() =>
+                {
+                    var p = new UdpTestParameters()
+                    {
+                        Port = options.Udp,
+                        Host = options.UdpHost,
+                        NumMessages = options.NumMessages,
+                        SleepTimeMilliseconds = options.UdpRate
+                    };
+                    _totalMessagesToSend += UdpTestGenerator.Generate(p);
+                }));
+            }
+        }
+
+        static void StartTcp(CommandLineOptions options)
+        {
+            if (options.Tcp > 0)
+            {
+                if (options.Verbose)
+                    LogManager.GetCurrentClassLogger()
+                        .Info("Starting Tcp Generator for {0}:{1}", options.TcpHost, options.Tcp);
+
+                _totalMessagesToSend += options.NumMessages;
+
+                _tasks.Add(Task.Factory.StartNew(() =>
+                {
+                    var p = new TcpTestParameters()
+                    {
+                        Port = options.Tcp,
+                        Host = options.TcpHost,
+                        NumMessages = options.NumMessages,
+                        SleepTimeMilliseconds = options.TcpRate
+                    };
+                    TcpTestGenerator.Generate(p);
+                }));
+            }
+        }
+
+    }
+}
--- a/TimberWinR.TestGenerator/Properties/AssemblyInfo.cs
+++ b/TimberWinR.TestGenerator/Properties/AssemblyInfo.cs
@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("TimberWinR.TestGenerator")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("TimberWinR.TestGenerator")]
+[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("a56bf91c-c5f8-4771-8ef8-ab9ad28179c4")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]
--- a/TimberWinR.TestGenerator/RedisTestGenerator.cs
+++ b/TimberWinR.TestGenerator/RedisTestGenerator.cs
@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+using ServiceStack.Redis;
+
+namespace TimberWinR.TestGenerator
+{
+    class RedisTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public RedisTestParameters()
+        {
+            NumMessages = 100;
+            Port = 6379;
+            Host = "localhost";
+        }
+    }
+
+    class RedisTestGenerator
+    {
+        public static void Generate(RedisTestParameters parms)
+        {
+            var hostName = System.Environment.MachineName + "." +
+               Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                   "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var rc = new RedisClient(parms.Host, parms.Port);
+
+            for (int i = 0; i < parms.NumMessages; i++)
+            {
+                JObject o = new JObject
+                {
+                    {"Application", "redis-generator"},               
+                    {"Host", hostName},
+                    {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                    {"Type", "redis"},                
+                    {"Message", "redis message " + DateTime.UtcNow.ToString("o")},
+                    {"Index", "logstash"}
+                };
+                byte[] bytes = System.Text.Encoding.UTF8.GetBytes(o.ToString());
+                var restult = rc.RPush("logstash", bytes);
+            }
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/TcpTestGenerator.cs
+++ b/TimberWinR.TestGenerator/TcpTestGenerator.cs
@@ -0,0 +1,62 @@
+using System.Threading;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using ServiceStack.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class TcpTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public TcpTestParameters()
+        {
+            NumMessages = 100;
+            Port = 5140;
+            Host = "localhost";
+            SleepTimeMilliseconds = 10;
+        }
+    }
+
+    class TcpTestGenerator
+    {
+        public static int Generate(TcpTestParameters parms)
+        {
+            TcpClient server = new TcpClient(parms.Host, parms.Port);
+            
+            var hostName = System.Environment.MachineName + "." +
+                       Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                           "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+
+            using (NetworkStream stream = server.GetStream())
+            {              
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"Application", "tcp-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "tcp"},
+                        {"Message", "tcp message " + DateTime.UtcNow.ToString("o")},
+                        {"Index", "logstash"}
+                    };
+                    byte[] data = Encoding.UTF8.GetBytes(string.Format("{0}\n", o.ToString()));
+                    stream.Write(data, 0, data.Length);
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+            }
+
+            return parms.NumMessages;
+        }
+
+    }
+}
--- a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
+++ b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
@@ -0,0 +1,124 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>TimberWinR.TestGenerator</RootNamespace>
+    <AssemblyName>TimberWinR.TestGenerator</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="CommandLine">
+      <HintPath>..\packages\CommandLineParser.1.9.71\lib\net40\CommandLine.dll</HintPath>
+    </Reference>
+    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="NLog, Version=3.2.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Common">
+      <HintPath>..\packages\ServiceStack.Common.Signed.4.0.38\lib\net40\ServiceStack.Common.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Interfaces">
+      <HintPath>..\packages\ServiceStack.Interfaces.4.0.38\lib\portable-wp80+sl5+net40+win8+monotouch+monoandroid\ServiceStack.Interfaces.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Redis">
+      <HintPath>..\packages\ServiceStack.Redis.Signed.4.0.38\lib\net40\ServiceStack.Redis.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Text">
+      <HintPath>..\packages\ServiceStack.Text.Signed.4.0.38\lib\net40\ServiceStack.Text.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CommandLineOptions.cs" />
+    <Compile Include="Dynamic.cs" />
+    <Compile Include="LogFileGenerator.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="RedisTestGenerator.cs" />
+    <Compile Include="TcpTestGenerator.cs" />
+    <Compile Include="JsonLogFileGenerator.cs" />
+    <Compile Include="UdpTestGenerator.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="default.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results1.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test1.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test1-twconfig.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <None Include="packages.config" />
+    <Content Include="test2-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">
+      <Project>{4ef96a08-21db-4178-be44-70dae594632c}</Project>
+      <Name>TimberWinR</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>
--- a/TimberWinR.TestGenerator/UdpTestGenerator.cs
+++ b/TimberWinR.TestGenerator/UdpTestGenerator.cs
@@ -0,0 +1,70 @@
+using System.Threading;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class UdpTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public UdpTestParameters()
+        {
+            NumMessages = 100;
+            Port = 6379;
+            Host = "localhost";
+            SleepTimeMilliseconds = 10;
+        }
+    }
+
+    class UdpTestGenerator
+    {
+        public static int Generate(UdpTestParameters parms)
+        {
+            var hostName = System.Environment.MachineName + "." +
+               Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                   "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+     
+            IPAddress broadcast;
+            if (!IPAddress.TryParse(parms.Host, out broadcast))
+                broadcast = Dns.GetHostEntry(parms.Host).AddressList[0];
+
+            Socket s = new Socket(broadcast.AddressFamily, SocketType.Dgram, ProtocolType.Udp);
+
+            LogManager.GetCurrentClassLogger().Info("Start UDP Generation");
+
+            for (int i = 0; i < parms.NumMessages; i++)
+            {
+                JObject o = new JObject
+                {
+                    {"Application", "udp-generator"},               
+                    {"Host", hostName},
+                    {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                    {"Type", "udp"},                
+                    {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
+                    {"Index", "logstash"}
+                };
+                byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString());
+                IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
+                s.SendTo(sendbuf, ep);
+                Thread.Sleep(parms.SleepTimeMilliseconds);
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished UDP Generation");
+
+            return parms.NumMessages;
+        }
+
+    }
+}
--- a/TimberWinR.TestGenerator/default.json
+++ b/TimberWinR.TestGenerator/default.json
@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "TailFiles": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/packages.config
+++ b/TimberWinR.TestGenerator/packages.config
@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="CommandLineParser" version="1.9.71" targetFramework="net40" />
+  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net40" />
+  <package id="NLog" version="3.2.0.0" targetFramework="net40" />
+  <package id="ServiceStack.Common.Signed" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Interfaces" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Redis.Signed" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Text.Signed" version="4.0.38" targetFramework="net40" />
+</packages>
--- a/TimberWinR.TestGenerator/results1.json
+++ b/TimberWinR.TestGenerator/results1.json
@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 20"                
+                }
+            },
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 1234",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 20"              
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/results2.json
+++ b/TimberWinR.TestGenerator/results2.json
@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            },
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 1234",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            }
+        ]
+    }
+}
--- a/TimberWinR.TestGenerator/test1-twconfig.json
+++ b/TimberWinR.TestGenerator/test1-twconfig.json
@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "TailFiles": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test1.json
+++ b/TimberWinR.TestGenerator/test1.json
@@ -0,0 +1,15 @@
+{
+    "test": "Test 1",
+    "arguments": {
+        "--testFile": "test1.json",
+        "--testDir": "test1",
+        "--timberWinRConfig": "test1-twconfig.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--udp-host":  "::1",
+        "--udp": "5140",
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results1.json"
+    }
+}
--- a/TimberWinR.TestGenerator/test2-tw.json
+++ b/TimberWinR.TestGenerator/test2-tw.json
@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "Logs": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}
--- a/TimberWinR.TestGenerator/test2.json
+++ b/TimberWinR.TestGenerator/test2.json
@@ -0,0 +1,14 @@
+{
+    "test": "Test 2",
+    "arguments": {
+        "--testFile": "test2.json",
+        "--testDir": "test2",
+        "--timberWinRConfig": "test2-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--udp": "5140",
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results2.json"
+    }
+}
--- a/TimberWinR.UnitTests/GrokFilterTests.cs
+++ b/TimberWinR.UnitTests/GrokFilterTests.cs
@@ -223,6 +223,7 @@ namespace TimberWinR.UnitTests
                    }
                },
                {"type", "Win32-FileLog"},
+                {"Type", "Win32-MyType"},
                {"ComputerName", "dev.mycompany.net"}
            };

@@ -281,11 +282,35 @@ namespace TimberWinR.UnitTests
                }
            }";

-            // Positive Tests
-            Configuration c = Configuration.FromString(grokJson1);
+            string grokJson4 = @"{  
+            ""TimberWinR"":{        
+                ""Filters"":[  
+	                {  
+		            ""grok"":{  
+		                ""condition"": ""!\""[Type]\"".StartsWith(\""[\"") && !\""[Type]\"".EndsWith(\""]\"") && (\""[type]\"" == \""Win32-FileLog\"")"",
+		                ""match"":[  
+			                ""Text"",
+			                """"
+		                ],
+                        ""remove_tag"":[  
+                            ""tag1""                          
+                        ]		               
+		            }
+	                }]
+                }
+            }";
+
+            
+            Configuration c = Configuration.FromString(grokJson4);
            Grok grok = c.Filters.First() as Grok;
            Assert.IsTrue(grok.Apply(json));

+
+            // Positive Tests
+            c = Configuration.FromString(grokJson1);
+            grok = c.Filters.First() as Grok;
+            Assert.IsTrue(grok.Apply(json));
+
            c = Configuration.FromString(grokJson2);
            grok = c.Filters.First() as Grok;
            Assert.IsTrue(grok.Apply(json));
--- a/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs
+++ b/TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs
@@ -1,60 +0,0 @@
-namespace TimberWinR.UnitTests.Inputs
-{
-    using System;
-    using System.Collections.Generic;
-
-    using Interop.MSUtil;
-
-    using Moq;
-
-    using NUnit.Framework;
-
-    using TimberWinR.Inputs;
-    using TimberWinR.Parser;
-
-    [TestFixture]
-    public class IisW3CRowReaderTests : TestBase
-    {
-        private IisW3CRowReader reader;
-
-        public override void Setup()
-        {
-            base.Setup();
-            var fields = new List<Field>
-                             {
-                                 new Field("date", "DateTime"),
-                                 new Field("time", "DateTime"),
-                                 new Field("uri")
-                             };
-            this.reader = new IisW3CRowReader(fields);
-
-            var recordset = this.GetRecordsetMock();
-            this.reader.ReadColumnMap(recordset.Object);
-        }
-
-        [Test]
-        public void GivenValidRowAddsTimestampColumn()
-        {
-            var record = this.MockRepository.Create<ILogRecord>();
-            record.Setup(x => x.getValue("date")).Returns(new DateTime(2014, 11, 30));
-            record.Setup(x => x.getValue("time")).Returns(new DateTime(1, 1, 1, 18, 45, 37, 590));
-            record.Setup(x => x.getValue("uri")).Returns("http://somedomain.com/someurl");
-
-            var json = this.reader.ReadToJson(record.Object);
-
-            Assert.AreEqual("2014-11-30T18:45:37.000Z", json["@timestamp"].ToString());
-            Assert.AreEqual("http://somedomain.com/someurl", json["uri"].ToString());
-        }
-
-        private Mock<ILogRecordset> GetRecordsetMock()
-        {
-            var recordset = this.MockRepository.Create<ILogRecordset>();
-            recordset.Setup(x => x.getColumnCount()).Returns(3);
-
-            recordset.Setup(x => x.getColumnName(0)).Returns("date");
-            recordset.Setup(x => x.getColumnName(1)).Returns("time");
-            recordset.Setup(x => x.getColumnName(2)).Returns("uri");
-            return recordset;
-        }
-    }
-}
--- a/TimberWinR.UnitTests/TailFileTests.cs
+++ b/TimberWinR.UnitTests/TailFileTests.cs
@@ -28,7 +28,7 @@ namespace TimberWinR.UnitTests
            var mgr = new Manager();
            mgr.LogfileDir = ".";

-            var tf = new TailFile();
+            var tf = new TailFileArguments();
            var cancelTokenSource = new CancellationTokenSource();
            tf.Location = "TestTailFile1.log";

--- a/TimberWinR.UnitTests/TestBase.cs
+++ b/TimberWinR.UnitTests/TestBase.cs
@@ -1,23 +0,0 @@
-namespace TimberWinR.UnitTests
-{
-    using Moq;
-
-    using NUnit.Framework;
-
-    public class TestBase
-    {
-        public MockRepository MockRepository { get; private set; }
-
-        [SetUp]
-        public virtual void Setup()
-        {
-            this.MockRepository = new MockRepository(MockBehavior.Default);
-        }
-
-        [TearDown]
-        public virtual void TearDown()
-        {
-            this.MockRepository.VerifyAll();
-        }
-    }
-}
--- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
+++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
@@ -12,6 +12,8 @@
    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <DebugSymbols>true</DebugSymbols>
@@ -38,16 +40,13 @@
      <EmbedInteropTypes>False</EmbedInteropTypes>
      <HintPath>..\TimberWinR\lib\com-logparser\Interop.MSUtil.dll</HintPath>
    </Reference>
-    <Reference Include="Moq">
-      <HintPath>..\packages\Moq.4.2.1409.1722\lib\net40\Moq.dll</HintPath>
-    </Reference>
    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <Private>True</Private>
    </Reference>
-    <Reference Include="nunit.framework, Version=2.6.3.13283, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\NUnit.2.6.3\lib\nunit.framework.dll</HintPath>
+    <Reference Include="nunit.framework">
+      <HintPath>..\packages\NUnit.2.6.4\lib\nunit.framework.dll</HintPath>
    </Reference>
    <Reference Include="System" />
    <Reference Include="System.Core" />
@@ -62,14 +61,12 @@
    <Compile Include="DateFilterTests.cs" />
    <Compile Include="FakeRediServer.cs" />
    <Compile Include="GeoIPFilterTests.cs" />
-    <Compile Include="Inputs\IisW3CRowReaderTests.cs" />
    <Compile Include="JsonFilterTests.cs" />
    <Compile Include="GrokFilterTests.cs" />
    <Compile Include="MultilineTests.cs" />
    <Compile Include="Parser\ElasticsearchOutputTests.cs" />
    <Compile Include="Properties\AssemblyInfo.cs" />
    <Compile Include="TailFileTests.cs" />
-    <Compile Include="TestBase.cs" />
    <Compile Include="TestDynamicBatchCount.cs" />
  </ItemGroup>
  <ItemGroup>
@@ -82,6 +79,7 @@
    <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
  </ItemGroup>
  <ItemGroup>
+    <None Include="app.config" />
    <None Include="packages.config">
      <SubType>Designer</SubType>
    </None>
@@ -94,7 +92,17 @@
      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
    </Content>
  </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Inputs\" />
+  </ItemGroup>
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
--- a/TimberWinR.UnitTests/app.config
+++ b/TimberWinR.UnitTests/app.config
@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.5.0.0" newVersion="4.5.0.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>
--- a/TimberWinR.UnitTests/packages.config
+++ b/TimberWinR.UnitTests/packages.config
@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<packages>
-  <package id="Moq" version="4.2.1409.1722" targetFramework="net40" />
-  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
-  <package id="NUnit" version="2.6.3" targetFramework="net40" />
+<packages> 
+  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net40" />
+  <package id="NUnit" version="2.6.4" targetFramework="net40" />
 </packages>
--- a/TimberWinR.sln
+++ b/TimberWinR.sln
@@ -7,6 +7,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR", "TimberWinR\Ti
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{2C4AC7DB-018D-4CCA-9579-06AC5AD2C9D9}"
 	ProjectSection(SolutionItems) = preProject
+		.nuget\NuGet.Config = .nuget\NuGet.Config
+		.nuget\NuGet.exe = .nuget\NuGet.exe
+		.nuget\NuGet.targets = .nuget\NuGet.targets
 		.nuget\packages.config = .nuget\packages.config
 	EndProjectSection
 EndProject
@@ -35,6 +38,8 @@ Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "TimberWinR.Wix", "TimberWix
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR.ExtractID", "TimberWinR.ExtractID\TimberWinR.ExtractID.csproj", "{99096939-E9DD-4499-883D-4726745A5843}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR.TestGenerator", "TimberWinR.TestGenerator\TimberWinR.TestGenerator.csproj", "{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -97,6 +102,16 @@ Global
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|x86.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|x86.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
--- a/TimberWinR/Configuration.cs
+++ b/TimberWinR/Configuration.cs
@@ -44,7 +44,7 @@ namespace TimberWinR
        {
            get { return _redisOutputs; }
        }
-
+       

        private List<ElasticsearchOutputParameters> _elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
        public IEnumerable<ElasticsearchOutputParameters> ElasticsearchOutputs
@@ -76,8 +76,8 @@ namespace TimberWinR
            get { return _logs; }
        }

-        private List<TailFile> _tails = new List<TailFile>();
-        public IEnumerable<TailFile> TailFiles
+        private List<TailFileArguments> _tails = new List<TailFileArguments>();
+        public IEnumerable<TailFileArguments> TailFiles
        {
            get { return _tails; }
        }    
@@ -241,8 +241,8 @@ namespace TimberWinR
                    c._stdins.AddRange(x.TimberWinR.Inputs.Stdins.ToList());
                if (x.TimberWinR.Inputs.Logs != null)
                    c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList());
-                if (x.TimberWinR.Inputs.TailFiles != null)
-                    c._tails.AddRange(x.TimberWinR.Inputs.TailFiles.ToList());
+                if (x.TimberWinR.Inputs.TailFilesArguments != null)
+                    c._tails.AddRange(x.TimberWinR.Inputs.TailFilesArguments.ToList());
                if (x.TimberWinR.Inputs.Tcps != null)
                    c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList());
                if (x.TimberWinR.Inputs.Udps != null)
--- a/TimberWinR/Diagnostics/Diagnostics.cs
+++ b/TimberWinR/Diagnostics/Diagnostics.cs
@@ -21,7 +21,7 @@ namespace TimberWinR.Diagnostics
        private CancellationToken CancelToken { get; set; }
        public int Port { get; set; }
        public Manager Manager { get; set; }
-
+        public bool Stop { get; set; }
        private HttpListener web;

        public Diagnostics(Manager manager, CancellationToken cancelToken, int port = 5141)
@@ -49,44 +49,60 @@ namespace TimberWinR.Diagnostics
        }


-        private void DiagnosticCallback(IAsyncResult result)
-        {       
-            if (web == null)         
-                return;
-
-            var context = web.EndGetContext(result);
-            var response = context.Response;
-
+        public JObject DiagnosticsOutput()
+        {
            JObject json = new JObject(
-                new JProperty("timberwinr",
-                    new JObject(
-                        new JProperty("version", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
-                        new JProperty("messages", Manager.NumMessages),
-                        new JProperty("startedon", Manager.StartedOn),
-                        new JProperty("configfile", Manager.JsonConfig),
-                        new JProperty("logdir", Manager.LogfileDir),
-                        new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
-                        new JProperty("inputs",
-                            new JArray(
-                                from i in Manager.Listeners
-                                select new JObject(i.ToJson()))),
-                        new JProperty("filters",
-                            new JArray(
-                                from f in Manager.Config.Filters
-                                select new JObject(f.ToJson()))),
-                        new JProperty("outputs",
-                            new JArray(
-                                from o in Manager.Outputs
-                                select new JObject(o.ToJson()))))));
-
-            response.StatusCode = (int)HttpStatusCode.OK;
-            response.StatusDescription = HttpStatusCode.OK.ToString();
-            byte[] buffer = Encoding.UTF8.GetBytes(json.ToString());
-            response.ContentLength64 = buffer.Length;
-            response.OutputStream.Write(buffer, 0, buffer.Length);
-            response.OutputStream.Close();
+                           new JProperty("timberwinr",
+                               new JObject(
+                                   new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                                   new JProperty("messages", Manager.NumMessages),
+                                   new JProperty("startedon", Manager.StartedOn),
+                                   new JProperty("configfile", Manager.JsonConfig),
+                                   new JProperty("logdir", Manager.LogfileDir),
+                                   new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
+                                   new JProperty("inputs",
+                                       new JArray(
+                                           from i in Manager.Listeners
+                                           select new JObject(i.ToJson()))),
+                                   new JProperty("filters",
+                                       new JArray(
+                                           from f in Manager.Config.Filters
+                                           select new JObject(f.ToJson()))),
+                                   new JProperty("outputs",
+                                       new JArray(
+                                           from o in Manager.Outputs
+                                           select new JObject(o.ToJson()))))));
+            return json;
        }

+        private void DiagnosticCallback(IAsyncResult result)
+        {
+            if (web == null)
+                return;
+
+            try
+            {
+                var context = web.EndGetContext(result);
+                var response = context.Response;
+                var json = DiagnosticsOutput();
+
+                response.StatusCode = (int) HttpStatusCode.OK;
+                response.StatusDescription = HttpStatusCode.OK.ToString();
+                byte[] buffer = Encoding.UTF8.GetBytes(json.ToString());
+                response.ContentLength64 = buffer.Length;
+                response.OutputStream.Write(buffer, 0, buffer.Length);
+                response.OutputStream.Close();
+            }
+            catch (SocketException)
+            {
+                // Shutdown
+            }
+            catch (Exception ex)
+            {
+                if (!Stop)
+                    LogManager.GetCurrentClassLogger().Error(ex);
+            }
+        }

        private void HttpListen(object o)
        {
@@ -97,14 +113,19 @@ namespace TimberWinR.Diagnostics
                web.Start();

                while (web != null && web.IsListening)
-                {                  
-                    processRequest();                  
-                }               
+                {
+                    processRequest();
+                }
+            }
+            catch (SocketException)
+            {
+                // Shutdown
            }
            catch (Exception ex)
            {
+                if (!Stop)
                LogManager.GetCurrentClassLogger().Error("Diagnostic Listener Error: {0}", ex.ToString());
-            }           
+            }
        }

        private void ListenForClients(object olistener)
@@ -140,7 +161,7 @@ namespace TimberWinR.Diagnostics
            var tcpClient = (TcpClient)client;
            NetworkStream clientStream = null;

-            Console.WriteLine("Handle new diag client: {0}, {1}", tcpClient.Connected, tcpClient.Client.RemoteEndPoint.ToString());
+            //     Console.WriteLine("Handle new diag client: {0}, {1}", tcpClient.Connected, tcpClient.Client.RemoteEndPoint.ToString());
            try
            {
                using (clientStream = tcpClient.GetStream())
@@ -183,7 +204,7 @@ namespace TimberWinR.Diagnostics

        public void Shutdown()
        {
-            
+            Stop = true;
            try
            {
                if (web != null && web.IsListening)
@@ -193,11 +214,9 @@ namespace TimberWinR.Diagnostics
                    web = null;
                }
            }
-            catch (Exception ex)
+            catch (Exception)
            {
-                LogManager.GetCurrentClassLogger().Error(ex);
            }
        }
-
    }
 }
--- a/TimberWinR/Filters/GrokFilter.cs
+++ b/TimberWinR/Filters/GrokFilter.cs
@@ -47,6 +47,7 @@ namespace TimberWinR.Parser
                         new JProperty("condition", Condition),
                         new JProperty("addfields", AddField),
                         new JProperty("addtags", AddTag),
+                         new JProperty("drop", DropIfMatch),
                         new JProperty("type", Type),
                         new JProperty("removefields", RemoveField),
                         new JProperty("removetag", RemoveTag)
--- a/TimberWinR/Inputs/InputListener.cs
+++ b/TimberWinR/Inputs/InputListener.cs
@@ -1,5 +1,6 @@
 using System.IO;
 using System.Runtime.InteropServices;
+using Microsoft.Win32;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -18,7 +19,9 @@ namespace TimberWinR.Inputs
        private string _typeName;
        public AutoResetEvent FinishedEvent { get; set; }
        public string CheckpointFileName { get; set; }
-      
+        private object _locker = new object();
+        public List<string> Files { get; set; }
+
        public string InputType
        {
            get { return _typeName; }
@@ -28,6 +31,7 @@ namespace TimberWinR.Inputs

        public InputListener(CancellationToken token, string typeName)
        {
+            Files = new List<string>();
            CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString()));          
          
            this.FinishedEvent = new AutoResetEvent(false);
@@ -40,6 +44,19 @@ namespace TimberWinR.Inputs
                             .ToString();    
        }

+        public bool HaveSeenFile(string fileName)
+        {
+            return Files.Contains(fileName);
+        }
+
+        protected void SaveVisitedFileName(string fileName)
+        {
+            lock (_locker)
+            {
+                if (!HaveSeenFile(fileName))
+                    Files.Add(fileName);
+            }
+        }
        protected string ToPrintable(string inputString)
        {
            string asAscii = Encoding.ASCII.GetString(
@@ -58,17 +75,17 @@ namespace TimberWinR.Inputs

        public void Finished()
        {
-            LogManager.GetCurrentClassLogger().Info("Signaling Event Shutdown {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Signalling Event Shutdown {1}", Thread.CurrentThread.ManagedThreadId, InputType);
            FinishedEvent.Set();
-            LogManager.GetCurrentClassLogger().Info("Finished signaling Shutdown {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Finished signalling Shutdown {1}", Thread.CurrentThread.ManagedThreadId, InputType);
        }
+
        public virtual void Shutdown()
        {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1}", Thread.CurrentThread.ManagedThreadId, InputType);

            FinishedEvent.WaitOne();
-
-            LogManager.GetCurrentClassLogger().Info("Finished Wait For {0}", InputType);
+          
            try
            {
                if (File.Exists(CheckpointFileName))
@@ -80,6 +97,32 @@ namespace TimberWinR.Inputs
            }          
        }

+        protected void EnsureRollingCaught()
+        {
+            try
+            {
+                const string mteKey = @"SYSTEM\CurrentControlSet\Control\FileSystem";
+
+                var mte = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(mteKey).GetValue("MaximumTunnelEntries");
+                if (mte == null || (int)mte != 0)
+                {
+                    LogManager.GetCurrentClassLogger()
+                        .Error(
+                            "HKLM\\{0}\\MaximumTunnelEntries is not set to accurately detect log rolling, a DWORD value of 0 is required.",
+                            mteKey);
+                    Microsoft.Win32.Registry.LocalMachine.CreateSubKey(mteKey).SetValue("MaximumTunnelEntries", 0, RegistryValueKind.DWord);
+                    LogManager.GetCurrentClassLogger()
+                      .Error(
+                          "HKLM\\{0}\\MaximumTunnelEntries is now set to 0, A reboot is now required to fix this issue.  See http://support.microsoft.com/en-us/kb/172190 for details",
+                          mteKey);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }           
+        }
+
        public virtual void AddDefaultFields(JObject json)
        {
            if (json["type"] == null)
--- a/TimberWinR/Inputs/LogsFileDatabase.cs
+++ b/TimberWinR/Inputs/LogsFileDatabase.cs
@@ -1,9 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using NLog;
 using TimberWinR.Parser;

@@ -34,7 +37,10 @@ namespace TimberWinR.Inputs
                return ExistingFileTest(logName);
            }
        }
-
+       
+        //
+        // Lookup the database entry for this log file, returns null if there isnt one.
+        //
        private LogsFileDatabaseEntry FindFile(string logName)
        {
            lock (_locker)
@@ -43,6 +49,7 @@ namespace TimberWinR.Inputs
                return existingEntry;
            }
        }
+
        private bool ExistingFileTest(string logName)
        {
            var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
@@ -67,45 +74,82 @@ namespace TimberWinR.Inputs
            var de = new LogsFileDatabaseEntry();
            lock (_locker)
            {
-                de.NewFile = true;
-                var fi = new FileInfo(logName);
+                var fi = new FileInfo(logName);             
                de.FileName = logName;
-                de.Size = fi.Length;
+                de.LogFileExists = fi.Exists;
+                de.NewFile = true;
+                de.ProcessedFile = false;
+                de.LastPosition = fi.Length;
                de.SampleTime = DateTime.UtcNow;
-                de.CreationTimeUtc = fi.CreationTimeUtc;
+                de.CreationTimeUtc = fi.CreationTimeUtc;               
+
                Entries.Add(de);
                WriteDatabaseFileNoLock();
            }
            return de;
-        }
+        }      

        public static LogsFileDatabaseEntry LookupLogFile(string logName)
        {
            LogsFileDatabaseEntry dbe = Instance.FindFile(logName);
            if (dbe == null)
                dbe = Instance.AddFileEntry(logName);
-            else
-                dbe.NewFile = false;
-    
+
+            FileInfo fi = new FileInfo(logName);          
+
+            dbe.LogFileExists = fi.Exists;
+            var creationTime = fi.CreationTimeUtc;
+
+            if (dbe.LogFileExists && creationTime != dbe.CreationTimeUtc)
+                dbe.NewFile = true;           
+
+            dbe.CreationTimeUtc = creationTime;
+      
            return dbe;
        }

-        public static void Update(LogsFileDatabaseEntry dbe)
+        // Find all the non-existent entries and remove them.
+        private void PruneFiles()
        {
-            Instance.UpdateEntry(dbe);
-        }
-
-        private void UpdateEntry(LogsFileDatabaseEntry dbe)
-        {
-            lock(_locker)
+            lock (_locker)
            {
-                var fi = new FileInfo(dbe.FileName);
-                dbe.CreationTimeUtc = fi.CreationTimeUtc;
-                dbe.SampleTime = DateTime.UtcNow;
-                dbe.Size = fi.Length;
+                foreach(var entry in Entries.ToList())
+                {
+                    FileInfo fi = new FileInfo(entry.FileName);
+                    if (!fi.Exists)
+                        Entries.Remove(entry);
+                }
+                WriteDatabaseFileNoLock();
+            }
+        }
+
+        public static void Update(LogsFileDatabaseEntry dbe, bool processedFile, long lastOffset)
+        {
+            dbe.ProcessedFile = processedFile;
+            dbe.LogFileExists = File.Exists(dbe.FileName);
+            Instance.UpdateEntry(dbe, lastOffset);           
+        }
+
+        public static void Roll(LogsFileDatabaseEntry dbe)
+        {
+            dbe.ProcessedFile = false;
+            dbe.LastPosition = 0;          
+            Instance.UpdateEntry(dbe, 0);
+            dbe.NewFile = true;           
+        }
+
+        private void UpdateEntry(LogsFileDatabaseEntry dbe, long lastOffset)
+        {
+            lock (_locker)
+            {
+                var fi = new FileInfo(dbe.FileName);                
+                dbe.NewFile = !fi.Exists;
+                dbe.CreationTimeUtc = fi.CreationTimeUtc;
+                dbe.SampleTime = DateTime.UtcNow;                 
+                dbe.LastPosition = lastOffset;
+              
                WriteDatabaseFileNoLock();
            }
-            
        }
        public static LogsFileDatabase Instance
        {
@@ -123,12 +167,19 @@ namespace TimberWinR.Inputs
                            instance.ReadDatabaseNoLock();
                        else
                            instance.WriteDatabaseFileNoLock();
+
+                        if (instance.Entries == null)
+                            instance.Entries = new List<LogsFileDatabaseEntry>();
+
+                        instance.PruneFiles();
                    }
                }
                return instance;
            }
        }

+
+        // Serialize in the Database
        private void ReadDatabaseNoLock()
        {
            try
@@ -152,7 +203,7 @@ namespace TimberWinR.Inputs
                catch (Exception ex2)
                {
                    LogManager.GetCurrentClassLogger().Info("Error Creating New Database '{0}': {1}", DatabaseFileName, ex2.ToString());
-                }              
+                }
            }
        }
        private void WriteDatabaseFileNoLock()
@@ -193,15 +244,30 @@ namespace TimberWinR.Inputs

    }

+
+    //
+    // Represents a log file to be tailed
+    // 
    public class LogsFileDatabaseEntry
    {
        [JsonIgnore]
        public bool NewFile { get; set; }
-        public string FileName { get; set; }
-        public Int64 MaxRecords { get; set; }
+        public bool ProcessedFile { get; set; }
+        public bool LogFileExists { get; set; }
+        public string FileName { get; set; }       
        public DateTime CreationTimeUtc { get; set; }
        public DateTime SampleTime { get; set; }
-        public long Size { get; set; }       
+        public long LastPosition { get; set; }
+        public long LinesProcessed
+        {
+            get { return _linesProcessed; }
+        }
+
+        private int _linesProcessed;
+        public void IncrementLineCount()
+        {
+            Interlocked.Increment(ref _linesProcessed);
+        }
    }

 }
--- a/TimberWinR/Inputs/LogsListener.cs
+++ b/TimberWinR/Inputs/LogsListener.cs
@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Net.Configuration;
 using System.Runtime.InteropServices;
@@ -9,6 +10,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.IO;
 using Interop.MSUtil;
+using Microsoft.Win32;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;
@@ -27,42 +29,45 @@ namespace TimberWinR.Inputs
    /// </summary>
    public class LogsListener : InputListener
    {
+        private object _locker = new object();
        private int _pollingIntervalInSeconds;
        private TimberWinR.Parser.LogParameters _arguments;
        private long _receivedMessages;
-        private Dictionary<string, Int64> _logFileMaxRecords;
-        private Dictionary<string, DateTime> _logFileCreationTimes;
-        private Dictionary<string, DateTime> _logFileSampleTimes;
-        private Dictionary<string, long> _logFileSizes;
-        private CodecArguments _codecArguments;       
-        private ICodec _codec;   
-
+        private CodecArguments _codecArguments;
+        private ICodec _codec;
+      
        public bool Stop { get; set; }
+        public bool IsWildcardFilePattern { get; set; }
+

        public LogsListener(TimberWinR.Parser.LogParameters arguments, CancellationToken cancelToken)
            : base(cancelToken, "Win32-FileLog")
        {
            Stop = false;
+        
+            EnsureRollingCaught();

-            _codecArguments = arguments.CodecArguments;
+             _codecArguments = arguments.CodecArguments;

            _codecArguments = arguments.CodecArguments;
            if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                _codec = new Multiline(_codecArguments);

-            _logFileMaxRecords = new Dictionary<string, Int64>();
-            _logFileCreationTimes = new Dictionary<string, DateTime>();
-            _logFileSampleTimes = new Dictionary<string, DateTime>();
-            _logFileSizes = new Dictionary<string, long>();
-
            _receivedMessages = 0;
            _arguments = arguments;
            _pollingIntervalInSeconds = arguments.Interval;

+            IsWildcardFilePattern = arguments.Location.Contains('*');
+
            foreach (string srcFile in _arguments.Location.Split(','))
            {
                string file = srcFile.Trim();
-                Task.Factory.StartNew(() => FileWatcher(file));
+                string dir = Path.GetDirectoryName(file);
+                if (string.IsNullOrEmpty(dir))
+                    dir = Environment.CurrentDirectory;
+                string fileSpec = Path.Combine(dir, file);
+
+                Task.Factory.StartNew(() => FileWatcher(fileSpec));
            }
        }

@@ -73,9 +78,9 @@ namespace TimberWinR.Inputs
            base.Shutdown();
        }

+      
        public override JObject ToJson()
        {
-
            JObject json = new JObject(
                new JProperty("log",
                    new JObject(
@@ -86,21 +91,11 @@ namespace TimberWinR.Inputs
                        new JProperty("codepage", _arguments.CodePage),
                        new JProperty("splitLongLines", _arguments.SplitLongLines),
                        new JProperty("recurse", _arguments.Recurse),
-
+                        new JProperty("filedb",
+                            new JArray(from f in Files.ToList()
+                                       select JObject.FromObject(LogsFileDatabase.LookupLogFile(f)))),
                        new JProperty("files",
-                            new JArray(from f in _logFileMaxRecords.Keys
-                                       select new JValue(f))),
-                        new JProperty("fileSampleTimes",
-                            new JArray(from f in _logFileSampleTimes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileSizes",
-                            new JArray(from f in _logFileSizes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileIndices",
-                            new JArray(from f in _logFileMaxRecords.Values
-                                       select new JValue(f))),
-                        new JProperty("fileCreationDates",
-                            new JArray(from f in _logFileCreationTimes.Values
+                            new JArray(from f in Files.ToList()
                                       select new JValue(f)))
                        )));

@@ -120,7 +115,7 @@ namespace TimberWinR.Inputs


            return json;
-        }   
+        }

        private void FileWatcher(string fileToWatch)
        {
@@ -149,46 +144,34 @@ namespace TimberWinR.Inputs
                            {
                                var record = rsfiles.getRecord();
                                string logName = record.getValue("LogFilename") as string;
-                                FileInfo fi = new FileInfo(logName);
+                                FileInfo fi = new FileInfo(logName);                             

-                                if (!fi.Exists)
-                                {
-                                    _logFileCreationTimes.Remove(logName);
-                                    _logFileMaxRecords.Remove(logName);
-                                    _logFileSizes.Remove(logName);
-                                }
-
-                                _logFileSampleTimes[logName] = DateTime.UtcNow;
+                                var dbe = LogsFileDatabase.LookupLogFile(logName);
+                    
+                                SaveVisitedFileName(dbe.FileName);

                                DateTime creationTime = fi.CreationTimeUtc;
-                                bool logHasRolled = (_logFileCreationTimes.ContainsKey(logName) &&
-                                                     creationTime > _logFileCreationTimes[logName]) ||
-                                                    (_logFileSizes.ContainsKey(logName) &&
-                                                     fi.Length < _logFileSizes[logName]);
+                                bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition);

-
-                                if (!_logFileMaxRecords.ContainsKey(logName) || logHasRolled)
+                                if (logHasRolled)
                                {
-                                    _logFileCreationTimes[logName] = creationTime;
-                                    _logFileSizes[logName] = fi.Length;
-                                    var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
-                                    var rcount = oLogQuery.Execute(qcount, iFmt);
-                                    var qr = rcount.getRecord();
-                                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
-                                    if (logHasRolled)
-                                    {
-                                        LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
-                                        lrn = 0;
-                                    }
-                                    _logFileMaxRecords[logName] = lrn;
+                                    LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
+                                    LogsFileDatabase.Roll(dbe);
                                }

-                                _logFileSizes[logName] = fi.Length;
+                                // Log has rolled or this is a new file, or we haven't processed yet.
+                                bool processWholeFile = logHasRolled || !dbe.ProcessedFile;
+
+                                if (processWholeFile)
+                                    LogsFileDatabase.Update(dbe, true, 0);
+
                            }
                            rsfiles.close();
-                            foreach (string fileName in _logFileMaxRecords.Keys.ToList())
+                            foreach (string fileName in Files.ToList())
                            {
-                                var lastRecordNumber = _logFileMaxRecords[fileName];
+                                var dbe = LogsFileDatabase.LookupLogFile(fileName);
+
+                                var lastRecordNumber = dbe.LastPosition;
                                var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName,
                                    lastRecordNumber);

@@ -231,21 +214,22 @@ namespace TimberWinR.Inputs
                                    string msg = json["Text"].ToString();
                                    if (!string.IsNullOrEmpty(msg))
                                    {
-                                        if (_codecArguments != null &&
-                                            _codecArguments.Type == CodecArguments.CodecType.multiline)
+                                        if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                                        {
                                            _codec.Apply(msg, this);
                                            _receivedMessages++;
+                                            dbe.IncrementLineCount();
                                        }
                                        else
                                        {
                                            ProcessJson(json);
-                                            _receivedMessages++;
+                                            dbe.IncrementLineCount();
+                                             _receivedMessages++;
                                        }
                                    }

                                    var lrn = (Int64)record.getValueEx("Index");
-                                    _logFileMaxRecords[fileName] = lrn;
+                                    LogsFileDatabase.Update(dbe, true, lrn);
                                    GC.Collect();
                                }

@@ -254,16 +238,18 @@ namespace TimberWinR.Inputs
                                rs.close();
                                rs = null;
                                GC.Collect();
+
                            }
                        }
                        catch (FileNotFoundException fnfex)
                        {
                            string fn = fnfex.FileName;

-                            if (!_fnfmap.ContainsKey(fn))
+                            if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn))
+                            {
                                LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
-
-                            _fnfmap[fn] = fn;
+                                _fnfmap[fn] = fn;
+                            }
                        }
                        catch (OperationCanceledException)
                        {
@@ -283,7 +269,7 @@ namespace TimberWinR.Inputs
                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                            }
                            catch (OperationCanceledException)
-                            {                               
+                            {
                            }
                            catch (Exception ex1)
                            {
--- a/TimberWinR/Inputs/StdinListener.cs
+++ b/TimberWinR/Inputs/StdinListener.cs
@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -15,10 +16,15 @@ namespace TimberWinR.Inputs
 {
    public class StdinListener : InputListener
    {
+        [DllImport("User32.Dll", EntryPoint = "PostMessageA")]
+        private static extern bool PostMessage(IntPtr hWnd, uint msg, int wParam, int lParam);
+  
        private Thread _listenThread;
        private CodecArguments _codecArguments;
-        private ICodec _codec;     
-       
+        private ICodec _codec;
+        const int VK_RETURN = 0x0D;
+        const int WM_KEYDOWN = 0x100;
+
        public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken)
            : base(cancelToken, "Win32-Console")
        {
@@ -54,7 +60,14 @@ namespace TimberWinR.Inputs

        public override void Shutdown()
        {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);    
+            // This must come from another thread.
+            ThreadPool.QueueUserWorkItem((o) =>
+            {
+                Thread.Sleep(100);
+                var hWnd = System.Diagnostics.Process.GetCurrentProcess().MainWindowHandle;
+                PostMessage(hWnd, WM_KEYDOWN, VK_RETURN, 0);
+            });
            base.Shutdown();
        }

--- a/TimberWinR/Inputs/TailFileListener.cs
+++ b/TimberWinR/Inputs/TailFileListener.cs
@@ -26,33 +26,27 @@ namespace TimberWinR.Inputs
    /// </summary>
    public class TailFileListener : InputListener
    {
+        private object _locker = new object();
        private int _pollingIntervalInSeconds;
-        private TimberWinR.Parser.TailFile _arguments;
+        private TimberWinR.Parser.TailFileArguments _arguments;
        private long _receivedMessages;
-        private Dictionary<string, Int64> _logFileMaxRecords;
-        private Dictionary<string, DateTime> _logFileCreationTimes;
-        private Dictionary<string, DateTime> _logFileSampleTimes;
-        private Dictionary<string, long> _logFileSizes;       
-        private CodecArguments _codecArguments;
-        private ICodec _codec;     

+        private CodecArguments _codecArguments;
+        private ICodec _codec;
+       
        public bool Stop { get; set; }

-        public TailFileListener(TimberWinR.Parser.TailFile arguments, CancellationToken cancelToken)
+        public TailFileListener(TimberWinR.Parser.TailFileArguments arguments, CancellationToken cancelToken)
            : base(cancelToken, "Win32-TailLog")
        {
            Stop = false;
+         
+            EnsureRollingCaught();    

            _codecArguments = arguments.CodecArguments;
            if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                _codec = new Multiline(_codecArguments);

-
-            _logFileMaxRecords = new Dictionary<string, Int64>();
-            _logFileCreationTimes = new Dictionary<string, DateTime>();
-            _logFileSampleTimes = new Dictionary<string, DateTime>();
-            _logFileSizes = new Dictionary<string, long>();
-
            _receivedMessages = 0;
            _arguments = arguments;
            _pollingIntervalInSeconds = arguments.Interval;
@@ -66,7 +60,7 @@ namespace TimberWinR.Inputs

        public override void Shutdown()
        {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1} for {2}", Thread.CurrentThread.ManagedThreadId, InputType, _arguments.Location);
            Stop = true;
            base.Shutdown();
        }
@@ -74,29 +68,19 @@ namespace TimberWinR.Inputs
        public override JObject ToJson()
        {
            JObject json = new JObject(
-                new JProperty("log",
+                new JProperty("taillog",
                    new JObject(
                        new JProperty("messages", _receivedMessages),
                        new JProperty("type", InputType),
                        new JProperty("location", _arguments.Location),
                        new JProperty("logSource", _arguments.LogSource),
                        new JProperty("recurse", _arguments.Recurse),
-
                        new JProperty("files",
-                            new JArray(from f in _logFileMaxRecords.Keys
+                            new JArray(from f in Files
                                       select new JValue(f))),
-                        new JProperty("fileSampleTimes",
-                            new JArray(from f in _logFileSampleTimes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileSizes",
-                            new JArray(from f in _logFileSizes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileIndices",
-                            new JArray(from f in _logFileMaxRecords.Values
-                                       select new JValue(f))),
-                        new JProperty("fileCreationDates",
-                            new JArray(from f in _logFileCreationTimes.Values
-                                       select new JValue(f)))
+                        new JProperty("filedb",
+                            new JArray(from f in Files
+                                       select JObject.FromObject(LogsFileDatabase.LookupLogFile(f))))
                        )));


@@ -117,10 +101,9 @@ namespace TimberWinR.Inputs
            return json;
        }

-        private void TailFileContents(string fileName, long offset)
+        private void TailFileContents(string fileName, long offset, LogsFileDatabaseEntry dbe)
        {
-            using (StreamReader reader = new StreamReader(new FileStream(fileName,
-                     FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
+            using (StreamReader reader = new StreamReader(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
            {
                //start at the end of the file
                long lastMaxOffset = offset;
@@ -130,6 +113,8 @@ namespace TimberWinR.Inputs
                    return;

                //seek to the last max offset
+                LogManager.GetCurrentClassLogger().Trace("{0}: File: {1} Seek to: {2}", Thread.CurrentThread.ManagedThreadId, fileName, lastMaxOffset);
+
                reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin);

                //read out of the file until the EOF
@@ -152,24 +137,31 @@ namespace TimberWinR.Inputs
                        else
                            json.Add(new JProperty("logSource", _arguments.LogSource));
                    }
+
                    json["Text"] = line;
                    json["Index"] = index;
                    json["LogFileName"] = fileName;

+
                    if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                    {
                        _codec.Apply(line, this);
                        Interlocked.Increment(ref _receivedMessages);
+                        dbe.IncrementLineCount();
                    }
                    else
                    {
                        ProcessJson(json);
                        Interlocked.Increment(ref _receivedMessages);
-                    }
-                    lineOffset += line.Length;                   
+                        dbe.IncrementLineCount();
+                        //LogManager.GetCurrentClassLogger().Info("{0}: File: {1} {2} {3}", Thread.CurrentThread.ManagedThreadId, fileName, dbe.LinesProcessed, line);
+                    }                 
+
+                    lineOffset += line.Length;
                }
                //update the last max offset
                lastMaxOffset = reader.BaseStream.Position;
+                LogsFileDatabase.Update(dbe, true, lastMaxOffset);               
            }
        }
        // One thread for each kind of file to watch, i.e. "*.log,*.txt" would be two separate
@@ -187,11 +179,14 @@ namespace TimberWinR.Inputs
                    {
                        if (!CancelToken.IsCancellationRequested)
                        {
+                            var isWildcardPattern = fileToWatch.Contains('*');
                            string path = Path.GetDirectoryName(fileToWatch);
                            string name = Path.GetFileName(fileToWatch);
                            if (string.IsNullOrEmpty(path))
                                path = ".";

+                            LogManager.GetCurrentClassLogger().Trace(":{0} Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, Path.Combine(path, name));
+
                            // Ok, we have a potential file filter here as 'fileToWatch' could be foo.log or *.log

                            SearchOption so = SearchOption.TopDirectoryOnly;
@@ -201,26 +196,41 @@ namespace TimberWinR.Inputs
                            foreach (string fileName in Directory.GetFiles(path, name, so))
                            {
                                var dbe = LogsFileDatabase.LookupLogFile(fileName);
-                                FileInfo fi = new FileInfo(dbe.FileName);
-                                //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);
-                                long length = fi.Length;
-                                bool logHasRolled = false;
-                                if (fi.Length < dbe.Size || fi.CreationTimeUtc != dbe.CreationTimeUtc)
+
+                                // We only spin up 1 thread for a file we haven't yet seen.                               
+                                if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile)
                                {
-                                    LogManager.GetCurrentClassLogger().Info("Log has Rolled: {0}", dbe.FileName);
-                                    logHasRolled = true;
+                                    LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                    LogsFileDatabase.Update(dbe, false, dbe.LastPosition);
+                                    SaveVisitedFileName(fileName);
+                                    Task.Factory.StartNew(() => TailFileWatcher(fileName));
                                }
-                                bool processWholeFile = logHasRolled || dbe.NewFile;
-                                if (processWholeFile)
+                                else if (!isWildcardPattern)
                                {
-                                    LogManager.GetCurrentClassLogger().Info("Process Whole File: {0}", dbe.FileName);
-                                    TailFileContents(dbe.FileName, 0);
+                                    FileInfo fi = new FileInfo(dbe.FileName);
+
+                                    //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);                                
+                                    long length = fi.Length;
+                                    bool logHasRolled = false;
+                                    if (fi.Length < dbe.LastPosition || fi.CreationTimeUtc != dbe.CreationTimeUtc)
+                                    {
+                                        LogManager.GetCurrentClassLogger().Info("{0}: Log has Rolled: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                        logHasRolled = true;
+                                        LogsFileDatabase.Roll(dbe);
+                                    }
+                                    // Log has rolled or this is a file we are seeing for the first time.
+                                    bool processWholeFile = logHasRolled || !dbe.ProcessedFile;
+                                    if (processWholeFile)
+                                    {
+                                        LogsFileDatabase.Update(dbe, true, 0);
+                                        LogManager.GetCurrentClassLogger().Debug("{0}: Process Whole File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                        TailFileContents(dbe.FileName, 0, dbe);
+                                    }
+                                    else
+                                    {
+                                        TailFileContents(dbe.FileName, dbe.LastPosition, dbe);
+                                    }                                  
                                }
-                                else
-                                {
-                                    TailFileContents(dbe.FileName, dbe.Size);
-                                }
-                                LogsFileDatabase.Update(dbe);
                            }
                        }
                    }
@@ -232,6 +242,10 @@ namespace TimberWinR.Inputs
                            LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
                        _fnfmap[fn] = fn;
                    }
+                    catch (IOException ioex)
+                    {
+                        LogManager.GetCurrentClassLogger().Debug("Log has rolled: {0}", ioex.Message);
+                    }
                    catch (OperationCanceledException)
                    {
                        break;
@@ -249,16 +263,17 @@ namespace TimberWinR.Inputs
                        }
                        catch (OperationCanceledException)
                        {
+                            Stop = true;
                        }
                        catch (Exception ex1)
                        {
                            LogManager.GetCurrentClassLogger().Warn(ex1);
-                        }                      
+                        }
                    }
                }
            }
            Finished();
-        }
+        }       
    }
 }

--- a/TimberWinR/Inputs/TcpInputListener.cs
+++ b/TimberWinR/Inputs/TcpInputListener.cs
@@ -52,7 +52,7 @@ namespace TimberWinR.Inputs

        public override void Shutdown()
        {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1}", Thread.CurrentThread.ManagedThreadId, InputType);

            this._tcpListenerV4.Stop();
            this._tcpListenerV6.Stop();
@@ -113,6 +113,10 @@ namespace TimberWinR.Inputs
                            }
                            catch (Exception ex)
                            {
+                                var jex1 = LogErrors.LogException("Bad Json", ex);
+                                if (jex1 != null)
+                                    ProcessJson(jex1);
+
                                LogManager.GetCurrentClassLogger().Warn(ex);
                            }

--- a/TimberWinR/Inputs/UdpInputListener.cs
+++ b/TimberWinR/Inputs/UdpInputListener.cs
@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Net;
 using System.Net.Sockets;
@@ -11,24 +12,14 @@ using NLog;
 namespace TimberWinR.Inputs
 {
    public class UdpInputListener : InputListener
-    {
-        private readonly System.Net.Sockets.UdpClient _udpListener;
-        private readonly IPEndPoint groupV4;
-        private readonly IPEndPoint groupV6;
-
-        private Thread _listenThreadV4;
-        private Thread _listenThreadV6;
+    {      
+        private UdpClient _udpListenerV6;      
+        private readonly Thread _listenThreadV6;

        private readonly int _port;
        private long _receivedMessages;
        private long _parsedErrors;
-
-        private struct listenProfile
-        {
-            public IPEndPoint endPoint;
-            public UdpClient client;
-        }
-
+       
        public override JObject ToJson()
        {
            JObject json = new JObject(
@@ -46,36 +37,46 @@ namespace TimberWinR.Inputs
            : base(cancelToken, "Win32-Udp")
        {
            _port = port;
-
-            groupV4 = new IPEndPoint(IPAddress.Any, 0);
-            groupV6 = new IPEndPoint(IPAddress.IPv6Any, 0);
-
-            LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port);
-
+     
+ 
            _receivedMessages = 0;

-            _udpListener = new System.Net.Sockets.UdpClient(port);
-
-            _listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener));
-            _listenThreadV4.Start(new listenProfile() { endPoint = groupV4, client = _udpListener });
-
-            _listenThreadV6 = new Thread(new ParameterizedThreadStart(StartListener));
-            _listenThreadV6.Start(new listenProfile() { endPoint = groupV6, client = _udpListener });
+            _listenThreadV6 = new Thread(StartListener);
+            _listenThreadV6.Start();
        }


        public override void Shutdown()
        {
            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
-            _udpListener.Close();
-            Finished();
+
+            // close UDP listeners, which will end the listener threads          
+            _udpListenerV6.Close();
+
+            // wait for completion of the threads         
+            _listenThreadV6.Join();          
+
            base.Shutdown();
        }

+        private void StartListener()
+        {                      
+            var groupV6 = new IPEndPoint(IPAddress.IPv6Any, _port);
+            // Create the socket as IPv6
+            var dualModeSocket = new Socket(AddressFamily.InterNetworkV6, SocketType.Dgram, ProtocolType.Udp);
+            
+            //
+            // Now, disable the IPV6only flag to make it compatable with both ipv4 and ipv6
+            // See: http://blogs.msdn.com/b/malarch/archive/2005/11/18/494769.aspx
+            //
+            dualModeSocket.SetSocketOption(SocketOptionLevel.IPv6, SocketOptionName.IPv6Only, 0);
+            dualModeSocket.Bind(groupV6);
+
+            _udpListenerV6 = new UdpClient();
+            _udpListenerV6.Client = dualModeSocket;
+
+            LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", groupV6);

-        private void StartListener(object useProfile)
-        {
-            var profile = (listenProfile)useProfile;
            string lastMessage = "";
            try
            {
@@ -83,21 +84,30 @@ namespace TimberWinR.Inputs
                {
                    try
                    {
-                        byte[] bytes = profile.client.Receive(ref profile.endPoint);  
-                        var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
+                        byte[] bytes = _udpListenerV6.Receive(ref groupV6);  
+                        var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);                                                
                        lastMessage = data;
-                        JObject json = JObject.Parse(data);
+                        var json = JObject.Parse(data);
                        ProcessJson(json);
-                        _receivedMessages++;
+                        Interlocked.Increment(ref _receivedMessages);                       
                    }
-                    catch (Exception ex1)
-                    {
+                    catch(SocketException)
+                    {                       
+                        break;
+                    }
+                    catch (Exception ex)
+                    {                      
+                        var jex1 = LogErrors.LogException(string.Format("Invalid JSON: {0}", lastMessage), ex);
+                        if (jex1 != null)
+                            ProcessJson(jex1);
+      
                        LogManager.GetCurrentClassLogger().Warn("Bad JSON: {0}", lastMessage);
-                        LogManager.GetCurrentClassLogger().Warn(ex1);
-                        _parsedErrors++;
+                        LogManager.GetCurrentClassLogger().Warn(ex);
+
+                        Interlocked.Increment(ref _parsedErrors);
                    }
                }
-                _udpListener.Close();
+                _udpListenerV6.Close();
            }
            catch (Exception ex)
            {
--- a/TimberWinR/LogErrors.cs
+++ b/TimberWinR/LogErrors.cs
@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+using TimberWinR.Parser;
+
+namespace TimberWinR
+{
+    public class LogErrors
+    {
+        public static JObject LogException(Exception ex)
+        {
+            return LogException("Exception", ex);
+        }
+
+        public static JObject LogException(string errorMessage, Exception ex)
+        {
+            JObject result = new JObject();
+            result["type"] = "TimberWinR-Error";
+            result["ErrorMessage"] = errorMessage;
+            
+            try
+            {  
+                JsonConvert.DefaultSettings = () => new JsonSerializerSettings
+                {
+                    Formatting = Formatting.Indented,
+                    NullValueHandling = NullValueHandling.Ignore
+                };
+
+                var exJson = JObject.Parse(JsonConvert.SerializeObject(ex));
+
+                result.Merge(exJson, new JsonMergeSettings
+                {
+                    MergeArrayHandling = MergeArrayHandling.Replace
+                });
+                return result;
+            }
+            catch (Exception ex1)
+            {
+                result["ErrorMessage"] = ex1.ToString();
+                return result;
+            }           
+        }
+    }
+}
--- a/TimberWinR/Manager.cs
+++ b/TimberWinR/Manager.cs
@@ -28,6 +28,8 @@ namespace TimberWinR
        public List<InputListener> Listeners { get; set; }
        public bool LiveMonitor { get; set; }

+        public event Action<Configuration> OnConfigurationProcessed;
+
        public DateTime StartedOn { get; set; }
        public string JsonConfig { get; set; }
        public string LogfileDir { get; set; }
@@ -67,7 +69,7 @@ namespace TimberWinR
            LogsFileDatabase.Manager = this;                   
        }

-        public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken)
+        public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken, bool processConfiguration = true)
        {
            LogsFileDatabase.Manager = this;           
  
@@ -106,12 +108,14 @@ namespace TimberWinR

            LogManager.GlobalThreshold = LogLevel.FromString(logLevel);

-            LogManager.GetCurrentClassLogger()
-                .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());
-
+            //LogManager.GetCurrentClassLogger()
+            //    .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());

            LogManager.GetCurrentClassLogger()
-                .Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
+                .Info("TimberWinR Version {0}", Assembly.GetEntryAssembly().GetName().Version.ToString());
+
+            LogManager.GetCurrentClassLogger()
+                .Info("Database Filename: {0}", LogsFileDatabase.Instance.DatabaseFileName);

            try
            {
@@ -146,7 +150,15 @@ namespace TimberWinR
            LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
            LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);

-            ProcessConfiguration(cancelToken, Config);
+            if (processConfiguration)
+            {
+                ProcessConfiguration(cancelToken, Config);               
+            }
+        }
+
+        public void Start(CancellationToken cancelToken)
+        {
+            ProcessConfiguration(cancelToken, Config);          
        }

        public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
@@ -154,6 +166,9 @@ namespace TimberWinR
 // Read the Configuration file
            if (config != null)
            {
+                if (OnConfigurationProcessed != null)
+                    OnConfigurationProcessed(config);
+  
                if (config.RedisOutputs != null)
                {
                    foreach (var ro in config.RedisOutputs)
@@ -256,7 +271,8 @@ namespace TimberWinR
                        new JProperty("TimberWinR",
                            new JObject(
                                new JProperty("version",
-                                    GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+                                    Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                                    //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
                                new JProperty("host", computerName),
                                new JProperty("output", output.Name),
                                new JProperty("initialized", DateTime.UtcNow)
@@ -264,7 +280,7 @@ namespace TimberWinR
                    json.Add(new JProperty("type", "Win32-TimberWinR"));
                    json.Add(new JProperty("host", computerName));
                    output.Startup(json);
-                }
+                }              
            }
        }

--- a/TimberWinR/Outputs/Elasticsearch.cs
+++ b/TimberWinR/Outputs/Elasticsearch.cs
@@ -268,7 +268,7 @@ namespace TimberWinR.Outputs
                ApplyFilters(jsonMessage);

            var message = jsonMessage.ToString();
-            LogManager.GetCurrentClassLogger().Debug(message);
+            LogManager.GetCurrentClassLogger().Trace(message);

            lock (_locker)
            {
--- a/TimberWinR/Outputs/Redis.cs
+++ b/TimberWinR/Outputs/Redis.cs
@@ -6,7 +6,7 @@ using System.Linq.Expressions;
 using System.Net.Sockets;
 using System.Text;
 using System.Threading;
-using ctstone.Redis;
+using CSRedis;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
@@ -28,7 +28,7 @@ namespace TimberWinR.Outputs
        private const int QUEUE_SAMPLE_SIZE = 30; // 30 samples over 2.5 minutes (default)
        private object _locker = new object();
        private bool _warnedReachedMax;
-    
+
        private readonly int _maxBatchCount;
        private readonly int _batchCount;
        private int _totalSamples;
@@ -53,7 +53,7 @@ namespace TimberWinR.Outputs
            {
                if (_totalSamples < QUEUE_SAMPLE_SIZE)
                    _totalSamples++;
-                
+
                // Take a sample of the queue depth
                if (_sampleCountIndex >= QUEUE_SAMPLE_SIZE)
                    _sampleCountIndex = 0;
@@ -69,7 +69,7 @@ namespace TimberWinR.Outputs
                if (_totalSamples > 0)
                {
                    var samples = _sampleQueueDepths.Take(_totalSamples);
-                    int avg = (int) samples.Average();
+                    int avg = (int)samples.Average();
                    return avg;
                }
                return 0;
@@ -81,7 +81,7 @@ namespace TimberWinR.Outputs
        {
            if (currentBatchCount < _maxBatchCount && currentBatchCount < queueSize && AverageQueueDepth() > currentBatchCount)
            {
-                currentBatchCount += Math.Max(_maxBatchCount/_batchCount, 1);
+                currentBatchCount += Math.Max(_maxBatchCount / _batchCount, 1);
                if (currentBatchCount >= _maxBatchCount && !_warnedReachedMax)
                {
                    LogManager.GetCurrentClassLogger().Warn("Maximum Batch Count of {0} reached.", currentBatchCount);
@@ -93,7 +93,7 @@ namespace TimberWinR.Outputs
            else // Reset to default
            {
                currentBatchCount = _batchCount;
-                _warnedReachedMax = false;                
+                _warnedReachedMax = false;
            }

            return currentBatchCount;
@@ -117,21 +117,21 @@ namespace TimberWinR.Outputs
        private readonly int _port;
        private readonly int _timeout;
        private readonly object _locker = new object();
-        private readonly List<string> _jsonQueue;       
+        private readonly List<string> _jsonQueue;
        private readonly string[] _redisHosts;
        private int _redisHostIndex;
        private TimberWinR.Manager _manager;
        private readonly int _batchCount;
        private int _currentBatchCount;
-        private readonly int _maxBatchCount;       
+        private readonly int _maxBatchCount;
        private readonly int _interval;
-        private readonly int _numThreads;      
+        private readonly int _numThreads;
        private long _sentMessages;
        private long _errorCount;
        private long _redisDepth;
-        private DateTime? _lastErrorTimeUTC;       
+        private DateTime? _lastErrorTimeUTC;
        private readonly int _maxQueueSize;
-        private readonly bool _queueOverflowDiscardOldest;       
+        private readonly bool _queueOverflowDiscardOldest;
        private BatchCounter _batchCounter;

        public bool Stop { get; set; }
@@ -150,7 +150,8 @@ namespace TimberWinR.Outputs
            {
                try
                {
-                    RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port, _timeout);
+                    RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port);
+                    client.SendTimeout = _timeout;
                    return client;
                }
                catch (Exception)
@@ -186,9 +187,9 @@ namespace TimberWinR.Outputs
                        new JProperty("threads", _numThreads),
                        new JProperty("batchcount", _batchCount),
                        new JProperty("currentBatchCount", _currentBatchCount),
-                        new JProperty("reachedMaxBatchCountTimes",  _batchCounter.ReachedMaxBatchCountTimes),
+                        new JProperty("reachedMaxBatchCountTimes", _batchCounter.ReachedMaxBatchCountTimes),
                        new JProperty("maxBatchCount", _maxBatchCount),
-                        new JProperty("averageQueueDepth", _batchCounter.AverageQueueDepth()),   
+                        new JProperty("averageQueueDepth", _batchCounter.AverageQueueDepth()),
                        new JProperty("queueSamples", new JArray(_batchCounter.Samples())),
                        new JProperty("index", _logstashIndexName),
                        new JProperty("hosts",
@@ -201,17 +202,17 @@ namespace TimberWinR.Outputs

        public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutputParameters parameters, CancellationToken cancelToken)
            : base(cancelToken, "Redis")
-        {              
+        {
            _redisDepth = 0;
            _batchCount = parameters.BatchCount;
            _maxBatchCount = parameters.MaxBatchCount;
            // Make sure maxBatchCount is larger than batchCount
-            if (_maxBatchCount < _batchCount)
-                _maxBatchCount = _batchCount*10;
-           
+            if (_maxBatchCount <= _batchCount)
+                _maxBatchCount = _batchCount * 10;
+
            _manager = manager;
            _redisHostIndex = 0;
-            _redisHosts = parameters.Host;          
+            _redisHosts = parameters.Host;
            _jsonQueue = new List<string>();
            _port = parameters.Port;
            _timeout = parameters.Timeout;
@@ -224,7 +225,7 @@ namespace TimberWinR.Outputs
            _queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
            _batchCounter = new BatchCounter(_batchCount, _maxBatchCount);
            _currentBatchCount = _batchCount;
-           
+
            for (int i = 0; i < parameters.NumThreads; i++)
            {
                var redisThread = new Task(RedisSender, cancelToken);
@@ -250,7 +251,7 @@ namespace TimberWinR.Outputs
            }

            var message = jsonMessage.ToString();
-            LogManager.GetCurrentClassLogger().Debug(message);
+            LogManager.GetCurrentClassLogger().Trace(message);

            lock (_locker)
            {
@@ -285,13 +286,13 @@ namespace TimberWinR.Outputs
            foreach (var filter in _manager.Config.Filters)
            {
                if (!filter.Apply(json))
-                {  
-                    LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
+                {
+                    LogManager.GetCurrentClassLogger().Debug("{0}: Dropping: {1}", Thread.CurrentThread.ManagedThreadId, json.ToString());
                    drop = true;
-                }              
+                }
            }
            return drop;
-        }      
+        }
        // 
        // Pull off messages from the Queue, batch them up and send them all across
        // 
@@ -313,11 +314,9 @@ namespace TimberWinR.Outputs
                                _batchCounter.SampleQueueDepth(_jsonQueue.Count);
                                // Re-compute current batch size
                                _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count, _currentBatchCount);
-                                                                              
+
                                messages = _jsonQueue.Take(_currentBatchCount).ToArray();
                                _jsonQueue.RemoveRange(0, messages.Length);
-
-                              
                            }

                            if (messages.Length > 0)
@@ -335,12 +334,12 @@ namespace TimberWinR.Outputs
                                            {
                                                client.StartPipe();
                                                LogManager.GetCurrentClassLogger()
-                                                    .Debug("Sending {0} Messages to {1}", messages.Length, client.Host);
+                                                    .Debug("{0}: Sending {1} Messages to {2}", Thread.CurrentThread.ManagedThreadId, messages.Length, client.Host);

                                                try
                                                {
                                                    _redisDepth = client.RPush(_logstashIndexName, messages);
-                                                    _sentMessages += messages.Length;                                                  
+                                                    Interlocked.Add(ref _sentMessages, messages.Length);
                                                    client.EndPipe();
                                                    sentSuccessfully = true;
                                                    if (messages.Length > 0)
@@ -357,7 +356,7 @@ namespace TimberWinR.Outputs
                                                    LogManager.GetCurrentClassLogger().Error(ex);
                                                    Interlocked.Increment(ref _errorCount);
                                                    _lastErrorTimeUTC = DateTime.UtcNow;
-                                                }                                               
+                                                }
                                                break;
                                            }
                                            else
@@ -374,20 +373,19 @@ namespace TimberWinR.Outputs
                                    {
                                        LogManager.GetCurrentClassLogger().Error(ex);
                                        Interlocked.Increment(ref _errorCount);
-                                        _lastErrorTimeUTC = DateTime.UtcNow;                                
+                                        _lastErrorTimeUTC = DateTime.UtcNow;
                                    }
                                } // No more hosts to try.

- 
-                                if (!sentSuccessfully)                                
+                                // Couldn't send, put it back into the queue.
+                                if (!sentSuccessfully)
                                {
                                    lock (_locker)
                                    {
                                        _jsonQueue.InsertRange(0, messages);
                                    }
                                }
-                            }
-                           // GC.Collect();
+                            }                          
                            if (!Stop)
                                syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                        }
@@ -395,7 +393,7 @@ namespace TimberWinR.Outputs
                        {
                            break;
                        }
-                        catch(ThreadAbortException)
+                        catch (ThreadAbortException)
                        {
                            break;
                        }
@@ -403,11 +401,11 @@ namespace TimberWinR.Outputs
                        {
                            _lastErrorTimeUTC = DateTime.UtcNow;
                            Interlocked.Increment(ref _errorCount);
-                            LogManager.GetCurrentClassLogger().Error(ex);                         
+                            LogManager.GetCurrentClassLogger().Error(ex);
                        }
                    }
                }
            }
-        }      
+        }
    }
 }
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -297,7 +297,7 @@ namespace TimberWinR.Parser
        }
    }

-    public class TailFile : IValidateSchema
+    public class TailFileArguments : IValidateSchema
    {
        [JsonProperty(PropertyName = "location")]
        public string Location { get; set; }       
@@ -312,7 +312,7 @@ namespace TimberWinR.Parser
        [JsonProperty(PropertyName = "codec")]
        public CodecArguments CodecArguments { get; set; }

-        public TailFile()
+        public TailFileArguments()
        {
            Fields = new List<Field>();
            Fields.Add(new Field("LogFilename", "string"));
@@ -605,7 +605,7 @@ namespace TimberWinR.Parser
            Index = "logstash";
            Host = new string[] { "localhost" };
            Timeout = 10000;
-            BatchCount = 10;
+            BatchCount = 50;
            MaxBatchCount = BatchCount*10;
            NumThreads = 1;
            Interval = 5000;
@@ -646,7 +646,7 @@ namespace TimberWinR.Parser
        public LogParameters[] Logs { get; set; }

        [JsonProperty("TailFiles")]
-        public TailFile[] TailFiles { get; set; }
+        public TailFileArguments[] TailFilesArguments { get; set; }

        [JsonProperty("Tcp")]
        public TcpParameters[] Tcps { get; set; }
--- a/TimberWinR/ReleaseNotes.md
+++ b/TimberWinR/ReleaseNotes.md
@@ -2,7 +2,16 @@
 ==================================
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.

-Version History
+Version / Date
+
+### 1.4.0.0 - 04/03/2015
+1. A re-factoring of Logs and TailLogs to be more efficient and detect log rolling correctly,
+   this requires http://support.microsoft.com/en-us/kb/172190 which will be detected and
+   set by TimberWinR, however, requires a reboot.
+2. Fixed issue [#38](https://github.com/Cimpress-MCP/TimberWinR/issues/38) diagnostic output not showing drop flag for Grok filter.
+3. Created TimberWinR.TestGenerator for complete testing of TimberWinR
+4. Fixed ipv4/ipv6 thread-safe issue with UdpInputListener which might lead to corrupted input data.
+
 ### 1.3.19.1 - 03/03/2015
 1. Added new Redis parameter _max\_batch\_count_ which increases the _batch\_count_ dynamically over time 
   to handle input flooding.   Default is _batch\_count_ * 10 
--- a/TimberWinR/TimberWinR.csproj
+++ b/TimberWinR/TimberWinR.csproj
@@ -12,6 +12,8 @@
    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
    <FileAlignment>512</FileAlignment>
    <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
  </PropertyGroup>
  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
    <DebugSymbols>true</DebugSymbols>
@@ -32,7 +34,7 @@
  </PropertyGroup>
  <ItemGroup>
    <Reference Include="csredis">
-      <HintPath>..\packages\csredis.1.4.7.1\lib\net40\csredis.dll</HintPath>
+      <HintPath>..\packages\csredis.3.2.1\lib\net40\csredis.dll</HintPath>
    </Reference>
    <Reference Include="Elasticsearch.Net">
      <HintPath>..\packages\Elasticsearch.Net.1.3.1\lib\Elasticsearch.Net.dll</HintPath>
@@ -43,42 +45,40 @@
      <HintPath>lib\com-logparser\Interop.MSUtil.dll</HintPath>
    </Reference>
    <Reference Include="MaxMind.Db">
-      <HintPath>..\packages\MaxMind.Db.0.2.3.0\lib\net40\MaxMind.Db.dll</HintPath>
-      <Private>True</Private>
+      <HintPath>..\packages\MaxMind.Db.1.0.0.0\lib\net40\MaxMind.Db.dll</HintPath>
    </Reference>
    <Reference Include="MaxMind.GeoIP2">
-      <HintPath>..\packages\MaxMind.GeoIP2.0.4.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
-      <Private>True</Private>
+      <HintPath>..\packages\MaxMind.GeoIP2.2.1.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
    </Reference>
    <Reference Include="Nest">
      <HintPath>..\packages\NEST.1.3.1\lib\Nest.dll</HintPath>
    </Reference>
    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net40\Newtonsoft.Json.dll</HintPath>
    </Reference>
    <Reference Include="NLog">
-      <HintPath>..\packages\NLog.3.1.0.0\lib\net40\NLog.dll</HintPath>
+      <HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
    </Reference>
    <Reference Include="RapidRegex.Core">
      <HintPath>..\packages\RapidRegex.Core.1.0.0.2\lib\net40\RapidRegex.Core.dll</HintPath>
    </Reference>
    <Reference Include="RestSharp">
-      <HintPath>..\packages\RestSharp.104.4.0\lib\net4\RestSharp.dll</HintPath>
+      <HintPath>..\packages\RestSharp.105.0.0\lib\net4\RestSharp.dll</HintPath>
    </Reference>
    <Reference Include="System" />
    <Reference Include="System.Core" />
    <Reference Include="System.Linq.Dynamic">
-      <HintPath>..\packages\System.Linq.Dynamic.1.0.3\lib\net40\System.Linq.Dynamic.dll</HintPath>
+      <HintPath>..\packages\System.Linq.Dynamic.1.0.4\lib\net40\System.Linq.Dynamic.dll</HintPath>
    </Reference>
    <Reference Include="System.Xml.Linq" />
    <Reference Include="System.Data.DataSetExtensions" />
    <Reference Include="Microsoft.CSharp" />
    <Reference Include="System.Data" />
    <Reference Include="System.Xml" />
-    <Reference Include="Topshelf, Version=3.1.122.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
+    <Reference Include="Topshelf, Version=3.1.135.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll</HintPath>
+      <HintPath>..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll</HintPath>
    </Reference>
  </ItemGroup>
  <ItemGroup>
@@ -107,6 +107,7 @@
    <Compile Include="Inputs\TcpInputListener.cs" />
    <Compile Include="Inputs\LogsListener.cs" />
    <Compile Include="Inputs\WindowsEvtInputListener.cs" />
+    <Compile Include="LogErrors.cs" />
    <Compile Include="Manager.cs" />
    <Compile Include="Outputs\Elasticsearch.cs" />
    <Compile Include="Outputs\OutputSender.cs" />
@@ -149,7 +150,9 @@
    <None Include="mdocs\Logs.md" />
    <None Include="mdocs\IISW3CInput.md" />
    <None Include="mdocs\WindowsEvents.md" />
-    <None Include="packages.config" />
+    <None Include="packages.config">
+      <SubType>Designer</SubType>
+    </None>
    <None Include="ReleaseNotes.md" />
  </ItemGroup>
  <ItemGroup>
@@ -160,6 +163,13 @@
  </ItemGroup>
  <ItemGroup />
  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
       Other similar extension points exist, see Microsoft.Common.targets.
  <Target Name="BeforeBuild">
--- a/TimberWinR/packages.config
+++ b/TimberWinR/packages.config
@@ -1,13 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="csredis" version="1.4.7.1" targetFramework="net40" />
+  <package id="csredis" version="3.2.1" targetFramework="net40" />
  <package id="Elasticsearch.Net" version="1.3.1" targetFramework="net40" />
-  <package id="MaxMind.Db" version="0.2.3.0" targetFramework="net40" />
-  <package id="MaxMind.GeoIP2" version="0.4.0.0" targetFramework="net40" />
+  <package id="MaxMind.Db" version="1.0.0.0" targetFramework="net40" />
+  <package id="MaxMind.GeoIP2" version="2.1.0.0" targetFramework="net40" />
  <package id="NEST" version="1.3.1" targetFramework="net40" />
-  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
-  <package id="NLog" version="3.1.0.0" targetFramework="net40" />
+  <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net40" />
+  <package id="NLog" version="3.2.0.0" targetFramework="net40" />
  <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
-  <package id="RestSharp" version="104.4.0" targetFramework="net40" />
-  <package id="System.Linq.Dynamic" version="1.0.3" targetFramework="net40" />
+  <package id="RestSharp" version="105.0.0" targetFramework="net40" />
+  <package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
+  <package id="Topshelf" version="3.1.4" targetFramework="net40" />
 </packages>
--- a/TimberWix/TimberWinR.Wix.wixproj
+++ b/TimberWix/TimberWinR.Wix.wixproj
@@ -63,9 +63,6 @@
    </WixExtension>
  </ItemGroup>
  <Import Project="$(WixTargetsPath)" />
-  <PropertyGroup>
-    <PreBuildEvent />
-  </PropertyGroup>
  <Target Name="BeforeBuild">
    <!-- Read the version from the to-be-installed .exe -->
    <GetAssemblyIdentity AssemblyFiles="..\TimberWinR.ServiceHost\bin\$(Configuration)\TimberWinR.ServiceHost.exe">
@@ -103,6 +100,9 @@
  <PropertyGroup>
    <PostBuildEvent>$(SolutionDir)\TimberWinR.ExtractID\$(OutDir)\TimberWinR.ExtractID.exe $(TargetDir) $(SolutionDir)chocolateyUninstall.ps1.guid  $(SolutionDir)chocolateyUninstall.ps1.template</PostBuildEvent>
  </PropertyGroup>
+  <PropertyGroup>
+    <PreBuildEvent>cmd.exe /c copy $(SolutionDir)chocolateyUninstall.ps1.template.orig  $(SolutionDir)chocolateyUninstall.ps1.template</PreBuildEvent>
+  </PropertyGroup>
  <!--
 	To modify your build process, add your task inside one of the targets below and uncomment it.
 	Other similar extension points exist, see Wix.targets.
--- a/chocolateyUninstall.ps1.template
+++ b/chocolateyUninstall.ps1.template
@@ -1,7 +1,7 @@
 $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
 $installerType = 'msi' #only one of these: exe, msi, msu
 $url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
-$silentArgs = '${PROJECTGUID} /quiet' 
+$silentArgs = '{D066A694-B9F7-4B35-BB7D-D34CB88CA89F} /quiet' 
 $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
 UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes

--- a/packages/repositories.config
+++ b/packages/repositories.config
@@ -1,6 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <repositories>
-  <repository path="..\TimberWinR.ServiceHost\packages.config" />
+  <repository path="..\TimberWinR.ServiceHost\packages.config" />  
+  <repository path="..\TimberWinR.TestGenerator\packages.config" />
  <repository path="..\TimberWinR.UnitTests\packages.config" />
  <repository path="..\TimberWinR\packages.config" />
 </repositories>