Updated to new version

Bug fix for UdpInputListener, re-factored TailFile and LogsListener to u...

.nuget/NuGet.Config

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <solution>
+    <add key="disableSourceControlIntegration" value="true" />
+  </solution>
+</configuration>

.nuget/NuGet.targets

@@ -0,0 +1,144 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+    <PropertyGroup>
+        <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">$(MSBuildProjectDirectory)\..\</SolutionDir>
+
+        <!-- Enable the restore command to run before builds -->
+        <RestorePackages Condition="  '$(RestorePackages)' == '' ">false</RestorePackages>
+
+        <!-- Property that enables building a package from a project -->
+        <BuildPackage Condition=" '$(BuildPackage)' == '' ">false</BuildPackage>
+
+        <!-- Determines if package restore consent is required to restore packages -->
+        <RequireRestoreConsent Condition=" '$(RequireRestoreConsent)' != 'false' ">true</RequireRestoreConsent>
+
+        <!-- Download NuGet.exe if it does not already exist -->
+        <DownloadNuGetExe Condition=" '$(DownloadNuGetExe)' == '' ">false</DownloadNuGetExe>
+    </PropertyGroup>
+
+    <ItemGroup Condition=" '$(PackageSources)' == '' ">
+        <!-- Package sources used to restore packages. By default, registered sources under %APPDATA%\NuGet\NuGet.Config will be used -->
+        <!-- The official NuGet package source (https://www.nuget.org/api/v2/) will be excluded if package sources are specified and it does not appear in the list -->
+        <!--
+            <PackageSource Include="https://www.nuget.org/api/v2/" />
+            <PackageSource Include="https://my-nuget-source/nuget/" />
+        -->
+    </ItemGroup>
+
+    <PropertyGroup Condition=" '$(OS)' == 'Windows_NT'">
+        <!-- Windows specific commands -->
+        <NuGetToolsPath>$([System.IO.Path]::Combine($(SolutionDir), ".nuget"))</NuGetToolsPath>
+    </PropertyGroup>
+
+    <PropertyGroup Condition=" '$(OS)' != 'Windows_NT'">
+        <!-- We need to launch nuget.exe with the mono command if we're not on windows -->
+        <NuGetToolsPath>$(SolutionDir).nuget</NuGetToolsPath>
+    </PropertyGroup>
+
+    <PropertyGroup>
+        <PackagesProjectConfig Condition=" '$(OS)' == 'Windows_NT'">$(MSBuildProjectDirectory)\packages.$(MSBuildProjectName.Replace(' ', '_')).config</PackagesProjectConfig>
+        <PackagesProjectConfig Condition=" '$(OS)' != 'Windows_NT'">$(MSBuildProjectDirectory)\packages.$(MSBuildProjectName).config</PackagesProjectConfig>
+    </PropertyGroup>
+
+    <PropertyGroup>
+      <PackagesConfig Condition="Exists('$(MSBuildProjectDirectory)\packages.config')">$(MSBuildProjectDirectory)\packages.config</PackagesConfig>
+      <PackagesConfig Condition="Exists('$(PackagesProjectConfig)')">$(PackagesProjectConfig)</PackagesConfig>
+    </PropertyGroup>
+    
+    <PropertyGroup>
+        <!-- NuGet command -->
+        <NuGetExePath Condition=" '$(NuGetExePath)' == '' ">$(NuGetToolsPath)\NuGet.exe</NuGetExePath>
+        <PackageSources Condition=" $(PackageSources) == '' ">@(PackageSource)</PackageSources>
+
+        <NuGetCommand Condition=" '$(OS)' == 'Windows_NT'">"$(NuGetExePath)"</NuGetCommand>
+        <NuGetCommand Condition=" '$(OS)' != 'Windows_NT' ">mono --runtime=v4.0.30319 "$(NuGetExePath)"</NuGetCommand>
+
+        <PackageOutputDir Condition="$(PackageOutputDir) == ''">$(TargetDir.Trim('\\'))</PackageOutputDir>
+
+        <RequireConsentSwitch Condition=" $(RequireRestoreConsent) == 'true' ">-RequireConsent</RequireConsentSwitch>
+        <NonInteractiveSwitch Condition=" '$(VisualStudioVersion)' != '' AND '$(OS)' == 'Windows_NT' ">-NonInteractive</NonInteractiveSwitch>
+
+        <PaddedSolutionDir Condition=" '$(OS)' == 'Windows_NT'">"$(SolutionDir) "</PaddedSolutionDir>
+        <PaddedSolutionDir Condition=" '$(OS)' != 'Windows_NT' ">"$(SolutionDir)"</PaddedSolutionDir>
+
+        <!-- Commands -->
+        <RestoreCommand>$(NuGetCommand) install "$(PackagesConfig)" -source "$(PackageSources)"  $(NonInteractiveSwitch) $(RequireConsentSwitch) -solutionDir $(PaddedSolutionDir)</RestoreCommand>
+        <BuildCommand>$(NuGetCommand) pack "$(ProjectPath)" -Properties "Configuration=$(Configuration);Platform=$(Platform)" $(NonInteractiveSwitch) -OutputDirectory "$(PackageOutputDir)" -symbols</BuildCommand>
+
+        <!-- We need to ensure packages are restored prior to assembly resolve -->
+        <BuildDependsOn Condition="$(RestorePackages) == 'true'">
+            RestorePackages;
+            $(BuildDependsOn);
+        </BuildDependsOn>
+
+        <!-- Make the build depend on restore packages -->
+        <BuildDependsOn Condition="$(BuildPackage) == 'true'">
+            $(BuildDependsOn);
+            BuildPackage;
+        </BuildDependsOn>
+    </PropertyGroup>
+
+    <Target Name="CheckPrerequisites">
+        <!-- Raise an error if we're unable to locate nuget.exe  -->
+        <Error Condition="'$(DownloadNuGetExe)' != 'true' AND !Exists('$(NuGetExePath)')" Text="Unable to locate '$(NuGetExePath)'" />
+        <!--
+        Take advantage of MsBuild's build dependency tracking to make sure that we only ever download nuget.exe once.
+        This effectively acts as a lock that makes sure that the download operation will only happen once and all
+        parallel builds will have to wait for it to complete.
+        -->
+        <MsBuild Targets="_DownloadNuGet" Projects="$(MSBuildThisFileFullPath)" Properties="Configuration=NOT_IMPORTANT;DownloadNuGetExe=$(DownloadNuGetExe)" />
+    </Target>
+
+    <Target Name="_DownloadNuGet">
+        <DownloadNuGet OutputFilename="$(NuGetExePath)" Condition=" '$(DownloadNuGetExe)' == 'true' AND !Exists('$(NuGetExePath)')" />
+    </Target>
+
+    <Target Name="RestorePackages" DependsOnTargets="CheckPrerequisites">        
+        <Exec Command="$(RestoreCommand)"
+              Condition="'$(OS)' != 'Windows_NT' And Exists('$(PackagesConfig)')" />
+
+        <Exec Command="$(RestoreCommand)"
+              LogStandardErrorAsError="true"
+              Condition="'$(OS)' == 'Windows_NT' And Exists('$(PackagesConfig)')" />
+    </Target>
+
+    <Target Name="BuildPackage" DependsOnTargets="CheckPrerequisites">
+        <Exec Command="$(BuildCommand)"
+              Condition=" '$(OS)' != 'Windows_NT' " />
+
+        <Exec Command="$(BuildCommand)"
+              LogStandardErrorAsError="true"
+              Condition=" '$(OS)' == 'Windows_NT' " />
+    </Target>
+
+    <UsingTask TaskName="DownloadNuGet" TaskFactory="CodeTaskFactory" AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.v4.0.dll">
+        <ParameterGroup>
+            <OutputFilename ParameterType="System.String" Required="true" />
+        </ParameterGroup>
+        <Task>
+            <Reference Include="System.Core" />
+            <Using Namespace="System" />
+            <Using Namespace="System.IO" />
+            <Using Namespace="System.Net" />
+            <Using Namespace="Microsoft.Build.Framework" />
+            <Using Namespace="Microsoft.Build.Utilities" />
+            <Code Type="Fragment" Language="cs">
+                <![CDATA[
+                try {
+                    OutputFilename = Path.GetFullPath(OutputFilename);
+
+                    Log.LogMessage("Downloading latest version of NuGet.exe...");
+                    WebClient webClient = new WebClient();
+                    webClient.DownloadFile("https://www.nuget.org/nuget.exe", OutputFilename);
+
+                    return true;
+                }
+                catch (Exception ex) {
+                    Log.LogErrorFromException(ex);
+                    return false;
+                }
+            ]]>
+            </Code>
+        </Task>
+    </UsingTask>
+</Project>

.nuget/packages.config

@@ -1,4 +1,7 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="NUnit.Runners" version="2.6.3" />
+  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
+  <package id="NUnit.Runners" version="2.6.4" />
+  <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
+  <package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
 </packages>

README.md

@@ -69,7 +69,6 @@ A single Json filter using the single tag (this is only provided as a convienien
     }
 ]
 ```
-
 Multiple Json filters must use the jsonFilters and array syntax, also mutateFilters, grokFilters, dateFilters, geoipFilters.
 ```json
 "Filters": [
@@ -220,7 +219,7 @@ order on disk.
 If you really just want to try it out, grab the binary distribution, extract the .zip file
 into a directory, e.g.  C:\TimberWinR
 
-Grab the [JSON example file](https://raw.githubusercontent.com/efontana/TimberWinR/master/TimberWinR.ServiceHost/default.json) and place it into C:\TimberWinR\default.json. 
+Grab the [JSON example file](https://raw.githubusercontent.com/Cimpress-MCP/TimberWinR/master/TimberWinR.ServiceHost/default.json) and place it into C:\TimberWinR\default.json. 
 Edit the default.json file and change the Redis instance to match yours, replace 'tstlexiceapp006.mycompany.svc' with the IP or DNS name
 of the machine running redis.  Fire up the collector, enable the verbose debugging to see some Windows Events.
 

TimberWinR.ServiceHost/Program.cs

@@ -7,7 +7,9 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
+using System.Xml;
 using Microsoft.Win32;
+
 using TimberWinR.Outputs;
 using TimberWinR.ServiceHost;
 using TimberWinR.Inputs;
@@ -26,6 +28,9 @@ namespace TimberWinR.ServiceHost
         
         private static void Main(string[] args)
         {
+
+           
+
             Arguments arguments = new Arguments();
            
             HostFactory.Run(hostConfigurator =>

TimberWinR.ServiceHost/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.20.0")]
-[assembly: AssemblyFileVersion("1.3.20.0")]
+[assembly: AssemblyVersion("1.3.21.0")]
+[assembly: AssemblyFileVersion("1.3.21.0")]

TimberWinR.ServiceHost/TimberWinR.ServiceHost.csproj

@@ -12,6 +12,8 @@
     <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <PlatformTarget>AnyCPU</PlatformTarget>
@@ -49,9 +51,8 @@
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
-    <Reference Include="Topshelf, Version=3.1.122.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll</HintPath>
+    <Reference Include="Topshelf">
+      <HintPath>..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>
@@ -87,6 +88,13 @@
     </ProjectReference>
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">

TimberWinR.ServiceHost/default.json

@@ -18,7 +18,7 @@
         "Filters": [
             {
                 "grok": {
-                    "condition": "[EventTypeName] == \"Information Event\"",
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
                     "match": [
                         "Text",
                         ""

TimberWinR.ServiceHost/packages.config

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
   <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
-  <package id="Topshelf" version="3.1.3" targetFramework="net40" />
+  <package id="Topshelf" version="3.1.4" targetFramework="net40" />
 </packages>

TimberWinR.TestGenerator/CommandLineOptions.cs

@@ -0,0 +1,82 @@
+using System.ComponentModel;
+using CommandLine;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using CommandLine.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class CommandLineOptions
+    {
+       // [Option('r', "read", Required = true, HelpText = "Input file to be processed.")]
+      //  public string InputFile { get; set; }
+        [Option("timberWinRConfig", DefaultValue = "default.json", HelpText = "Config file/directory to use")]
+        public string TimberWinRConfigFile { get; set; }
+
+        [Option("testDir", DefaultValue = ".", HelpText = "Test directory to use (created if necessary)")]
+        public string TestDir { get; set; }
+
+        [Option("testFile", DefaultValue = "", HelpText = "Config file/directory to use")]
+        public string TestFile { get; set; }
+
+        [Option("resultsFile", HelpText = "Expected results Results json file")]
+        public string ExpectedResultsFile { get; set; }
+
+        [Option('n', "numMessages", DefaultValue = 1000, HelpText = "The number of messages to send to the output(s)")]
+        public int NumMessages { get; set; }
+
+        [Option('l', "logLevel", DefaultValue = "debug", HelpText = "Logging Level Debug|Error|Fatal|Info|Off|Trace|Warn")]
+        public string LogLevel { get; set; }
+
+        [Option('v', "verbose", DefaultValue = true, HelpText = "Prints all messages to standard output.")]
+        public bool Verbose { get; set; }
+
+        [Option("jsonLogDir", DefaultValue = ".", HelpText = "Json LogGenerator Log directory")]
+        public string JsonLogDir { get; set; }
+
+        [OptionArray('j', "json", DefaultValue = new string[] {})]
+        public string[] JsonLogFiles { get; set; }
+
+        [OptionArray("jroll", DefaultValue = new string[] { })]
+        public string[] JsonRollingLogFiles { get; set; }
+        
+        [Option("jsonRate", DefaultValue = 30, HelpText = "Json Rate in Milliseconds between generation of log lines")]
+        public int JsonRate { get; set; }
+
+        [Option('u', "udp", DefaultValue = 0, HelpText = "Enable UDP generator on this Port")]
+        public int Udp { get; set; }
+
+        [Option("udp-host", DefaultValue = "localhost", HelpText = "Host to send Udp data to")]
+        public string UdpHost { get; set; }
+
+        [Option("udp-rate", DefaultValue = 10, HelpText = "Udp Rate in Milliseconds between generation of log lines")]
+        public int UdpRate { get; set; }
+
+        [Option('t', "tcp", DefaultValue = 0, HelpText = "Enable Tcp generator on this Port")]
+        public int Tcp { get; set; }
+
+        [Option("tcp-host", DefaultValue = "localhost", HelpText = "Host to send Tcp data to")]
+        public string TcpHost { get; set; }
+
+        [Option("tcp-rate", DefaultValue = 10, HelpText = "Tcp Rate in Milliseconds between generation of log lines")]
+        public int TcpRate { get; set; }
+
+        [Option('r', "redis", DefaultValue = 0, HelpText = "Enable Redis generator on this Port")]
+        public int Redis { get; set; }
+
+        [Option("redis-host", DefaultValue = "", HelpText = "Host to send Redis data to")]
+        public string RedisHost { get; set; }
+
+        [ParserState]
+        public IParserState LastParserState { get; set; }
+
+        [HelpOption]
+        public string GetUsage()
+        {
+            return HelpText.AutoBuild(this,
+              (HelpText current) => HelpText.DefaultParsingErrorsHandler(this, current));
+        }
+    }
+}

TimberWinR.TestGenerator/JsonLogFileGenerator.cs

@@ -0,0 +1,252 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.IO;
+
+namespace TimberWinR.TestGenerator
+{
+    class JsonLogFileTestParameters
+    {
+        public int NumMessages { get; set; }
+        public string LogFileDir { get; set; }
+        public string LogFileName { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public JsonLogFileTestParameters()
+        {
+            SleepTimeMilliseconds = 30;
+            LogFileDir = ".";
+            NumMessages = 10;
+        }
+    }
+
+    class JsonLogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start JSON LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                {
+                    LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath);
+                    File.Delete(logFilePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var watch = Stopwatch.StartNew();
+
+            // This text is always added, making the file longer over time 
+            // if it is not deleted. 
+            using (StreamWriter sw = File.AppendText(logFilePath))
+            {
+                sw.AutoFlush = true;
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"LineNumber", i+1},
+                        {"Application", "jsonlogfile-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "jsonlog"},
+                        {"Message", string.Format("{0}: Testgenerator jsonlogfile message {1}", i+1, DateTime.UtcNow.ToString("o"))},
+                        {"Index", "logstash"}
+                    };
+                    sw.WriteLine(o.ToString(Formatting.None));
+
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+                LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+                watch.Reset();
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished JSON Log File Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+    }
+
+    class JsonRollingLogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start JSON RollingLogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                    File.Delete(logFilePath);
+
+                if (File.Exists(logFilePath + ".rolled"))
+                    File.Delete(logFilePath + ".rolled");
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+         
+            int quarters = parms.NumMessages/4;
+
+            int[] segments = new int[] {quarters, quarters, quarters, quarters + parms.NumMessages%4};
+            var watch = Stopwatch.StartNew();
+
+
+            int recordNumber = 0;
+            int currentTotal = 0;
+            for (int segment = 0; segment < 4; segment++)
+            {
+                currentTotal += segments[segment];
+             
+                // This text is always added, making the file longer over time 
+                // if it is not deleted. 
+                using (StreamWriter sw = File.AppendText(logFilePath))
+                {
+                    sw.AutoFlush = true;
+   
+                    var lwatch = Stopwatch.StartNew();
+
+                    // The Rolling Generator will roll 1/2 way through the log             
+                    for (int i = 0; i < segments[segment]; i++)
+                    {
+                        JObject o = new JObject
+                        {
+                            {"LineNumber", recordNumber + 1},
+                            {"Application", "jsonrollinglogfile-generator"},
+                            {"Host", hostName},
+                            {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                            {"Type", "jsonrollinglog"},
+                            {
+                                "Message",
+                                string.Format("{0}: Testgenerator jsonrollinglogfile message {1}", recordNumber + 1,
+                                    DateTime.UtcNow.ToString("o"))
+                            },
+                            {"Index", "logstash"}
+                        };
+                        sw.WriteLine(o.ToString(Formatting.None));
+                        recordNumber++;
+                        Thread.Sleep(parms.SleepTimeMilliseconds);
+                    }
+                    LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds for {2} logs", Path.GetFullPath(parms.LogFileName), lwatch.Elapsed, segments[segment]);
+                   
+                }
+               
+                // 
+                // We might not have yet processed all the lines from the first file, so wait till
+                // we catch up before rolling the log file.
+                //
+                LogManager.GetCurrentClassLogger().Info("{0}: Waiting for output to catch up: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, currentTotal);
+                WaitOutputToCatchUp(logFilePath, currentTotal);
+
+                //
+                // Roll the log + wait for the reader to catch up.
+                //  
+                
+                LogManager.GetCurrentClassLogger().Info("{0}: Rolling Log File: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, File.GetCreationTimeUtc(logFilePath));
+
+                RollLogFile(logFilePath); 
+                
+                LogManager.GetCurrentClassLogger().Info("{0}: Finished Rolling Log File: {1}", Thread.CurrentThread.ManagedThreadId, logFilePath);           
+            }
+            
+            watch.Stop();
+           
+            LogManager.GetCurrentClassLogger().Info("Finished JSON RollingLogFile File Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+
+        private static void WaitOutputToCatchUp(string logFilePath, int firstPart)
+        {
+            bool caughtUp = false;
+            do
+            {
+                var json = Program.Diagnostics.DiagnosticsOutput();
+
+                IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+                foreach (JToken t in inputs)
+                {
+                    JProperty inputProp = t.First as JProperty;
+                    if (inputProp.Name == "taillog" || inputProp.Name == "log")
+                    {
+                        var files = inputProp.Value["filedb"].Children().ToList();
+                        foreach (var file in files)
+                        {
+                            var fileName = file["FileName"].ToString();
+                            FileInfo fi1 = new FileInfo(fileName);
+                            FileInfo fi2 = new FileInfo(logFilePath);
+                            if (fi1.FullName == fi2.FullName)
+                            {
+                                var linesProcessed = file["LinesProcessed"].Value<int>();
+                                if (linesProcessed >= firstPart)
+                                {
+                                    caughtUp = true;
+                                    break;
+                                }
+                            }
+                        }
+                    }
+                }
+
+                Thread.Sleep(300);
+            } while (!caughtUp);
+
+            LogManager.GetCurrentClassLogger().Info("{0}:  Finished Waiting for output to catch up: {1} {2}", Thread.CurrentThread.ManagedThreadId, logFilePath, firstPart);            
+
+        }
+
+        private static void RollLogFile(string logFilePath)
+        {
+            bool moved = false;           
+            do
+            {
+                try
+                {
+                    if (File.Exists(logFilePath + ".rolled"))
+                        File.Delete(logFilePath + ".rolled");
+
+                    File.Move(logFilePath, logFilePath + ".rolled");
+                    moved = true;                   
+                }
+                catch (Exception)
+                {
+                    Thread.Sleep(100);
+                }
+            } while (!moved);
+            Thread.Sleep(1000);
+        }
+    }
+
+}

TimberWinR.TestGenerator/LogFileGenerator.cs

@@ -0,0 +1,94 @@
+using System.Diagnostics;
+using System.Runtime.InteropServices;
+using System.Runtime.Remoting.Messaging;
+using System.Threading;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using System.IO;
+
+
+namespace TimberWinR.TestGenerator
+{
+    class LogFileTestParameters
+    {
+        public int NumMessages { get; set; }
+        public string LogFileDir { get; set; }
+        public string LogFileName { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public LogFileTestParameters()
+        {
+            SleepTimeMilliseconds = 30;
+            LogFileDir = ".";
+            NumMessages = 10;
+        }
+    }
+
+    class LogFileGenerator
+    {
+        public static int Generate(JsonLogFileTestParameters parms)
+        {
+            LogManager.GetCurrentClassLogger().Info("Start LogFile Generation for: {0} on Thread: {1}", Path.GetFullPath(parms.LogFileName), Thread.CurrentThread.ManagedThreadId);
+
+            var logFilePath = Path.Combine(parms.LogFileDir, parms.LogFileName);
+
+            try
+            {
+                if (File.Exists(logFilePath))
+                {
+                    LogManager.GetCurrentClassLogger().Info("Deleting file: {0}", logFilePath);
+                    File.Delete(logFilePath);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }
+
+
+            var hostName = System.Environment.MachineName + "." +
+                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                       "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var watch = Stopwatch.StartNew();
+
+            // This text is always added, making the file longer over time 
+            // if it is not deleted. 
+            using (StreamWriter sw = File.AppendText(logFilePath))
+            {
+                sw.AutoFlush = true;
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"LineNumber", i+1},
+                        {"Application", "logfile-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "log"},
+                        {"Message", string.Format("{0}: Testgenerator logfile message {1}", i+1, DateTime.UtcNow.ToString("o"))},
+                        {"Index", "logstash"}
+                    };
+                    sw.WriteLine(o.ToString(Formatting.None));
+
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+                LogManager.GetCurrentClassLogger().Info("Elapsed Time for {0} was {1} seconds", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+                watch.Reset();
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished LogFile Generation for: {0} elapsed: {1}", Path.GetFullPath(parms.LogFileName), watch.Elapsed);
+
+            return parms.NumMessages;
+        }
+    }
+}

TimberWinR.TestGenerator/Program.cs

@@ -0,0 +1,584 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Linq.Expressions;
+using System.Text;
+using System.Threading;
+using System.Threading.Tasks;
+using System.CodeDom.Compiler;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+using ServiceStack.Text.Jsv;
+using TimberWinR.Parser;
+
+
+namespace TimberWinR.TestGenerator
+{
+    public class Program
+    {
+        private static List<Task> _tasks = new List<Task>();
+        private static CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource();
+        private static Manager _timberWinR;
+
+        public static Diagnostics.Diagnostics Diagnostics { get; set; }
+
+        private static PerformanceCounter cpuCounter = new PerformanceCounter();
+        private static PerformanceCounter ramCounter = new PerformanceCounter();
+        private static Task _monitorTask;
+
+        private static int _totalMessagesToSend;
+        private static int _cpuSampleCount;
+        private static double _avgCpuUsage;
+        private static double _totalCpuUsage;
+        private static double _maxCpuUsage;
+
+        private static int _memSampleCount;
+        private static double _avgMemUsage;
+        private static double _totalMemUsage;
+        private static double _maxMemUsage;
+
+        private static CommandLineOptions Options;
+
+        static int Main(string[] args)
+        {
+            _totalMessagesToSend = 0;
+
+            cpuCounter.CategoryName = "Processor";
+            cpuCounter.CounterName = "% Processor Time";
+            cpuCounter.InstanceName = "_Total";
+
+            ramCounter.CategoryName = "Memory";
+            ramCounter.CounterName = "% Committed Bytes In Use";
+           
+            Options = new CommandLineOptions();
+
+            if (CommandLine.Parser.Default.ParseArguments(args, Options))
+            {
+                var testFile = Options.TestFile;
+                if (!string.IsNullOrEmpty(testFile))
+                {
+                    if (!File.Exists(Options.TestFile))
+                        throw new Exception(string.Format("No such test file: {0} found", Options.TestFile));
+
+                    var fargs = ParseTestArguments(testFile, ref Options);
+                    if (!CommandLine.Parser.Default.ParseArguments(fargs, Options))
+                        return 2;
+                }
+
+                SetupTestDirectory(Options);
+
+                var swOverall = Stopwatch.StartNew();
+                swOverall.Start();
+
+                InitializeLogging(Options.LogLevel);
+
+                LogManager.GetCurrentClassLogger().Info("Starting CPU Usage: {0}, RAM Usage: {1}", getCurrentCpuUsage(), getAvailableRAM());
+
+                // Reset the tests.
+                ResetTests(Options);
+
+                var sw = Stopwatch.StartNew();
+
+                // Startup TimberWinR
+                StartTimberWinR(Options.TimberWinRConfigFile, Options.LogLevel, ".", false);
+
+                // Run the Generators
+                var arrayOfTasks = RunGenerators(Options);
+
+                // Wait for all Generators to finish
+                try
+                {
+                    Task.WaitAll(arrayOfTasks);
+                }
+                catch (AggregateException aex)
+                {
+                    LogManager.GetCurrentClassLogger().Error(aex);
+                }
+
+
+                LogManager.GetCurrentClassLogger().Info("Generation Finished: " + sw.Elapsed);
+                sw.Reset();
+                sw.Start();
+
+                // All generators are finished, wait till senders are done.
+                WaitForOutputTransmission();
+
+                LogManager.GetCurrentClassLogger().Info("Finished Transmission: " + sw.Elapsed);
+                sw.Reset();
+                sw.Start();
+
+                // Get all the stats
+                var jsonTimberWinr = ShutdownTimberWinR();
+
+                LogManager.GetCurrentClassLogger().Info("Finished Shutdown: " + sw.Elapsed);
+                sw.Stop();
+
+                swOverall.Stop();
+                LogManager.GetCurrentClassLogger().Info("Total Elapsed Time: {0}", swOverall.Elapsed);
+
+                int results = VerifyResults(Options, jsonTimberWinr);
+
+                Console.ReadKey();
+                return results;
+            }
+
+            return 1;
+        }
+
+        private static void CopySourceFile(string fileName, string outputDir)
+        {
+            FileInfo fi = new FileInfo(fileName);
+            if (fi.Exists)
+                File.Copy(fileName, Path.Combine(outputDir, fi.Name));
+        }
+
+        private static void SetupTestDirectory(CommandLineOptions options)
+        {
+            if (options.TestDir != "." && Directory.Exists(options.TestDir))
+                Directory.Delete(options.TestDir, true);
+
+            if (!Directory.Exists(options.TestDir))
+                Directory.CreateDirectory(options.TestDir);
+
+            CopySourceFile(options.TestFile, options.TestDir);
+            CopySourceFile(options.TimberWinRConfigFile, options.TestDir);
+            CopySourceFile(options.ExpectedResultsFile, options.TestDir);
+
+            Directory.SetCurrentDirectory(options.TestDir);
+        }
+
+        private static string[] ParseTestArguments(string testFile, ref CommandLineOptions options)
+        {
+            options = new CommandLineOptions();
+            JObject jtest = JObject.Parse(File.ReadAllText(testFile));
+            IList<JToken> inputs = jtest["arguments"].Children().ToList();
+            List<string> testargs = new List<string>();
+            foreach (JProperty it in inputs)
+            {
+                testargs.Add(it.Name);
+
+                var cc = it.Value.Children().Count();
+                if (cc > 0)
+                {
+                    for (int i = 0; i < cc; i++)
+                    {
+                        testargs.Add(it.Value[i].ToString());
+                    }
+                }
+                else
+                {
+                    testargs.Add(it.Value.ToString());
+                }
+            }
+            var fargs = testargs.ToArray();
+            return fargs;
+        }
+
+        private static int VerifyResults(CommandLineOptions options, JObject json)
+        {
+            var jresult = JObject.Parse(File.ReadAllText(options.ExpectedResultsFile));
+
+            json["maxCpuUsage"] = _maxCpuUsage;
+            json["avgCpuUsage"] = _avgCpuUsage;
+
+            json["maxMemUsage"] = _maxMemUsage;
+            json["avgMemUsage"] = _avgMemUsage;
+
+            // TailLogs
+
+            IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+            foreach (JToken t in inputs)
+            {
+                JProperty inputProp = t.First as JProperty;
+                switch (inputProp.Name)
+                {
+                    case "udp":
+                        if (VerifyConditions(json, new string[] {"udp"}, inputProp, jresult) != 0)
+                            return 1;
+                        break;
+                    case "tcp":
+                        if (VerifyConditions(json, new string[] {"tcp"}, inputProp, jresult) != 0)
+                            return 1;
+                        break;
+                    case "log":
+                    case "taillog":
+                        if (VerifyConditions(json, new string[] {"log", "taillog"}, inputProp, jresult) != 0)
+                            return 1;
+                        break;
+                }
+            }
+
+            return 0;
+        }
+
+        private static int VerifyConditions(JObject json, string[] logTypes, JProperty inputProp, JObject jresult)
+        {
+            var ttail = inputProp.Value as JObject;
+            foreach (var resultInput in jresult["Results"]["Inputs"].Children().ToList())
+            {
+                JProperty rinputProp = resultInput.First as JProperty;
+                if (logTypes.Contains(rinputProp.Name))
+                {
+                    foreach (JProperty testProp in rinputProp.Value)
+                    {
+                        try
+                        {
+                            var cond1 = testProp.Value.ToString();
+                            IList<string> tkeys = ttail.Properties().Select(pn => pn.Name).ToList();
+                            foreach (string tkey in tkeys)
+                                cond1 = cond1.Replace(string.Format("[{0}]", tkey), string.Format("{0}", ttail[tkey].ToString()));
+
+                            // Add builtins
+                            cond1 = cond1.Replace("[avgCpuUsage]", json["avgCpuUsage"].ToString());
+                            cond1 = cond1.Replace("[maxCpuUsage]", json["maxCpuUsage"].ToString());
+                            cond1 = cond1.Replace("[avgMemUsage]", json["avgMemUsage"].ToString());
+                            cond1 = cond1.Replace("[maxMemUsage]", json["maxMemUsage"].ToString());
+
+                            var p1 = Expression.Parameter(typeof(JObject), "json");
+                            var e1 = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] { p1 },
+                                typeof(bool), cond1);
+                            bool r1 = (bool)e1.Compile().DynamicInvoke(ttail);
+                            if (!r1)
+                            {
+                                LogManager.GetCurrentClassLogger().Error("Test Failed: '{0}: ({1})'", testProp.Name, cond1);
+                                return 1;
+
+                            }
+                            else
+                            {
+                                LogManager.GetCurrentClassLogger()
+                                    .Info("PASSED({0}): '{1}: ({2})'", inputProp.Name, testProp.Name, cond1);
+                            }
+                        }
+                        catch (Exception ex)
+                        {
+                            LogManager.GetCurrentClassLogger()
+                                .Error("Error parsing expression '{0}': {1}", testProp.Value.ToString(),
+                                    ex.Message);
+                            return 2;
+                        }
+                    }
+                }
+            }
+            return 0;
+        }
+
+        // Wait till all output has been transmitted.
+        private static void WaitForOutputTransmission()
+        {
+            bool completed = false;
+            do
+            {
+                var json = Diagnostics.DiagnosticsOutput();
+
+                //Console.WriteLine(json.ToString(Formatting.Indented));
+
+                IList<JToken> inputs = json["timberwinr"]["inputs"].Children().ToList();
+                foreach (var so in inputs.Children())
+                {
+                    var token = so.First;
+                    var messages = token["messages"].Value<int>();
+                    //  Console.WriteLine("{0} messages", messages);
+                }
+
+
+                IList<JToken> outputs = json["timberwinr"]["outputs"].Children().ToList();
+                foreach (var so in outputs.Children())
+                {
+                    var outputToken = so.First;
+
+                    var mbc = outputToken["queuedMessageCount"].Value<int>();
+                    var smc = outputToken["sentMessageCount"].Value<int>();
+
+                    // LogManager.GetCurrentClassLogger().Info("Queued: {0}, Sent: {1}", mbc, smc);
+
+                    completed = mbc == 0 && smc >= _totalMessagesToSend;
+                }
+                Thread.Sleep(250);
+            } while (!completed);
+        }
+
+        private static void sampleUsages()
+        {
+            getCurrentCpuUsage();
+            getAvailableRAM();
+        }
+
+        private static string getCurrentCpuUsage()
+        {
+            _cpuSampleCount++;
+            var v = cpuCounter.NextValue();
+            if (v > _maxCpuUsage)
+                _maxCpuUsage = v;
+
+            _totalCpuUsage += v;
+            _avgCpuUsage = _totalCpuUsage / _cpuSampleCount;
+
+            return v + "%";
+        }
+
+        private static string getAvailableRAM()
+        {
+            _memSampleCount++;
+            var v = ramCounter.NextValue();
+            if (v > _maxMemUsage)
+                _maxMemUsage = v;
+
+            _totalMemUsage += v;
+            _avgMemUsage = _totalMemUsage / _memSampleCount;
+            return v + "MB";
+        }
+
+        private static JObject ShutdownTimberWinR()
+        {
+            _timberWinR.Shutdown();
+
+            // Cancel any/all other threads
+            _cancellationTokenSource.Cancel();
+
+            var json = Diagnostics.DiagnosticsOutput();
+
+            LogManager.GetCurrentClassLogger()
+                .Info("Average CPU Usage: {0}%, Average RAM Usage: {1}MB, Max CPU: {2}%, Max Mem: {3}MB", _avgCpuUsage, _avgMemUsage, _maxCpuUsage, _maxMemUsage);
+
+            LogManager.GetCurrentClassLogger().Info(json.ToString());
+
+            Diagnostics.Shutdown();
+
+            return json;
+        }
+
+        static void StartTimberWinR(string configFile, string logLevel, string logFileDir, bool enableLiveMonitor)
+        {
+            _timberWinR = new TimberWinR.Manager(configFile, logLevel, logFileDir, enableLiveMonitor, _cancellationTokenSource.Token, false);
+            _timberWinR.OnConfigurationProcessed += TimberWinROnOnConfigurationProcessed;
+            _timberWinR.Start(_cancellationTokenSource.Token);
+            Diagnostics = new Diagnostics.Diagnostics(_timberWinR, _cancellationTokenSource.Token, 5141);
+        }
+
+        private static void TimberWinROnOnConfigurationProcessed(Configuration configuration)
+        {         
+            if (!string.IsNullOrEmpty(Options.RedisHost) && configuration.RedisOutputs != null && configuration.RedisOutputs.Count() > 0)
+            {
+                foreach (var ro in configuration.RedisOutputs)
+                {
+                    ro.Host = new string[] { Options.RedisHost };
+                }
+            }
+
+        }
+
+        static void InitializeLogging(string logLevel)
+        {
+            var loggingConfiguration = new LoggingConfiguration();
+
+            // Create our default targets
+            var coloredConsoleTarget = new ColoredConsoleTarget();
+
+            var logFileDir = ".";
+
+            Target fileTarget = CreateDefaultFileTarget(logFileDir);
+
+            loggingConfiguration.AddTarget("Console", coloredConsoleTarget);
+            loggingConfiguration.AddTarget("DailyFile", fileTarget);
+
+            // The LogLevel.Trace means has to be at least Trace to show up on console
+            loggingConfiguration.LoggingRules.Add(new LoggingRule("*", LogLevel.Trace, coloredConsoleTarget));
+            // LogLevel.Debug means has to be at least Debug to show up in logfile
+            loggingConfiguration.LoggingRules.Add(new LoggingRule("*", LogLevel.Debug, fileTarget));
+
+            LogManager.Configuration = loggingConfiguration;
+            LogManager.EnableLogging();
+
+            LogManager.GlobalThreshold = LogLevel.FromString(logLevel);
+        }
+
+        static FileTarget CreateDefaultFileTarget(string logPath)
+        {
+            return new FileTarget
+            {
+                ArchiveEvery = FileArchivePeriod.None,
+                ArchiveAboveSize = 5 * 1024 * 1024,
+                MaxArchiveFiles = 5,
+                BufferSize = 10,
+                FileName = Path.Combine(logPath, "TimberWinR.TestGenerator", "TimberWinRTestGen.log"),
+                ArchiveFileName = Path.Combine(logPath, "TimberWinR-TestGenerator_log-{#######}.log"),
+            };
+        }
+
+        static void ResetTests(CommandLineOptions options)
+        {
+            if (File.Exists(".timberwinrdb"))
+                File.Delete(".timberwinrdb");
+
+            if (File.Exists("TimberWinR.TestGenerator\\TimberWinRTestGen.log"))
+                File.Delete("TimberWinR.TestGenerator\\TimberWinRTestGen.log");
+
+            if (File.Exists("TimberWinR\\TimberWinR.log"))
+                File.Delete("TimberWinR\\TimberWinR.log");
+
+            if (options.JsonLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonLogFiles)
+                {
+                    if (File.Exists(logFile))
+                        File.Delete(logFile);
+                }
+            }
+
+            if (options.JsonRollingLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonRollingLogFiles)
+                {
+                    if (File.Exists(logFile))
+                        File.Delete(logFile);
+                }
+            }
+        }
+
+        static Task[] RunGenerators(CommandLineOptions options)
+        {
+            _monitorTask = Task.Factory.StartNew(() =>
+                {
+                    using (var syncHandle = new ManualResetEventSlim())
+                    {
+                        try
+                        {
+                            // Execute the query
+                            while (!_cancellationTokenSource.Token.IsCancellationRequested)
+                            {
+                                sampleUsages();
+                                // LogManager.GetCurrentClassLogger().Info("Starting CPU Usage: {0}, RAM Usage: {1}", getCurrentCpuUsage(), getAvailableRAM());
+                                syncHandle.Wait(TimeSpan.FromMilliseconds(options.JsonRate), _cancellationTokenSource.Token);
+                            }
+                        }
+                        catch (OperationCanceledException)
+                        {
+                        }
+                        catch (Exception ex)
+                        {
+                            LogManager.GetCurrentClassLogger().Error(ex);
+                        }
+                    }
+                }, _cancellationTokenSource.Token, TaskCreationOptions.LongRunning, TaskScheduler.Current);
+
+            StartJson(options);
+            StartJsonRolling(options);
+            StartUdp(options);
+            StartTcp(options);
+
+            return _tasks.ToArray();
+        }
+
+        static void StartJson(CommandLineOptions options)
+        {
+            if (options.JsonLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonLogFiles)
+                {
+                    _totalMessagesToSend += options.NumMessages;
+
+                    if (options.Verbose)
+                        LogManager.GetCurrentClassLogger()
+                            .Info("Starting LogFile Generator for {0}",
+                                Path.GetFullPath(Path.Combine(options.JsonLogDir, logFile)));
+                    _tasks.Add(Task.Factory.StartNew(() =>
+                    {
+                        var p = new JsonLogFileTestParameters()
+                        {
+                            NumMessages = options.NumMessages,
+                            LogFileDir = options.JsonLogDir,
+                            LogFileName = logFile,
+                            SleepTimeMilliseconds = options.JsonRate
+                        };
+                        JsonLogFileGenerator.Generate(p);
+                        Thread.Sleep(250);
+                    }));
+
+                }
+            }
+        }
+
+        private static void StartJsonRolling(CommandLineOptions options)
+        {
+            if (options.JsonRollingLogFiles.Length > 0)
+            {
+                foreach (var logFile in options.JsonRollingLogFiles)
+                {
+                    _totalMessagesToSend += options.NumMessages;
+
+                    if (options.Verbose)
+                        LogManager.GetCurrentClassLogger()
+                            .Info("Starting RollingLogFile Generator for {0}",
+                                Path.GetFullPath(Path.Combine(options.JsonLogDir, logFile)));
+                    _tasks.Add(Task.Factory.StartNew(() =>
+                    {
+                        var p = new JsonLogFileTestParameters()
+                        {
+                            NumMessages = options.NumMessages,
+                            LogFileDir = options.JsonLogDir,
+                            LogFileName = logFile,
+                            SleepTimeMilliseconds = options.JsonRate
+                        };
+                        JsonRollingLogFileGenerator.Generate(p);
+                        Thread.Sleep(250);
+                    }));
+
+                }
+            }
+        }
+
+        static void StartUdp(CommandLineOptions options)
+        {
+            if (options.Udp > 0)
+            {
+                if (options.Verbose)
+                    LogManager.GetCurrentClassLogger()
+                        .Info("Starting UDP Generator for {0}:{1}", options.UdpHost, options.Udp);
+
+                _tasks.Add(Task.Factory.StartNew(() =>
+                {
+                    var p = new UdpTestParameters()
+                    {
+                        Port = options.Udp,
+                        Host = options.UdpHost,
+                        NumMessages = options.NumMessages,
+                        SleepTimeMilliseconds = options.UdpRate
+                    };
+                    _totalMessagesToSend += UdpTestGenerator.Generate(p);
+                }));
+            }
+        }
+
+        static void StartTcp(CommandLineOptions options)
+        {
+            if (options.Tcp > 0)
+            {
+                if (options.Verbose)
+                    LogManager.GetCurrentClassLogger()
+                        .Info("Starting Tcp Generator for {0}:{1}", options.TcpHost, options.Tcp);
+
+                _totalMessagesToSend += options.NumMessages;
+
+                _tasks.Add(Task.Factory.StartNew(() =>
+                {
+                    var p = new TcpTestParameters()
+                    {
+                        Port = options.Tcp,
+                        Host = options.TcpHost,
+                        NumMessages = options.NumMessages,
+                        SleepTimeMilliseconds = options.TcpRate
+                    };
+                    TcpTestGenerator.Generate(p);
+                }));
+            }
+        }
+
+    }
+}

TimberWinR.TestGenerator/Properties/AssemblyInfo.cs

@@ -0,0 +1,36 @@
+using System.Reflection;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+
+// General Information about an assembly is controlled through the following 
+// set of attributes. Change these attribute values to modify the information
+// associated with an assembly.
+[assembly: AssemblyTitle("TimberWinR.TestGenerator")]
+[assembly: AssemblyDescription("")]
+[assembly: AssemblyConfiguration("")]
+[assembly: AssemblyCompany("")]
+[assembly: AssemblyProduct("TimberWinR.TestGenerator")]
+[assembly: AssemblyCopyright("Copyright ©  2015")]
+[assembly: AssemblyTrademark("")]
+[assembly: AssemblyCulture("")]
+
+// Setting ComVisible to false makes the types in this assembly not visible 
+// to COM components.  If you need to access a type in this assembly from 
+// COM, set the ComVisible attribute to true on that type.
+[assembly: ComVisible(false)]
+
+// The following GUID is for the ID of the typelib if this project is exposed to COM
+[assembly: Guid("a56bf91c-c5f8-4771-8ef8-ab9ad28179c4")]
+
+// Version information for an assembly consists of the following four values:
+//
+//      Major Version
+//      Minor Version 
+//      Build Number
+//      Revision
+//
+// You can specify all the values or you can default the Build and Revision Numbers 
+// by using the '*' as shown below:
+// [assembly: AssemblyVersion("1.0.*")]
+[assembly: AssemblyVersion("1.0.0.0")]
+[assembly: AssemblyFileVersion("1.0.0.0")]

TimberWinR.TestGenerator/RedisTestGenerator.cs

@@ -0,0 +1,49 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json.Linq;
+using ServiceStack.Redis;
+
+namespace TimberWinR.TestGenerator
+{
+    class RedisTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public RedisTestParameters()
+        {
+            NumMessages = 100;
+            Port = 6379;
+            Host = "localhost";
+        }
+    }
+
+    class RedisTestGenerator
+    {
+        public static void Generate(RedisTestParameters parms)
+        {
+            var hostName = System.Environment.MachineName + "." +
+               Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                   "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+            var rc = new RedisClient(parms.Host, parms.Port);
+
+            for (int i = 0; i < parms.NumMessages; i++)
+            {
+                JObject o = new JObject
+                {
+                    {"Application", "redis-generator"},               
+                    {"Host", hostName},
+                    {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                    {"Type", "redis"},                
+                    {"Message", "redis message " + DateTime.UtcNow.ToString("o")},
+                    {"Index", "logstash"}
+                };
+                byte[] bytes = System.Text.Encoding.UTF8.GetBytes(o.ToString());
+                var restult = rc.RPush("logstash", bytes);
+            }
+        }
+    }
+}

TimberWinR.TestGenerator/TcpTestGenerator.cs

@@ -0,0 +1,62 @@
+using System.Threading;
+using Newtonsoft.Json.Linq;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+using ServiceStack.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class TcpTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public TcpTestParameters()
+        {
+            NumMessages = 100;
+            Port = 5140;
+            Host = "localhost";
+            SleepTimeMilliseconds = 10;
+        }
+    }
+
+    class TcpTestGenerator
+    {
+        public static int Generate(TcpTestParameters parms)
+        {
+            TcpClient server = new TcpClient(parms.Host, parms.Port);
+            
+            var hostName = System.Environment.MachineName + "." +
+                       Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                           "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+
+
+            using (NetworkStream stream = server.GetStream())
+            {              
+                for (int i = 0; i < parms.NumMessages; i++)
+                {
+                    JObject o = new JObject
+                    {
+                        {"Application", "tcp-generator"},
+                        {"Host", hostName},
+                        {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                        {"Type", "tcp"},
+                        {"Message", "tcp message " + DateTime.UtcNow.ToString("o")},
+                        {"Index", "logstash"}
+                    };
+                    byte[] data = Encoding.UTF8.GetBytes(string.Format("{0}\n", o.ToString()));
+                    stream.Write(data, 0, data.Length);
+                    Thread.Sleep(parms.SleepTimeMilliseconds);
+                }
+            }
+
+            return parms.NumMessages;
+        }
+
+    }
+}

TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj

@@ -0,0 +1,133 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="12.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <Import Project="$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props" Condition="Exists('$(MSBuildExtensionsPath)\$(MSBuildToolsVersion)\Microsoft.Common.props')" />
+  <PropertyGroup>
+    <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
+    <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
+    <ProjectGuid>{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}</ProjectGuid>
+    <OutputType>Exe</OutputType>
+    <AppDesignerFolder>Properties</AppDesignerFolder>
+    <RootNamespace>TimberWinR.TestGenerator</RootNamespace>
+    <AssemblyName>TimberWinR.TestGenerator</AssemblyName>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
+    <FileAlignment>512</FileAlignment>
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugSymbols>true</DebugSymbols>
+    <DebugType>full</DebugType>
+    <Optimize>false</Optimize>
+    <OutputPath>bin\Debug\</OutputPath>
+    <DefineConstants>DEBUG;TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
+    <PlatformTarget>AnyCPU</PlatformTarget>
+    <DebugType>pdbonly</DebugType>
+    <Optimize>true</Optimize>
+    <OutputPath>bin\Release\</OutputPath>
+    <DefineConstants>TRACE</DefineConstants>
+    <ErrorReport>prompt</ErrorReport>
+    <WarningLevel>4</WarningLevel>
+  </PropertyGroup>
+  <ItemGroup>
+    <Reference Include="CommandLine">
+      <HintPath>..\packages\CommandLineParser.1.9.71\lib\net40\CommandLine.dll</HintPath>
+    </Reference>
+    <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll</HintPath>
+    </Reference>
+    <Reference Include="NLog, Version=3.2.0.0, Culture=neutral, PublicKeyToken=5120e14c03d0593c, processorArchitecture=MSIL">
+      <SpecificVersion>False</SpecificVersion>
+      <HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Common">
+      <HintPath>..\packages\ServiceStack.Common.Signed.4.0.38\lib\net40\ServiceStack.Common.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Interfaces">
+      <HintPath>..\packages\ServiceStack.Interfaces.4.0.38\lib\portable-wp80+sl5+net40+win8+monotouch+monoandroid\ServiceStack.Interfaces.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Redis">
+      <HintPath>..\packages\ServiceStack.Redis.Signed.4.0.38\lib\net40\ServiceStack.Redis.dll</HintPath>
+    </Reference>
+    <Reference Include="ServiceStack.Text">
+      <HintPath>..\packages\ServiceStack.Text.Signed.4.0.38\lib\net40\ServiceStack.Text.dll</HintPath>
+    </Reference>
+    <Reference Include="System" />
+    <Reference Include="System.Core" />
+    <Reference Include="System.Xml.Linq" />
+    <Reference Include="System.Data.DataSetExtensions" />
+    <Reference Include="Microsoft.CSharp" />
+    <Reference Include="System.Data" />
+    <Reference Include="System.Xml" />
+  </ItemGroup>
+  <ItemGroup>
+    <Compile Include="CommandLineOptions.cs" />
+    <Compile Include="Dynamic.cs" />
+    <Compile Include="LogFileGenerator.cs" />
+    <Compile Include="Program.cs" />
+    <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="RedisTestGenerator.cs" />
+    <Compile Include="TcpTestGenerator.cs" />
+    <Compile Include="JsonLogFileGenerator.cs" />
+    <Compile Include="UdpTestGenerator.cs" />
+  </ItemGroup>
+  <ItemGroup>
+    <Content Include="default.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results1.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test1.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test1-twconfig.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <None Include="packages.config" />
+    <Content Include="test2-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results2.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test3.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="test3-tw.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="results3.json">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">
+      <Project>{4ef96a08-21db-4178-be44-70dae594632c}</Project>
+      <Name>TimberWinR</Name>
+    </ProjectReference>
+  </ItemGroup>
+  <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
+  <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
+       Other similar extension points exist, see Microsoft.Common.targets.
+  <Target Name="BeforeBuild">
+  </Target>
+  <Target Name="AfterBuild">
+  </Target>
+  -->
+</Project>

TimberWinR.TestGenerator/UdpTestGenerator.cs

@@ -0,0 +1,70 @@
+using System.Threading;
+using Newtonsoft.Json.Linq;
+using NLog;
+using NLog.Config;
+using NLog.Targets;
+
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Net;
+using System.Net.Sockets;
+using System.Text;
+
+namespace TimberWinR.TestGenerator
+{
+    class UdpTestParameters
+    {
+        public int Port { get; set; }
+        public string Host { get; set; }
+        public int NumMessages { get; set; }
+        public int SleepTimeMilliseconds { get; set; }
+        public UdpTestParameters()
+        {
+            NumMessages = 100;
+            Port = 6379;
+            Host = "localhost";
+            SleepTimeMilliseconds = 10;
+        }
+    }
+
+    class UdpTestGenerator
+    {
+        public static int Generate(UdpTestParameters parms)
+        {
+            var hostName = System.Environment.MachineName + "." +
+               Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                   "SYSTEM\\CurrentControlSet\\services\\Tcpip\\Parameters").GetValue("Domain", "").ToString();
+     
+            IPAddress broadcast;
+            if (!IPAddress.TryParse(parms.Host, out broadcast))
+                broadcast = Dns.GetHostEntry(parms.Host).AddressList[0];
+
+            Socket s = new Socket(broadcast.AddressFamily, SocketType.Dgram, ProtocolType.Udp);
+
+            LogManager.GetCurrentClassLogger().Info("Start UDP Generation");
+
+            for (int i = 0; i < parms.NumMessages; i++)
+            {
+                JObject o = new JObject
+                {
+                    {"Application", "udp-generator"},               
+                    {"Host", hostName},
+                    {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
+                    {"Type", "udp"},                
+                    {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
+                    {"Index", "logstash"}
+                };
+                byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString());
+                IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
+                s.SendTo(sendbuf, ep);
+                Thread.Sleep(parms.SleepTimeMilliseconds);
+            }
+
+            LogManager.GetCurrentClassLogger().Info("Finished UDP Generation");
+
+            return parms.NumMessages;
+        }
+
+    }
+}

TimberWinR.TestGenerator/default.json

@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "TailFiles": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}

TimberWinR.TestGenerator/packages.config

@@ -0,0 +1,10 @@
+<?xml version="1.0" encoding="utf-8"?>
+<packages>
+  <package id="CommandLineParser" version="1.9.71" targetFramework="net40" />
+  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net40" />
+  <package id="NLog" version="3.2.0.0" targetFramework="net40" />
+  <package id="ServiceStack.Common.Signed" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Interfaces" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Redis.Signed" version="4.0.38" targetFramework="net40" />
+  <package id="ServiceStack.Text.Signed" version="4.0.38" targetFramework="net40" />
+</packages>

TimberWinR.TestGenerator/results1.json

@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 20"                
+                }
+            },
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 1234",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 20"              
+                }
+            }
+        ]
+    }
+}

TimberWinR.TestGenerator/results2.json

@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            },
+            {
+                "udp": {
+                    "test1: message sent count": "[messages] == 1234",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            }
+        ]
+    }
+}

TimberWinR.TestGenerator/results3.json

@@ -0,0 +1,20 @@
+{
+    "Results": {
+        "Inputs": [
+            {
+                "taillog": {
+                    "test1: message sent count": "[messages] == 7404",
+                    "test2: average cpu": "[avgCpuUsage] <= 30",
+                    "test3: maximum memory": "[maxMemUsage] <= 15"
+                }
+            },
+            {
+                "tcp": {
+                    "test4: message sent count": "[messages] == 1234",
+                    "test5: average cpu": "[avgCpuUsage] <= 30",
+                    "test6: maximum memory": "[maxMemUsage] <= 15"
+                }
+            }
+        ]
+    }
+}

TimberWinR.TestGenerator/test1-twconfig.json

@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "TailFiles": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}

TimberWinR.TestGenerator/test1.json

@@ -0,0 +1,15 @@
+{
+    "test": "Test 1",
+    "arguments": {
+        "--testFile": "test1.json",
+        "--testDir": "test1",
+        "--timberWinRConfig": "test1-twconfig.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--udp-host":  "::1",
+        "--udp": "5140",
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results1.json"
+    }
+}

TimberWinR.TestGenerator/test2-tw.json

@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Udp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "Logs": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}

TimberWinR.TestGenerator/test2.json

@@ -0,0 +1,14 @@
+{
+    "test": "Test 2",
+    "arguments": {
+        "--testFile": "test2.json",
+        "--testDir": "test2",
+        "--timberWinRConfig": "test2-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--udp": "5140",
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results2.json"
+    }
+}

TimberWinR.TestGenerator/test3-tw.json

@@ -0,0 +1,45 @@
+{
+    "TimberWinR": {
+        "Inputs": {           
+            "Tcp": [
+                {
+                    "_comment": "Output from NLog",
+                    "port": 5140
+                }
+            ],
+             "Logs": [
+                {  
+                    "interval": 5,
+                    "logSource": "log files",
+                    "location": "*.jlog",
+                    "recurse": -1
+                }
+            ]
+        },
+        "Filters": [
+            {
+                "grok": {
+                    "condition": "\"[EventTypeName]\" == \"Information Event\"",
+                    "match": [
+                        "Text",
+                        ""
+                    ],
+                    "drop": "true"
+                }
+            }
+        ],
+        "Outputs": {
+            "Redis": [
+                {
+                    "_comment": "Change the host to your Redis instance",
+                    "port": 6379,
+                    "batch_count": 500,
+                    "threads":  2,
+                    "host": [
+                        "tstlexiceapp006.vistaprint.svc"
+                    ]
+                }
+            ]
+        }
+    }
+}

TimberWinR.TestGenerator/test3.json

@@ -0,0 +1,14 @@
+{
+    "test": "Test 3",
+    "arguments": {
+        "--testFile": "test3.json",
+        "--testDir": "test3",
+        "--timberWinRConfig": "test3-tw.json",
+        "--numMessages": 1234,
+        "--logLevel": "debug",
+        "--tcp": "5140",
+        "--jroll": ["r1.jlog", "r2.jlog"],
+        "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
+        "--resultsFile": "results3.json"
+    }
+}

TimberWinR.UnitTests/GrokFilterTests.cs

@@ -223,6 +223,7 @@ namespace TimberWinR.UnitTests
                     }
                 },
                 {"type", "Win32-FileLog"},
+                {"Type", "Win32-MyType"},
                 {"ComputerName", "dev.mycompany.net"}
             };
 
@@ -281,11 +282,35 @@ namespace TimberWinR.UnitTests
                 }
             }";
 
-            // Positive Tests
-            Configuration c = Configuration.FromString(grokJson1);
+            string grokJson4 = @"{  
+            ""TimberWinR"":{        
+                ""Filters"":[  
+	                {  
+		            ""grok"":{  
+		                ""condition"": ""!\""[Type]\"".StartsWith(\""[\"") && !\""[Type]\"".EndsWith(\""]\"") && (\""[type]\"" == \""Win32-FileLog\"")"",
+		                ""match"":[  
+			                ""Text"",
+			                """"
+		                ],
+                        ""remove_tag"":[  
+                            ""tag1""                          
+                        ]		               
+		            }
+	                }]
+                }
+            }";
+
+            
+            Configuration c = Configuration.FromString(grokJson4);
             Grok grok = c.Filters.First() as Grok;
             Assert.IsTrue(grok.Apply(json));
 
+
+            // Positive Tests
+            c = Configuration.FromString(grokJson1);
+            grok = c.Filters.First() as Grok;
+            Assert.IsTrue(grok.Apply(json));
+
             c = Configuration.FromString(grokJson2);
             grok = c.Filters.First() as Grok;
             Assert.IsTrue(grok.Apply(json));

TimberWinR.UnitTests/Inputs/IisW3CRowReaderTests.cs

@@ -1,60 +0,0 @@
-namespace TimberWinR.UnitTests.Inputs
-{
-    using System;
-    using System.Collections.Generic;
-
-    using Interop.MSUtil;
-
-    using Moq;
-
-    using NUnit.Framework;
-
-    using TimberWinR.Inputs;
-    using TimberWinR.Parser;
-
-    [TestFixture]
-    public class IisW3CRowReaderTests : TestBase
-    {
-        private IisW3CRowReader reader;
-
-        public override void Setup()
-        {
-            base.Setup();
-            var fields = new List<Field>
-                             {
-                                 new Field("date", "DateTime"),
-                                 new Field("time", "DateTime"),
-                                 new Field("uri")
-                             };
-            this.reader = new IisW3CRowReader(fields);
-
-            var recordset = this.GetRecordsetMock();
-            this.reader.ReadColumnMap(recordset.Object);
-        }
-
-        [Test]
-        public void GivenValidRowAddsTimestampColumn()
-        {
-            var record = this.MockRepository.Create<ILogRecord>();
-            record.Setup(x => x.getValue("date")).Returns(new DateTime(2014, 11, 30));
-            record.Setup(x => x.getValue("time")).Returns(new DateTime(1, 1, 1, 18, 45, 37, 590));
-            record.Setup(x => x.getValue("uri")).Returns("http://somedomain.com/someurl");
-
-            var json = this.reader.ReadToJson(record.Object);
-
-            Assert.AreEqual("2014-11-30T18:45:37.000Z", json["@timestamp"].ToString());
-            Assert.AreEqual("http://somedomain.com/someurl", json["uri"].ToString());
-        }
-
-        private Mock<ILogRecordset> GetRecordsetMock()
-        {
-            var recordset = this.MockRepository.Create<ILogRecordset>();
-            recordset.Setup(x => x.getColumnCount()).Returns(3);
-
-            recordset.Setup(x => x.getColumnName(0)).Returns("date");
-            recordset.Setup(x => x.getColumnName(1)).Returns("time");
-            recordset.Setup(x => x.getColumnName(2)).Returns("uri");
-            return recordset;
-        }
-    }
-}

TimberWinR.UnitTests/TailFileTests.cs

@@ -28,7 +28,7 @@ namespace TimberWinR.UnitTests
             var mgr = new Manager();
             mgr.LogfileDir = ".";
 
-            var tf = new TailFile();
+            var tf = new TailFileArguments();
             var cancelTokenSource = new CancellationTokenSource();
             tf.Location = "TestTailFile1.log";
 

TimberWinR.UnitTests/TestBase.cs

@@ -1,23 +0,0 @@
-namespace TimberWinR.UnitTests
-{
-    using Moq;
-
-    using NUnit.Framework;
-
-    public class TestBase
-    {
-        public MockRepository MockRepository { get; private set; }
-
-        [SetUp]
-        public virtual void Setup()
-        {
-            this.MockRepository = new MockRepository(MockBehavior.Default);
-        }
-
-        [TearDown]
-        public virtual void TearDown()
-        {
-            this.MockRepository.VerifyAll();
-        }
-    }
-}

TimberWinR.UnitTests/TimberWinR.UnitTests.csproj

@@ -12,6 +12,8 @@
     <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -38,16 +40,13 @@
       <EmbedInteropTypes>False</EmbedInteropTypes>
       <HintPath>..\TimberWinR\lib\com-logparser\Interop.MSUtil.dll</HintPath>
     </Reference>
-    <Reference Include="Moq">
-      <HintPath>..\packages\Moq.4.2.1409.1722\lib\net40\Moq.dll</HintPath>
-    </Reference>
     <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.8\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <Private>True</Private>
     </Reference>
-    <Reference Include="nunit.framework, Version=2.6.3.13283, Culture=neutral, PublicKeyToken=96d09a1eb7f44a77, processorArchitecture=MSIL">
-      <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\NUnit.2.6.3\lib\nunit.framework.dll</HintPath>
+    <Reference Include="nunit.framework">
+      <HintPath>..\packages\NUnit.2.6.4\lib\nunit.framework.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
@@ -62,14 +61,12 @@
     <Compile Include="DateFilterTests.cs" />
     <Compile Include="FakeRediServer.cs" />
     <Compile Include="GeoIPFilterTests.cs" />
-    <Compile Include="Inputs\IisW3CRowReaderTests.cs" />
     <Compile Include="JsonFilterTests.cs" />
     <Compile Include="GrokFilterTests.cs" />
     <Compile Include="MultilineTests.cs" />
     <Compile Include="Parser\ElasticsearchOutputTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="TailFileTests.cs" />
-    <Compile Include="TestBase.cs" />
     <Compile Include="TestDynamicBatchCount.cs" />
   </ItemGroup>
   <ItemGroup>
@@ -82,6 +79,7 @@
     <Service Include="{82A7F48D-3B50-4B1E-B82E-3ADA8210C358}" />
   </ItemGroup>
   <ItemGroup>
+    <None Include="app.config" />
     <None Include="packages.config">
       <SubType>Designer</SubType>
     </None>
@@ -94,7 +92,17 @@
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
   </ItemGroup>
+  <ItemGroup>
+    <Folder Include="Inputs\" />
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">

TimberWinR.UnitTests/app.config

@@ -0,0 +1,11 @@
+<?xml version="1.0" encoding="utf-8"?>
+<configuration>
+  <runtime>
+    <assemblyBinding xmlns="urn:schemas-microsoft-com:asm.v1">
+      <dependentAssembly>
+        <assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
+        <bindingRedirect oldVersion="0.0.0.0-4.5.0.0" newVersion="4.5.0.0" />
+      </dependentAssembly>
+    </assemblyBinding>
+  </runtime>
+</configuration>

TimberWinR.UnitTests/packages.config

@@ -1,6 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<packages>
-  <package id="Moq" version="4.2.1409.1722" targetFramework="net40" />
-  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
-  <package id="NUnit" version="2.6.3" targetFramework="net40" />
+<packages> 
+  <package id="Newtonsoft.Json" version="6.0.8" targetFramework="net40" />
+  <package id="NUnit" version="2.6.4" targetFramework="net40" />
 </packages>

TimberWinR.sln

@@ -7,6 +7,9 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR", "TimberWinR\Ti
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".nuget", ".nuget", "{2C4AC7DB-018D-4CCA-9579-06AC5AD2C9D9}"
 	ProjectSection(SolutionItems) = preProject
+		.nuget\NuGet.Config = .nuget\NuGet.Config
+		.nuget\NuGet.exe = .nuget\NuGet.exe
+		.nuget\NuGet.targets = .nuget\NuGet.targets
 		.nuget\packages.config = .nuget\packages.config
 	EndProjectSection
 EndProject
@@ -35,6 +38,8 @@ Project("{930C7802-8A8C-48F9-8165-68863BCCD9DD}") = "TimberWinR.Wix", "TimberWix
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR.ExtractID", "TimberWinR.ExtractID\TimberWinR.ExtractID.csproj", "{99096939-E9DD-4499-883D-4726745A5843}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TimberWinR.TestGenerator", "TimberWinR.TestGenerator\TimberWinR.TestGenerator.csproj", "{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -97,11 +102,18 @@ Global
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|Mixed Platforms.Build.0 = Release|Any CPU
 		{99096939-E9DD-4499-883D-4726745A5843}.Release|x86.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Mixed Platforms.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|Mixed Platforms.Build.0 = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Any CPU.Build.0 = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Mixed Platforms.ActiveCfg = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|Mixed Platforms.Build.0 = Release|Any CPU
+		{F3960D6E-1EA0-4F4E-8F08-82FC185A0D29}.Release|x86.ActiveCfg = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
 EndGlobal

TimberWinR/Configuration.cs

@@ -44,7 +44,7 @@ namespace TimberWinR
         {
             get { return _redisOutputs; }
         }
-
+       
 
         private List<ElasticsearchOutputParameters> _elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
         public IEnumerable<ElasticsearchOutputParameters> ElasticsearchOutputs
@@ -76,8 +76,8 @@ namespace TimberWinR
             get { return _logs; }
         }
 
-        private List<TailFile> _tails = new List<TailFile>();
-        public IEnumerable<TailFile> TailFiles
+        private List<TailFileArguments> _tails = new List<TailFileArguments>();
+        public IEnumerable<TailFileArguments> TailFiles
         {
             get { return _tails; }
         }    
@@ -241,8 +241,8 @@ namespace TimberWinR
                     c._stdins.AddRange(x.TimberWinR.Inputs.Stdins.ToList());
                 if (x.TimberWinR.Inputs.Logs != null)
                     c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList());
-                if (x.TimberWinR.Inputs.TailFiles != null)
-                    c._tails.AddRange(x.TimberWinR.Inputs.TailFiles.ToList());
+                if (x.TimberWinR.Inputs.TailFilesArguments != null)
+                    c._tails.AddRange(x.TimberWinR.Inputs.TailFilesArguments.ToList());
                 if (x.TimberWinR.Inputs.Tcps != null)
                     c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList());
                 if (x.TimberWinR.Inputs.Udps != null)

TimberWinR/Diagnostics/Diagnostics.cs

@@ -21,7 +21,7 @@ namespace TimberWinR.Diagnostics
         private CancellationToken CancelToken { get; set; }
         public int Port { get; set; }
         public Manager Manager { get; set; }
-
+        public bool Stop { get; set; }
         private HttpListener web;
 
         public Diagnostics(Manager manager, CancellationToken cancelToken, int port = 5141)
@@ -49,44 +49,60 @@ namespace TimberWinR.Diagnostics
         }
 
 
-        private void DiagnosticCallback(IAsyncResult result)
-        {       
-            if (web == null)         
-                return;
-
-            var context = web.EndGetContext(result);
-            var response = context.Response;
-
+        public JObject DiagnosticsOutput()
+        {
             JObject json = new JObject(
-                new JProperty("timberwinr",
-                    new JObject(
-                        new JProperty("version", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
-                        new JProperty("messages", Manager.NumMessages),
-                        new JProperty("startedon", Manager.StartedOn),
-                        new JProperty("configfile", Manager.JsonConfig),
-                        new JProperty("logdir", Manager.LogfileDir),
-                        new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
-                        new JProperty("inputs",
-                            new JArray(
-                                from i in Manager.Listeners
-                                select new JObject(i.ToJson()))),
-                        new JProperty("filters",
-                            new JArray(
-                                from f in Manager.Config.Filters
-                                select new JObject(f.ToJson()))),
-                        new JProperty("outputs",
-                            new JArray(
-                                from o in Manager.Outputs
-                                select new JObject(o.ToJson()))))));
-
-            response.StatusCode = (int)HttpStatusCode.OK;
-            response.StatusDescription = HttpStatusCode.OK.ToString();
-            byte[] buffer = Encoding.UTF8.GetBytes(json.ToString());
-            response.ContentLength64 = buffer.Length;
-            response.OutputStream.Write(buffer, 0, buffer.Length);
-            response.OutputStream.Close();
+                           new JProperty("timberwinr",
+                               new JObject(
+                                   new JProperty("version", Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                                   new JProperty("messages", Manager.NumMessages),
+                                   new JProperty("startedon", Manager.StartedOn),
+                                   new JProperty("configfile", Manager.JsonConfig),
+                                   new JProperty("logdir", Manager.LogfileDir),
+                                   new JProperty("logginglevel", LogManager.GlobalThreshold.ToString()),
+                                   new JProperty("inputs",
+                                       new JArray(
+                                           from i in Manager.Listeners
+                                           select new JObject(i.ToJson()))),
+                                   new JProperty("filters",
+                                       new JArray(
+                                           from f in Manager.Config.Filters
+                                           select new JObject(f.ToJson()))),
+                                   new JProperty("outputs",
+                                       new JArray(
+                                           from o in Manager.Outputs
+                                           select new JObject(o.ToJson()))))));
+            return json;
         }
 
+        private void DiagnosticCallback(IAsyncResult result)
+        {
+            if (web == null)
+                return;
+
+            try
+            {
+                var context = web.EndGetContext(result);
+                var response = context.Response;
+                var json = DiagnosticsOutput();
+
+                response.StatusCode = (int) HttpStatusCode.OK;
+                response.StatusDescription = HttpStatusCode.OK.ToString();
+                byte[] buffer = Encoding.UTF8.GetBytes(json.ToString());
+                response.ContentLength64 = buffer.Length;
+                response.OutputStream.Write(buffer, 0, buffer.Length);
+                response.OutputStream.Close();
+            }
+            catch (SocketException)
+            {
+                // Shutdown
+            }
+            catch (Exception ex)
+            {
+                if (!Stop)
+                    LogManager.GetCurrentClassLogger().Error(ex);
+            }
+        }
 
         private void HttpListen(object o)
         {
@@ -97,14 +113,19 @@ namespace TimberWinR.Diagnostics
                 web.Start();
 
                 while (web != null && web.IsListening)
-                {                  
-                    processRequest();                  
-                }               
+                {
+                    processRequest();
+                }
+            }
+            catch (SocketException)
+            {
+                // Shutdown
             }
             catch (Exception ex)
             {
+                if (!Stop)
                 LogManager.GetCurrentClassLogger().Error("Diagnostic Listener Error: {0}", ex.ToString());
-            }           
+            }
         }
 
         private void ListenForClients(object olistener)
@@ -140,7 +161,7 @@ namespace TimberWinR.Diagnostics
             var tcpClient = (TcpClient)client;
             NetworkStream clientStream = null;
 
-            Console.WriteLine("Handle new diag client: {0}, {1}", tcpClient.Connected, tcpClient.Client.RemoteEndPoint.ToString());
+            //     Console.WriteLine("Handle new diag client: {0}, {1}", tcpClient.Connected, tcpClient.Client.RemoteEndPoint.ToString());
             try
             {
                 using (clientStream = tcpClient.GetStream())
@@ -183,7 +204,7 @@ namespace TimberWinR.Diagnostics
 
         public void Shutdown()
         {
-            
+            Stop = true;
             try
             {
                 if (web != null && web.IsListening)
@@ -193,11 +214,9 @@ namespace TimberWinR.Diagnostics
                     web = null;
                 }
             }
-            catch (Exception ex)
+            catch (Exception)
             {
-                LogManager.GetCurrentClassLogger().Error(ex);
             }
         }
-
     }
 }

TimberWinR/Filters/GrokFilter.cs

@@ -47,6 +47,7 @@ namespace TimberWinR.Parser
                          new JProperty("condition", Condition),
                          new JProperty("addfields", AddField),
                          new JProperty("addtags", AddTag),
+                         new JProperty("drop", DropIfMatch),
                          new JProperty("type", Type),
                          new JProperty("removefields", RemoveField),
                          new JProperty("removetag", RemoveTag)

TimberWinR/Inputs/InputListener.cs

@@ -1,5 +1,6 @@
 using System.IO;
 using System.Runtime.InteropServices;
+using Microsoft.Win32;
 using Newtonsoft.Json.Linq;
 using System;
 using System.Collections.Generic;
@@ -18,7 +19,9 @@ namespace TimberWinR.Inputs
         private string _typeName;
         public AutoResetEvent FinishedEvent { get; set; }
         public string CheckpointFileName { get; set; }
-      
+        private object _locker = new object();
+        public List<string> Files { get; set; }
+
         public string InputType
         {
             get { return _typeName; }
@@ -28,6 +31,7 @@ namespace TimberWinR.Inputs
 
         public InputListener(CancellationToken token, string typeName)
         {
+            Files = new List<string>();
             CheckpointFileName = Path.Combine(System.IO.Path.GetTempPath(), string.Format("{0}.lpc", Guid.NewGuid().ToString()));          
           
             this.FinishedEvent = new AutoResetEvent(false);
@@ -40,6 +44,19 @@ namespace TimberWinR.Inputs
                              .ToString();    
         }
 
+        public bool HaveSeenFile(string fileName)
+        {
+            return Files.Contains(fileName);
+        }
+
+        protected void SaveVisitedFileName(string fileName)
+        {
+            lock (_locker)
+            {
+                if (!HaveSeenFile(fileName))
+                    Files.Add(fileName);
+            }
+        }
         protected string ToPrintable(string inputString)
         {
             string asAscii = Encoding.ASCII.GetString(
@@ -58,17 +75,17 @@ namespace TimberWinR.Inputs
 
         public void Finished()
         {
-            LogManager.GetCurrentClassLogger().Info("Signaling Event Shutdown {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Signalling Event Shutdown {1}", Thread.CurrentThread.ManagedThreadId, InputType);
             FinishedEvent.Set();
-            LogManager.GetCurrentClassLogger().Info("Finished signaling Shutdown {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Finished signalling Shutdown {1}", Thread.CurrentThread.ManagedThreadId, InputType);
         }
+
         public virtual void Shutdown()
         {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1}", Thread.CurrentThread.ManagedThreadId, InputType);
 
             FinishedEvent.WaitOne();
-
-            LogManager.GetCurrentClassLogger().Info("Finished Wait For {0}", InputType);
+          
             try
             {
                 if (File.Exists(CheckpointFileName))
@@ -80,6 +97,32 @@ namespace TimberWinR.Inputs
             }          
         }
 
+        protected void EnsureRollingCaught()
+        {
+            try
+            {
+                const string mteKey = @"SYSTEM\CurrentControlSet\Control\FileSystem";
+
+                var mte = Microsoft.Win32.Registry.LocalMachine.OpenSubKey(mteKey).GetValue("MaximumTunnelEntries");
+                if (mte == null || (int)mte != 0)
+                {
+                    LogManager.GetCurrentClassLogger()
+                        .Error(
+                            "HKLM\\{0}\\MaximumTunnelEntries is not set to accurately detect log rolling, a DWORD value of 0 is required.",
+                            mteKey);
+                    Microsoft.Win32.Registry.LocalMachine.CreateSubKey(mteKey).SetValue("MaximumTunnelEntries", 0, RegistryValueKind.DWord);
+                    LogManager.GetCurrentClassLogger()
+                      .Error(
+                          "HKLM\\{0}\\MaximumTunnelEntries is now set to 0, A reboot is now required to fix this issue.  See http://support.microsoft.com/en-us/kb/172190 for details",
+                          mteKey);
+                }
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);
+            }           
+        }
+
         public virtual void AddDefaultFields(JObject json)
         {
             if (json["type"] == null)

TimberWinR/Inputs/LogsFileDatabase.cs

@@ -1,9 +1,12 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Linq;
 using System.Text;
+using System.Threading;
 using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
 using NLog;
 using TimberWinR.Parser;
 
@@ -34,7 +37,10 @@ namespace TimberWinR.Inputs
                 return ExistingFileTest(logName);
             }
         }
-
+       
+        //
+        // Lookup the database entry for this log file, returns null if there isnt one.
+        //
         private LogsFileDatabaseEntry FindFile(string logName)
         {
             lock (_locker)
@@ -43,6 +49,7 @@ namespace TimberWinR.Inputs
                 return existingEntry;
             }
         }
+
         private bool ExistingFileTest(string logName)
         {
             var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
@@ -67,45 +74,82 @@ namespace TimberWinR.Inputs
             var de = new LogsFileDatabaseEntry();
             lock (_locker)
             {
-                de.NewFile = true;
-                var fi = new FileInfo(logName);
+                var fi = new FileInfo(logName);             
                 de.FileName = logName;
-                de.Size = fi.Length;
+                de.LogFileExists = fi.Exists;
+                de.NewFile = true;
+                de.ProcessedFile = false;
+                de.LastPosition = fi.Length;
                 de.SampleTime = DateTime.UtcNow;
-                de.CreationTimeUtc = fi.CreationTimeUtc;
+                de.CreationTimeUtc = fi.CreationTimeUtc;               
+
                 Entries.Add(de);
                 WriteDatabaseFileNoLock();
             }
             return de;
-        }
+        }      
 
         public static LogsFileDatabaseEntry LookupLogFile(string logName)
         {
             LogsFileDatabaseEntry dbe = Instance.FindFile(logName);
             if (dbe == null)
                 dbe = Instance.AddFileEntry(logName);
-            else
-                dbe.NewFile = false;
-    
+
+            FileInfo fi = new FileInfo(logName);          
+
+            dbe.LogFileExists = fi.Exists;
+            var creationTime = fi.CreationTimeUtc;
+
+            if (dbe.LogFileExists && creationTime != dbe.CreationTimeUtc)
+                dbe.NewFile = true;           
+
+            dbe.CreationTimeUtc = creationTime;
+      
             return dbe;
         }
 
-        public static void Update(LogsFileDatabaseEntry dbe)
+        // Find all the non-existent entries and remove them.
+        private void PruneFiles()
         {
-            Instance.UpdateEntry(dbe);
-        }
-
-        private void UpdateEntry(LogsFileDatabaseEntry dbe)
-        {
-            lock(_locker)
+            lock (_locker)
             {
-                var fi = new FileInfo(dbe.FileName);
-                dbe.CreationTimeUtc = fi.CreationTimeUtc;
-                dbe.SampleTime = DateTime.UtcNow;
-                dbe.Size = fi.Length;
+                foreach(var entry in Entries.ToList())
+                {
+                    FileInfo fi = new FileInfo(entry.FileName);
+                    if (!fi.Exists)
+                        Entries.Remove(entry);
+                }
+                WriteDatabaseFileNoLock();
+            }
+        }
+
+        public static void Update(LogsFileDatabaseEntry dbe, bool processedFile, long lastOffset)
+        {
+            dbe.ProcessedFile = processedFile;
+            dbe.LogFileExists = File.Exists(dbe.FileName);
+            Instance.UpdateEntry(dbe, lastOffset);           
+        }
+
+        public static void Roll(LogsFileDatabaseEntry dbe)
+        {
+            dbe.ProcessedFile = false;
+            dbe.LastPosition = 0;          
+            Instance.UpdateEntry(dbe, 0);
+            dbe.NewFile = true;           
+        }
+
+        private void UpdateEntry(LogsFileDatabaseEntry dbe, long lastOffset)
+        {
+            lock (_locker)
+            {
+                var fi = new FileInfo(dbe.FileName);                
+                dbe.NewFile = !fi.Exists;
+                dbe.CreationTimeUtc = fi.CreationTimeUtc;
+                dbe.SampleTime = DateTime.UtcNow;                 
+                dbe.LastPosition = lastOffset;
+              
                 WriteDatabaseFileNoLock();
             }
-            
         }
         public static LogsFileDatabase Instance
         {
@@ -123,12 +167,19 @@ namespace TimberWinR.Inputs
                             instance.ReadDatabaseNoLock();
                         else
                             instance.WriteDatabaseFileNoLock();
+
+                        if (instance.Entries == null)
+                            instance.Entries = new List<LogsFileDatabaseEntry>();
+
+                        instance.PruneFiles();
                     }
                 }
                 return instance;
             }
         }
 
+
+        // Serialize in the Database
         private void ReadDatabaseNoLock()
         {
             try
@@ -152,7 +203,7 @@ namespace TimberWinR.Inputs
                 catch (Exception ex2)
                 {
                     LogManager.GetCurrentClassLogger().Info("Error Creating New Database '{0}': {1}", DatabaseFileName, ex2.ToString());
-                }              
+                }
             }
         }
         private void WriteDatabaseFileNoLock()
@@ -193,15 +244,30 @@ namespace TimberWinR.Inputs
 
     }
 
+
+    //
+    // Represents a log file to be tailed
+    // 
     public class LogsFileDatabaseEntry
     {
         [JsonIgnore]
         public bool NewFile { get; set; }
-        public string FileName { get; set; }
-        public Int64 MaxRecords { get; set; }
+        public bool ProcessedFile { get; set; }
+        public bool LogFileExists { get; set; }
+        public string FileName { get; set; }       
         public DateTime CreationTimeUtc { get; set; }
         public DateTime SampleTime { get; set; }
-        public long Size { get; set; }       
+        public long LastPosition { get; set; }
+        public long LinesProcessed
+        {
+            get { return _linesProcessed; }
+        }
+
+        private int _linesProcessed;
+        public void IncrementLineCount()
+        {
+            Interlocked.Increment(ref _linesProcessed);
+        }
     }
 
 }

TimberWinR/Inputs/LogsListener.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Net.Configuration;
 using System.Runtime.InteropServices;
@@ -9,6 +10,7 @@ using System.Threading;
 using System.Threading.Tasks;
 using System.IO;
 using Interop.MSUtil;
+using Microsoft.Win32;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using Newtonsoft.Json.Serialization;
@@ -27,42 +29,45 @@ namespace TimberWinR.Inputs
     /// </summary>
     public class LogsListener : InputListener
     {
+        private object _locker = new object();
         private int _pollingIntervalInSeconds;
         private TimberWinR.Parser.LogParameters _arguments;
         private long _receivedMessages;
-        private Dictionary<string, Int64> _logFileMaxRecords;
-        private Dictionary<string, DateTime> _logFileCreationTimes;
-        private Dictionary<string, DateTime> _logFileSampleTimes;
-        private Dictionary<string, long> _logFileSizes;
-        private CodecArguments _codecArguments;       
-        private ICodec _codec;   
-
+        private CodecArguments _codecArguments;
+        private ICodec _codec;
+      
         public bool Stop { get; set; }
+        public bool IsWildcardFilePattern { get; set; }
+
 
         public LogsListener(TimberWinR.Parser.LogParameters arguments, CancellationToken cancelToken)
             : base(cancelToken, "Win32-FileLog")
         {
             Stop = false;
+        
+            EnsureRollingCaught();
 
-            _codecArguments = arguments.CodecArguments;
+             _codecArguments = arguments.CodecArguments;
 
             _codecArguments = arguments.CodecArguments;
             if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                 _codec = new Multiline(_codecArguments);
 
-            _logFileMaxRecords = new Dictionary<string, Int64>();
-            _logFileCreationTimes = new Dictionary<string, DateTime>();
-            _logFileSampleTimes = new Dictionary<string, DateTime>();
-            _logFileSizes = new Dictionary<string, long>();
-
             _receivedMessages = 0;
             _arguments = arguments;
             _pollingIntervalInSeconds = arguments.Interval;
 
+            IsWildcardFilePattern = arguments.Location.Contains('*');
+
             foreach (string srcFile in _arguments.Location.Split(','))
             {
                 string file = srcFile.Trim();
-                Task.Factory.StartNew(() => FileWatcher(file));
+                string dir = Path.GetDirectoryName(file);
+                if (string.IsNullOrEmpty(dir))
+                    dir = Environment.CurrentDirectory;
+                string fileSpec = Path.Combine(dir, file);
+
+                Task.Factory.StartNew(() => FileWatcher(fileSpec));
             }
         }
 
@@ -73,9 +78,9 @@ namespace TimberWinR.Inputs
             base.Shutdown();
         }
 
+      
         public override JObject ToJson()
         {
-
             JObject json = new JObject(
                 new JProperty("log",
                     new JObject(
@@ -86,21 +91,11 @@ namespace TimberWinR.Inputs
                         new JProperty("codepage", _arguments.CodePage),
                         new JProperty("splitLongLines", _arguments.SplitLongLines),
                         new JProperty("recurse", _arguments.Recurse),
-
+                        new JProperty("filedb",
+                            new JArray(from f in Files.ToList()
+                                       select JObject.FromObject(LogsFileDatabase.LookupLogFile(f)))),
                         new JProperty("files",
-                            new JArray(from f in _logFileMaxRecords.Keys
-                                       select new JValue(f))),
-                        new JProperty("fileSampleTimes",
-                            new JArray(from f in _logFileSampleTimes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileSizes",
-                            new JArray(from f in _logFileSizes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileIndices",
-                            new JArray(from f in _logFileMaxRecords.Values
-                                       select new JValue(f))),
-                        new JProperty("fileCreationDates",
-                            new JArray(from f in _logFileCreationTimes.Values
+                            new JArray(from f in Files.ToList()
                                        select new JValue(f)))
                         )));
 
@@ -120,7 +115,7 @@ namespace TimberWinR.Inputs
 
 
             return json;
-        }   
+        }
 
         private void FileWatcher(string fileToWatch)
         {
@@ -149,46 +144,34 @@ namespace TimberWinR.Inputs
                             {
                                 var record = rsfiles.getRecord();
                                 string logName = record.getValue("LogFilename") as string;
-                                FileInfo fi = new FileInfo(logName);
+                                FileInfo fi = new FileInfo(logName);                             
 
-                                if (!fi.Exists)
-                                {
-                                    _logFileCreationTimes.Remove(logName);
-                                    _logFileMaxRecords.Remove(logName);
-                                    _logFileSizes.Remove(logName);
-                                }
-
-                                _logFileSampleTimes[logName] = DateTime.UtcNow;
+                                var dbe = LogsFileDatabase.LookupLogFile(logName);
+                    
+                                SaveVisitedFileName(dbe.FileName);
 
                                 DateTime creationTime = fi.CreationTimeUtc;
-                                bool logHasRolled = (_logFileCreationTimes.ContainsKey(logName) &&
-                                                     creationTime > _logFileCreationTimes[logName]) ||
-                                                    (_logFileSizes.ContainsKey(logName) &&
-                                                     fi.Length < _logFileSizes[logName]);
+                                bool logHasRolled = dbe.NewFile || (creationTime != dbe.CreationTimeUtc || fi.Length < dbe.LastPosition);
 
-
-                                if (!_logFileMaxRecords.ContainsKey(logName) || logHasRolled)
+                                if (logHasRolled)
                                 {
-                                    _logFileCreationTimes[logName] = creationTime;
-                                    _logFileSizes[logName] = fi.Length;
-                                    var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
-                                    var rcount = oLogQuery.Execute(qcount, iFmt);
-                                    var qr = rcount.getRecord();
-                                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
-                                    if (logHasRolled)
-                                    {
-                                        LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
-                                        lrn = 0;
-                                    }
-                                    _logFileMaxRecords[logName] = lrn;
+                                    LogManager.GetCurrentClassLogger().Info("Log {0} has rolled", logName);
+                                    LogsFileDatabase.Roll(dbe);
                                 }
 
-                                _logFileSizes[logName] = fi.Length;
+                                // Log has rolled or this is a new file, or we haven't processed yet.
+                                bool processWholeFile = logHasRolled || !dbe.ProcessedFile;
+
+                                if (processWholeFile)
+                                    LogsFileDatabase.Update(dbe, true, 0);
+
                             }
                             rsfiles.close();
-                            foreach (string fileName in _logFileMaxRecords.Keys.ToList())
+                            foreach (string fileName in Files.ToList())
                             {
-                                var lastRecordNumber = _logFileMaxRecords[fileName];
+                                var dbe = LogsFileDatabase.LookupLogFile(fileName);
+
+                                var lastRecordNumber = dbe.LastPosition;
                                 var query = string.Format("SELECT * FROM {0} where Index > {1}", fileName,
                                     lastRecordNumber);
 
@@ -231,21 +214,22 @@ namespace TimberWinR.Inputs
                                     string msg = json["Text"].ToString();
                                     if (!string.IsNullOrEmpty(msg))
                                     {
-                                        if (_codecArguments != null &&
-                                            _codecArguments.Type == CodecArguments.CodecType.multiline)
+                                        if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                                         {
                                             _codec.Apply(msg, this);
                                             _receivedMessages++;
+                                            dbe.IncrementLineCount();
                                         }
                                         else
                                         {
                                             ProcessJson(json);
-                                            _receivedMessages++;
+                                            dbe.IncrementLineCount();
+                                             _receivedMessages++;
                                         }
                                     }
 
                                     var lrn = (Int64)record.getValueEx("Index");
-                                    _logFileMaxRecords[fileName] = lrn;
+                                    LogsFileDatabase.Update(dbe, true, lrn);
                                     GC.Collect();
                                 }
 
@@ -254,16 +238,18 @@ namespace TimberWinR.Inputs
                                 rs.close();
                                 rs = null;
                                 GC.Collect();
+
                             }
                         }
                         catch (FileNotFoundException fnfex)
                         {
                             string fn = fnfex.FileName;
 
-                            if (!_fnfmap.ContainsKey(fn))
+                            if (!string.IsNullOrEmpty(fn) && !_fnfmap.ContainsKey(fn))
+                            {
                                 LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
-
-                            _fnfmap[fn] = fn;
+                                _fnfmap[fn] = fn;
+                            }
                         }
                         catch (OperationCanceledException)
                         {
@@ -283,7 +269,7 @@ namespace TimberWinR.Inputs
                                     syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                             }
                             catch (OperationCanceledException)
-                            {                               
+                            {
                             }
                             catch (Exception ex1)
                             {

TimberWinR/Inputs/StdinListener.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.Linq;
+using System.Runtime.InteropServices;
 using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
@@ -15,10 +16,15 @@ namespace TimberWinR.Inputs
 {
     public class StdinListener : InputListener
     {
+        [DllImport("User32.Dll", EntryPoint = "PostMessageA")]
+        private static extern bool PostMessage(IntPtr hWnd, uint msg, int wParam, int lParam);
+  
         private Thread _listenThread;
         private CodecArguments _codecArguments;
-        private ICodec _codec;     
-       
+        private ICodec _codec;
+        const int VK_RETURN = 0x0D;
+        const int WM_KEYDOWN = 0x100;
+
         public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken)
             : base(cancelToken, "Win32-Console")
         {
@@ -54,7 +60,14 @@ namespace TimberWinR.Inputs
 
         public override void Shutdown()
         {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);    
+            // This must come from another thread.
+            ThreadPool.QueueUserWorkItem((o) =>
+            {
+                Thread.Sleep(100);
+                var hWnd = System.Diagnostics.Process.GetCurrentProcess().MainWindowHandle;
+                PostMessage(hWnd, WM_KEYDOWN, VK_RETURN, 0);
+            });
             base.Shutdown();
         }
 

TimberWinR/Inputs/TailFileListener.cs

@@ -26,33 +26,27 @@ namespace TimberWinR.Inputs
     /// </summary>
     public class TailFileListener : InputListener
     {
+        private object _locker = new object();
         private int _pollingIntervalInSeconds;
-        private TimberWinR.Parser.TailFile _arguments;
+        private TimberWinR.Parser.TailFileArguments _arguments;
         private long _receivedMessages;
-        private Dictionary<string, Int64> _logFileMaxRecords;
-        private Dictionary<string, DateTime> _logFileCreationTimes;
-        private Dictionary<string, DateTime> _logFileSampleTimes;
-        private Dictionary<string, long> _logFileSizes;       
-        private CodecArguments _codecArguments;
-        private ICodec _codec;     
 
+        private CodecArguments _codecArguments;
+        private ICodec _codec;
+       
         public bool Stop { get; set; }
 
-        public TailFileListener(TimberWinR.Parser.TailFile arguments, CancellationToken cancelToken)
+        public TailFileListener(TimberWinR.Parser.TailFileArguments arguments, CancellationToken cancelToken)
             : base(cancelToken, "Win32-TailLog")
         {
             Stop = false;
+         
+            EnsureRollingCaught();    
 
             _codecArguments = arguments.CodecArguments;
             if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                 _codec = new Multiline(_codecArguments);
 
-
-            _logFileMaxRecords = new Dictionary<string, Int64>();
-            _logFileCreationTimes = new Dictionary<string, DateTime>();
-            _logFileSampleTimes = new Dictionary<string, DateTime>();
-            _logFileSizes = new Dictionary<string, long>();
-
             _receivedMessages = 0;
             _arguments = arguments;
             _pollingIntervalInSeconds = arguments.Interval;
@@ -66,7 +60,7 @@ namespace TimberWinR.Inputs
 
         public override void Shutdown()
         {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1} for {2}", Thread.CurrentThread.ManagedThreadId, InputType, _arguments.Location);
             Stop = true;
             base.Shutdown();
         }
@@ -74,29 +68,19 @@ namespace TimberWinR.Inputs
         public override JObject ToJson()
         {
             JObject json = new JObject(
-                new JProperty("log",
+                new JProperty("taillog",
                     new JObject(
                         new JProperty("messages", _receivedMessages),
                         new JProperty("type", InputType),
                         new JProperty("location", _arguments.Location),
                         new JProperty("logSource", _arguments.LogSource),
                         new JProperty("recurse", _arguments.Recurse),
-
                         new JProperty("files",
-                            new JArray(from f in _logFileMaxRecords.Keys
+                            new JArray(from f in Files
                                        select new JValue(f))),
-                        new JProperty("fileSampleTimes",
-                            new JArray(from f in _logFileSampleTimes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileSizes",
-                            new JArray(from f in _logFileSizes.Values
-                                       select new JValue(f))),
-                        new JProperty("fileIndices",
-                            new JArray(from f in _logFileMaxRecords.Values
-                                       select new JValue(f))),
-                        new JProperty("fileCreationDates",
-                            new JArray(from f in _logFileCreationTimes.Values
-                                       select new JValue(f)))
+                        new JProperty("filedb",
+                            new JArray(from f in Files
+                                       select JObject.FromObject(LogsFileDatabase.LookupLogFile(f))))
                         )));
 
 
@@ -117,10 +101,9 @@ namespace TimberWinR.Inputs
             return json;
         }
 
-        private void TailFileContents(string fileName, long offset)
+        private void TailFileContents(string fileName, long offset, LogsFileDatabaseEntry dbe)
         {
-            using (StreamReader reader = new StreamReader(new FileStream(fileName,
-                     FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
+            using (StreamReader reader = new StreamReader(new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
             {
                 //start at the end of the file
                 long lastMaxOffset = offset;
@@ -130,6 +113,8 @@ namespace TimberWinR.Inputs
                     return;
 
                 //seek to the last max offset
+                LogManager.GetCurrentClassLogger().Trace("{0}: File: {1} Seek to: {2}", Thread.CurrentThread.ManagedThreadId, fileName, lastMaxOffset);
+
                 reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin);
 
                 //read out of the file until the EOF
@@ -152,24 +137,31 @@ namespace TimberWinR.Inputs
                         else
                             json.Add(new JProperty("logSource", _arguments.LogSource));
                     }
+
                     json["Text"] = line;
                     json["Index"] = index;
                     json["LogFileName"] = fileName;
 
+
                     if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                     {
                         _codec.Apply(line, this);
                         Interlocked.Increment(ref _receivedMessages);
+                        dbe.IncrementLineCount();
                     }
                     else
                     {
                         ProcessJson(json);
                         Interlocked.Increment(ref _receivedMessages);
-                    }
-                    lineOffset += line.Length;                   
+                        dbe.IncrementLineCount();
+                        //LogManager.GetCurrentClassLogger().Info("{0}: File: {1} {2} {3}", Thread.CurrentThread.ManagedThreadId, fileName, dbe.LinesProcessed, line);
+                    }                 
+
+                    lineOffset += line.Length;
                 }
                 //update the last max offset
                 lastMaxOffset = reader.BaseStream.Position;
+                LogsFileDatabase.Update(dbe, true, lastMaxOffset);               
             }
         }
         // One thread for each kind of file to watch, i.e. "*.log,*.txt" would be two separate
@@ -187,11 +179,14 @@ namespace TimberWinR.Inputs
                     {
                         if (!CancelToken.IsCancellationRequested)
                         {
+                            var isWildcardPattern = fileToWatch.Contains('*');
                             string path = Path.GetDirectoryName(fileToWatch);
                             string name = Path.GetFileName(fileToWatch);
                             if (string.IsNullOrEmpty(path))
                                 path = ".";
 
+                            LogManager.GetCurrentClassLogger().Trace(":{0} Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, Path.Combine(path, name));
+
                             // Ok, we have a potential file filter here as 'fileToWatch' could be foo.log or *.log
 
                             SearchOption so = SearchOption.TopDirectoryOnly;
@@ -201,26 +196,41 @@ namespace TimberWinR.Inputs
                             foreach (string fileName in Directory.GetFiles(path, name, so))
                             {
                                 var dbe = LogsFileDatabase.LookupLogFile(fileName);
-                                FileInfo fi = new FileInfo(dbe.FileName);
-                                //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);
-                                long length = fi.Length;
-                                bool logHasRolled = false;
-                                if (fi.Length < dbe.Size || fi.CreationTimeUtc != dbe.CreationTimeUtc)
+
+                                // We only spin up 1 thread for a file we haven't yet seen.                               
+                                if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile)
                                 {
-                                    LogManager.GetCurrentClassLogger().Info("Log has Rolled: {0}", dbe.FileName);
-                                    logHasRolled = true;
+                                    LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                    LogsFileDatabase.Update(dbe, false, dbe.LastPosition);
+                                    SaveVisitedFileName(fileName);
+                                    Task.Factory.StartNew(() => TailFileWatcher(fileName));
                                 }
-                                bool processWholeFile = logHasRolled || dbe.NewFile;
-                                if (processWholeFile)
+                                else if (!isWildcardPattern)
                                 {
-                                    LogManager.GetCurrentClassLogger().Info("Process Whole File: {0}", dbe.FileName);
-                                    TailFileContents(dbe.FileName, 0);
+                                    FileInfo fi = new FileInfo(dbe.FileName);
+
+                                    //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);                                
+                                    long length = fi.Length;
+                                    bool logHasRolled = false;
+                                    if (fi.Length < dbe.LastPosition || fi.CreationTimeUtc != dbe.CreationTimeUtc)
+                                    {
+                                        LogManager.GetCurrentClassLogger().Info("{0}: Log has Rolled: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                        logHasRolled = true;
+                                        LogsFileDatabase.Roll(dbe);
+                                    }
+                                    // Log has rolled or this is a file we are seeing for the first time.
+                                    bool processWholeFile = logHasRolled || !dbe.ProcessedFile;
+                                    if (processWholeFile)
+                                    {
+                                        LogsFileDatabase.Update(dbe, true, 0);
+                                        LogManager.GetCurrentClassLogger().Debug("{0}: Process Whole File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
+                                        TailFileContents(dbe.FileName, 0, dbe);
+                                    }
+                                    else
+                                    {
+                                        TailFileContents(dbe.FileName, dbe.LastPosition, dbe);
+                                    }                                  
                                 }
-                                else
-                                {
-                                    TailFileContents(dbe.FileName, dbe.Size);
-                                }
-                                LogsFileDatabase.Update(dbe);
                             }
                         }
                     }
@@ -232,6 +242,10 @@ namespace TimberWinR.Inputs
                             LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
                         _fnfmap[fn] = fn;
                     }
+                    catch (IOException ioex)
+                    {
+                        LogManager.GetCurrentClassLogger().Debug("Log has rolled: {0}", ioex.Message);
+                    }
                     catch (OperationCanceledException)
                     {
                         break;
@@ -249,16 +263,17 @@ namespace TimberWinR.Inputs
                         }
                         catch (OperationCanceledException)
                         {
+                            Stop = true;
                         }
                         catch (Exception ex1)
                         {
                             LogManager.GetCurrentClassLogger().Warn(ex1);
-                        }                      
+                        }
                     }
                 }
             }
             Finished();
-        }
+        }       
     }
 }
 

TimberWinR/Inputs/TcpInputListener.cs

@@ -16,7 +16,9 @@ namespace TimberWinR.Inputs
         private Thread _listenThreadV4;
         private Thread _listenThreadV6;
         private readonly int _port;
+       
         private long _receivedMessages;
+        private long _errorCount;
 
         public override JObject ToJson()
         {
@@ -24,9 +26,9 @@ namespace TimberWinR.Inputs
                 new JProperty("tcp",
                     new JObject(
                         new JProperty("port", _port),
+                        new JProperty("errors", _errorCount),
                         new JProperty("messages", _receivedMessages)
                         )));
-
             return json;
         }
 
@@ -52,7 +54,7 @@ namespace TimberWinR.Inputs
 
         public override void Shutdown()
         {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            LogManager.GetCurrentClassLogger().Info("{0}: Shutting Down {1}", Thread.CurrentThread.ManagedThreadId, InputType);
 
             this._tcpListenerV4.Stop();
             this._tcpListenerV6.Stop();
@@ -68,7 +70,6 @@ namespace TimberWinR.Inputs
 
             listener.Start();
 
-
             while (!CancelToken.IsCancellationRequested)
             {
                 try
@@ -79,7 +80,7 @@ namespace TimberWinR.Inputs
                     // Wait for a client, spin up a thread.
                     var clientThread = new Thread(new ParameterizedThreadStart(HandleNewClient));
                     clientThread.Start(client);
-                }
+                }                
                 catch (SocketException ex)
                 {
                     if (ex.SocketErrorCode == SocketError.Interrupted)
@@ -109,17 +110,25 @@ namespace TimberWinR.Inputs
                             {
                                 JObject json = JObject.Load(reader);
                                 ProcessJson(json);
-                                _receivedMessages++;
+                                Interlocked.Increment(ref _receivedMessages);
                             }
                             catch (Exception ex)
                             {
+                                var jex1 = LogErrors.LogException("Bad Json", ex);
+                                if (jex1 != null)
+                                    ProcessJson(jex1);
+
                                 LogManager.GetCurrentClassLogger().Warn(ex);
+                                Interlocked.Increment(ref _errorCount);
                             }
 
                         }
                     }
                 }
             }
+            catch(OperationCanceledException)
+            {
+            }
             catch (Exception ex)
             {
                 LogManager.GetCurrentClassLogger().Error(ex);

TimberWinR/Inputs/UdpInputListener.cs

@@ -1,6 +1,7 @@
 using System;
 using System.IO;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Net;
 using System.Net.Sockets;
@@ -11,24 +12,14 @@ using NLog;
 namespace TimberWinR.Inputs
 {
     public class UdpInputListener : InputListener
-    {
-        private readonly System.Net.Sockets.UdpClient _udpListener;
-        private readonly IPEndPoint groupV4;
-        private readonly IPEndPoint groupV6;
-
-        private Thread _listenThreadV4;
-        private Thread _listenThreadV6;
+    {      
+        private UdpClient _udpListenerV4;      
+        private readonly Thread _listenThreadV4;
 
         private readonly int _port;
         private long _receivedMessages;
         private long _parsedErrors;
-
-        private struct listenProfile
-        {
-            public IPEndPoint endPoint;
-            public UdpClient client;
-        }
-
+       
         public override JObject ToJson()
         {
             JObject json = new JObject(
@@ -45,37 +36,35 @@ namespace TimberWinR.Inputs
         public UdpInputListener(CancellationToken cancelToken, int port = 5140)
             : base(cancelToken, "Win32-Udp")
         {
-            _port = port;
-
-            groupV4 = new IPEndPoint(IPAddress.Any, 0);
-            groupV6 = new IPEndPoint(IPAddress.IPv6Any, 0);
-
-            LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", _port);
-
+            _port = port;     
             _receivedMessages = 0;
 
-            _udpListener = new System.Net.Sockets.UdpClient(port);
-
-            _listenThreadV4 = new Thread(new ParameterizedThreadStart(StartListener));
-            _listenThreadV4.Start(new listenProfile() { endPoint = groupV4, client = _udpListener });
-
-            _listenThreadV6 = new Thread(new ParameterizedThreadStart(StartListener));
-            _listenThreadV6.Start(new listenProfile() { endPoint = groupV6, client = _udpListener });
+            _listenThreadV4 = new Thread(StartListener);
+            _listenThreadV4.Start();
         }
 
 
         public override void Shutdown()
         {
             LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
-            _udpListener.Close();
-            Finished();
+
+            // close UDP listeners, which will end the listener threads          
+            _udpListenerV4.Close();
+
+            // wait for completion of the threads         
+            _listenThreadV4.Join();          
+
             base.Shutdown();
         }
 
+        private void StartListener()
+        {                      
+            var groupV4 = new IPEndPoint(IPAddress.Any, _port);
+
+            _udpListenerV4 = new UdpClient(_port);            
+    
+            LogManager.GetCurrentClassLogger().Info("Udp Input on Port {0} Ready", groupV4);
 
-        private void StartListener(object useProfile)
-        {
-            var profile = (listenProfile)useProfile;
             string lastMessage = "";
             try
             {
@@ -83,21 +72,35 @@ namespace TimberWinR.Inputs
                 {
                     try
                     {
-                        byte[] bytes = profile.client.Receive(ref profile.endPoint);  
-                        var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
+                        byte[] bytes = _udpListenerV4.Receive(ref groupV4);  
+                        var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);                                                
                         lastMessage = data;
-                        JObject json = JObject.Parse(data);
+                        var json = JObject.Parse(data);
                         ProcessJson(json);
-                        _receivedMessages++;
+                        Interlocked.Increment(ref _receivedMessages);                       
                     }
-                    catch (Exception ex1)
+                    catch(ArgumentException aex)
                     {
+                        LogManager.GetCurrentClassLogger().Error(aex);
+                        break;
+                    }
+                    catch(SocketException)
+                    {                       
+                        break;
+                    }
+                    catch (Exception ex)
+                    {                      
+                        var jex1 = LogErrors.LogException(string.Format("Invalid JSON: {0}", lastMessage), ex);
+                        if (jex1 != null)
+                            ProcessJson(jex1);
+      
                         LogManager.GetCurrentClassLogger().Warn("Bad JSON: {0}", lastMessage);
-                        LogManager.GetCurrentClassLogger().Warn(ex1);
-                        _parsedErrors++;
+                        LogManager.GetCurrentClassLogger().Warn(ex);
+
+                        Interlocked.Increment(ref _parsedErrors);
                     }
                 }
-                _udpListener.Close();
+                _udpListenerV4.Close();
             }
             catch (Exception ex)
             {

TimberWinR/LogErrors.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+using TimberWinR.Parser;
+
+namespace TimberWinR
+{
+    public class LogErrors
+    {
+        public static JObject LogException(Exception ex)
+        {
+            return LogException("Exception", ex);
+        }
+
+        public static JObject LogException(string errorMessage, Exception ex)
+        {
+            JObject result = new JObject();
+            result["type"] = "TimberWinR-Error";
+            result["ErrorMessage"] = errorMessage;
+            
+            try
+            {  
+                JsonConvert.DefaultSettings = () => new JsonSerializerSettings
+                {
+                    Formatting = Formatting.Indented,
+                    NullValueHandling = NullValueHandling.Ignore
+                };
+
+                var exJson = JObject.Parse(JsonConvert.SerializeObject(ex));
+
+                result.Merge(exJson, new JsonMergeSettings
+                {
+                    MergeArrayHandling = MergeArrayHandling.Replace
+                });
+                return result;
+            }
+            catch (Exception ex1)
+            {
+                result["ErrorMessage"] = ex1.ToString();
+                return result;
+            }           
+        }
+    }
+}

TimberWinR/Manager.cs

@@ -28,6 +28,8 @@ namespace TimberWinR
         public List<InputListener> Listeners { get; set; }
         public bool LiveMonitor { get; set; }
 
+        public event Action<Configuration> OnConfigurationProcessed;
+
         public DateTime StartedOn { get; set; }
         public string JsonConfig { get; set; }
         public string LogfileDir { get; set; }
@@ -67,7 +69,7 @@ namespace TimberWinR
             LogsFileDatabase.Manager = this;                   
         }
 
-        public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken)
+        public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken, bool processConfiguration = true)
         {
             LogsFileDatabase.Manager = this;           
   
@@ -106,12 +108,14 @@ namespace TimberWinR
 
             LogManager.GlobalThreshold = LogLevel.FromString(logLevel);
 
-            LogManager.GetCurrentClassLogger()
-                .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());
-
+            //LogManager.GetCurrentClassLogger()
+            //    .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());
 
             LogManager.GetCurrentClassLogger()
-                .Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
+                .Info("TimberWinR Version {0}", Assembly.GetEntryAssembly().GetName().Version.ToString());
+
+            LogManager.GetCurrentClassLogger()
+                .Info("Database Filename: {0}", LogsFileDatabase.Instance.DatabaseFileName);
 
             try
             {
@@ -146,7 +150,15 @@ namespace TimberWinR
             LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
             LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);
 
-            ProcessConfiguration(cancelToken, Config);
+            if (processConfiguration)
+            {
+                ProcessConfiguration(cancelToken, Config);               
+            }
+        }
+
+        public void Start(CancellationToken cancelToken)
+        {
+            ProcessConfiguration(cancelToken, Config);          
         }
 
         public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
@@ -154,6 +166,9 @@ namespace TimberWinR
 // Read the Configuration file
             if (config != null)
             {
+                if (OnConfigurationProcessed != null)
+                    OnConfigurationProcessed(config);
+  
                 if (config.RedisOutputs != null)
                 {
                     foreach (var ro in config.RedisOutputs)
@@ -256,7 +271,8 @@ namespace TimberWinR
                         new JProperty("TimberWinR",
                             new JObject(
                                 new JProperty("version",
-                                    GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+                                    Assembly.GetEntryAssembly().GetName().Version.ToString()),
+                                    //GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
                                 new JProperty("host", computerName),
                                 new JProperty("output", output.Name),
                                 new JProperty("initialized", DateTime.UtcNow)
@@ -264,7 +280,7 @@ namespace TimberWinR
                     json.Add(new JProperty("type", "Win32-TimberWinR"));
                     json.Add(new JProperty("host", computerName));
                     output.Startup(json);
-                }
+                }              
             }
         }
 

TimberWinR/Outputs/Elasticsearch.cs

@@ -268,7 +268,7 @@ namespace TimberWinR.Outputs
                 ApplyFilters(jsonMessage);
 
             var message = jsonMessage.ToString();
-            LogManager.GetCurrentClassLogger().Debug(message);
+            LogManager.GetCurrentClassLogger().Trace(message);
 
             lock (_locker)
             {

TimberWinR/Outputs/Redis.cs

@@ -6,7 +6,7 @@ using System.Linq.Expressions;
 using System.Net.Sockets;
 using System.Text;
 using System.Threading;
-using ctstone.Redis;
+using CSRedis;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
@@ -28,7 +28,7 @@ namespace TimberWinR.Outputs
         private const int QUEUE_SAMPLE_SIZE = 30; // 30 samples over 2.5 minutes (default)
         private object _locker = new object();
         private bool _warnedReachedMax;
-    
+
         private readonly int _maxBatchCount;
         private readonly int _batchCount;
         private int _totalSamples;
@@ -53,7 +53,7 @@ namespace TimberWinR.Outputs
             {
                 if (_totalSamples < QUEUE_SAMPLE_SIZE)
                     _totalSamples++;
-                
+
                 // Take a sample of the queue depth
                 if (_sampleCountIndex >= QUEUE_SAMPLE_SIZE)
                     _sampleCountIndex = 0;
@@ -69,7 +69,7 @@ namespace TimberWinR.Outputs
                 if (_totalSamples > 0)
                 {
                     var samples = _sampleQueueDepths.Take(_totalSamples);
-                    int avg = (int) samples.Average();
+                    int avg = (int)samples.Average();
                     return avg;
                 }
                 return 0;
@@ -81,7 +81,7 @@ namespace TimberWinR.Outputs
         {
             if (currentBatchCount < _maxBatchCount && currentBatchCount < queueSize && AverageQueueDepth() > currentBatchCount)
             {
-                currentBatchCount += Math.Max(_maxBatchCount/_batchCount, 1);
+                currentBatchCount += Math.Max(_maxBatchCount / _batchCount, 1);
                 if (currentBatchCount >= _maxBatchCount && !_warnedReachedMax)
                 {
                     LogManager.GetCurrentClassLogger().Warn("Maximum Batch Count of {0} reached.", currentBatchCount);
@@ -93,7 +93,7 @@ namespace TimberWinR.Outputs
             else // Reset to default
             {
                 currentBatchCount = _batchCount;
-                _warnedReachedMax = false;                
+                _warnedReachedMax = false;
             }
 
             return currentBatchCount;
@@ -117,21 +117,21 @@ namespace TimberWinR.Outputs
         private readonly int _port;
         private readonly int _timeout;
         private readonly object _locker = new object();
-        private readonly List<string> _jsonQueue;       
+        private readonly List<string> _jsonQueue;
         private readonly string[] _redisHosts;
         private int _redisHostIndex;
         private TimberWinR.Manager _manager;
         private readonly int _batchCount;
         private int _currentBatchCount;
-        private readonly int _maxBatchCount;       
+        private readonly int _maxBatchCount;
         private readonly int _interval;
-        private readonly int _numThreads;      
+        private readonly int _numThreads;
         private long _sentMessages;
         private long _errorCount;
         private long _redisDepth;
-        private DateTime? _lastErrorTimeUTC;       
+        private DateTime? _lastErrorTimeUTC;
         private readonly int _maxQueueSize;
-        private readonly bool _queueOverflowDiscardOldest;       
+        private readonly bool _queueOverflowDiscardOldest;
         private BatchCounter _batchCounter;
 
         public bool Stop { get; set; }
@@ -150,11 +150,13 @@ namespace TimberWinR.Outputs
             {
                 try
                 {
-                    RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port, _timeout);
+                    RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port);                   
                     return client;
                 }
-                catch (Exception)
+                catch (Exception ex)
                 {
+                    LogManager.GetCurrentClassLogger().Error(ex);
+
                 }
                 finally
                 {
@@ -186,9 +188,9 @@ namespace TimberWinR.Outputs
                         new JProperty("threads", _numThreads),
                         new JProperty("batchcount", _batchCount),
                         new JProperty("currentBatchCount", _currentBatchCount),
-                        new JProperty("reachedMaxBatchCountTimes",  _batchCounter.ReachedMaxBatchCountTimes),
+                        new JProperty("reachedMaxBatchCountTimes", _batchCounter.ReachedMaxBatchCountTimes),
                         new JProperty("maxBatchCount", _maxBatchCount),
-                        new JProperty("averageQueueDepth", _batchCounter.AverageQueueDepth()),   
+                        new JProperty("averageQueueDepth", _batchCounter.AverageQueueDepth()),
                         new JProperty("queueSamples", new JArray(_batchCounter.Samples())),
                         new JProperty("index", _logstashIndexName),
                         new JProperty("hosts",
@@ -201,17 +203,17 @@ namespace TimberWinR.Outputs
 
         public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutputParameters parameters, CancellationToken cancelToken)
             : base(cancelToken, "Redis")
-        {              
+        {
             _redisDepth = 0;
             _batchCount = parameters.BatchCount;
             _maxBatchCount = parameters.MaxBatchCount;
             // Make sure maxBatchCount is larger than batchCount
-            if (_maxBatchCount < _batchCount)
-                _maxBatchCount = _batchCount*10;
-           
+            if (_maxBatchCount <= _batchCount)
+                _maxBatchCount = _batchCount * 10;
+
             _manager = manager;
             _redisHostIndex = 0;
-            _redisHosts = parameters.Host;          
+            _redisHosts = parameters.Host;
             _jsonQueue = new List<string>();
             _port = parameters.Port;
             _timeout = parameters.Timeout;
@@ -224,7 +226,7 @@ namespace TimberWinR.Outputs
             _queueOverflowDiscardOldest = parameters.QueueOverflowDiscardOldest;
             _batchCounter = new BatchCounter(_batchCount, _maxBatchCount);
             _currentBatchCount = _batchCount;
-           
+
             for (int i = 0; i < parameters.NumThreads; i++)
             {
                 var redisThread = new Task(RedisSender, cancelToken);
@@ -250,7 +252,7 @@ namespace TimberWinR.Outputs
             }
 
             var message = jsonMessage.ToString();
-            LogManager.GetCurrentClassLogger().Debug(message);
+            LogManager.GetCurrentClassLogger().Trace(message);
 
             lock (_locker)
             {
@@ -285,13 +287,13 @@ namespace TimberWinR.Outputs
             foreach (var filter in _manager.Config.Filters)
             {
                 if (!filter.Apply(json))
-                {  
-                    LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
+                {
+                    LogManager.GetCurrentClassLogger().Debug("{0}: Dropping: {1}", Thread.CurrentThread.ManagedThreadId, json.ToString());
                     drop = true;
-                }              
+                }
             }
             return drop;
-        }      
+        }
         // 
         // Pull off messages from the Queue, batch them up and send them all across
         // 
@@ -313,11 +315,9 @@ namespace TimberWinR.Outputs
                                 _batchCounter.SampleQueueDepth(_jsonQueue.Count);
                                 // Re-compute current batch size
                                 _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count, _currentBatchCount);
-                                                                              
+
                                 messages = _jsonQueue.Take(_currentBatchCount).ToArray();
                                 _jsonQueue.RemoveRange(0, messages.Length);
-
-                              
                             }
 
                             if (messages.Length > 0)
@@ -335,12 +335,12 @@ namespace TimberWinR.Outputs
                                             {
                                                 client.StartPipe();
                                                 LogManager.GetCurrentClassLogger()
-                                                    .Debug("Sending {0} Messages to {1}", messages.Length, client.Host);
+                                                    .Debug("{0}: Sending {1} Messages to {2}", Thread.CurrentThread.ManagedThreadId, messages.Length, client.Host);
 
                                                 try
                                                 {
                                                     _redisDepth = client.RPush(_logstashIndexName, messages);
-                                                    _sentMessages += messages.Length;                                                  
+                                                    Interlocked.Add(ref _sentMessages, messages.Length);
                                                     client.EndPipe();
                                                     sentSuccessfully = true;
                                                     if (messages.Length > 0)
@@ -357,7 +357,7 @@ namespace TimberWinR.Outputs
                                                     LogManager.GetCurrentClassLogger().Error(ex);
                                                     Interlocked.Increment(ref _errorCount);
                                                     _lastErrorTimeUTC = DateTime.UtcNow;
-                                                }                                               
+                                                }
                                                 break;
                                             }
                                             else
@@ -374,20 +374,19 @@ namespace TimberWinR.Outputs
                                     {
                                         LogManager.GetCurrentClassLogger().Error(ex);
                                         Interlocked.Increment(ref _errorCount);
-                                        _lastErrorTimeUTC = DateTime.UtcNow;                                
+                                        _lastErrorTimeUTC = DateTime.UtcNow;
                                     }
                                 } // No more hosts to try.
 
- 
-                                if (!sentSuccessfully)                                
+                                // Couldn't send, put it back into the queue.
+                                if (!sentSuccessfully)
                                 {
                                     lock (_locker)
                                     {
                                         _jsonQueue.InsertRange(0, messages);
                                     }
                                 }
-                            }
-                           // GC.Collect();
+                            }                          
                             if (!Stop)
                                 syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
                         }
@@ -395,7 +394,7 @@ namespace TimberWinR.Outputs
                         {
                             break;
                         }
-                        catch(ThreadAbortException)
+                        catch (ThreadAbortException)
                         {
                             break;
                         }
@@ -403,11 +402,11 @@ namespace TimberWinR.Outputs
                         {
                             _lastErrorTimeUTC = DateTime.UtcNow;
                             Interlocked.Increment(ref _errorCount);
-                            LogManager.GetCurrentClassLogger().Error(ex);                         
+                            LogManager.GetCurrentClassLogger().Error(ex);
                         }
                     }
                 }
             }
-        }      
+        }
     }
 }

TimberWinR/Parser.cs

@@ -297,7 +297,7 @@ namespace TimberWinR.Parser
         }
     }
 
-    public class TailFile : IValidateSchema
+    public class TailFileArguments : IValidateSchema
     {
         [JsonProperty(PropertyName = "location")]
         public string Location { get; set; }       
@@ -312,7 +312,7 @@ namespace TimberWinR.Parser
         [JsonProperty(PropertyName = "codec")]
         public CodecArguments CodecArguments { get; set; }
 
-        public TailFile()
+        public TailFileArguments()
         {
             Fields = new List<Field>();
             Fields.Add(new Field("LogFilename", "string"));
@@ -605,7 +605,7 @@ namespace TimberWinR.Parser
             Index = "logstash";
             Host = new string[] { "localhost" };
             Timeout = 10000;
-            BatchCount = 10;
+            BatchCount = 200;
             MaxBatchCount = BatchCount*10;
             NumThreads = 1;
             Interval = 5000;
@@ -646,7 +646,7 @@ namespace TimberWinR.Parser
         public LogParameters[] Logs { get; set; }
 
         [JsonProperty("TailFiles")]
-        public TailFile[] TailFiles { get; set; }
+        public TailFileArguments[] TailFilesArguments { get; set; }
 
         [JsonProperty("Tcp")]
         public TcpParameters[] Tcps { get; set; }

TimberWinR/ReleaseNotes.md

@@ -2,7 +2,19 @@
 ==================================
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
 
-Version History
+Version / Date
+### 1.3.21.0 - 04/13/2015
+1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets
+   and hosts file.  If we want to add this back, I will add a Udpv6 input.
+
+### 1.3.20.0 - 04/03/2015
+1. A re-factoring of Logs and TailLogs to be more efficient and detect log rolling correctly,
+   this requires http://support.microsoft.com/en-us/kb/172190 which will be detected and
+   set by TimberWinR, however, requires a reboot.
+2. Fixed issue [#38](https://github.com/Cimpress-MCP/TimberWinR/issues/38) diagnostic output not showing drop flag for Grok filter.
+3. Created TimberWinR.TestGenerator for complete testing of TimberWinR
+4. Fixed ipv4/ipv6 thread-safe issue with UdpInputListener which might lead to corrupted input data.
+
 ### 1.3.19.1 - 03/03/2015
 1. Added new Redis parameter _max\_batch\_count_ which increases the _batch\_count_ dynamically over time 
    to handle input flooding.   Default is _batch\_count_ * 10 

TimberWinR/TimberWinR.csproj

@@ -12,6 +12,8 @@
     <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
     <FileAlignment>512</FileAlignment>
     <TargetFrameworkProfile />
+    <SolutionDir Condition="$(SolutionDir) == '' Or $(SolutionDir) == '*Undefined*'">..\</SolutionDir>
+    <RestorePackages>true</RestorePackages>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <DebugSymbols>true</DebugSymbols>
@@ -32,7 +34,7 @@
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="csredis">
-      <HintPath>..\packages\csredis.1.4.7.1\lib\net40\csredis.dll</HintPath>
+      <HintPath>..\packages\csredis.3.2.1\lib\net40\csredis.dll</HintPath>
     </Reference>
     <Reference Include="Elasticsearch.Net">
       <HintPath>..\packages\Elasticsearch.Net.1.3.1\lib\Elasticsearch.Net.dll</HintPath>
@@ -43,42 +45,40 @@
       <HintPath>lib\com-logparser\Interop.MSUtil.dll</HintPath>
     </Reference>
     <Reference Include="MaxMind.Db">
-      <HintPath>..\packages\MaxMind.Db.0.2.3.0\lib\net40\MaxMind.Db.dll</HintPath>
-      <Private>True</Private>
+      <HintPath>..\packages\MaxMind.Db.1.0.0.0\lib\net40\MaxMind.Db.dll</HintPath>
     </Reference>
     <Reference Include="MaxMind.GeoIP2">
-      <HintPath>..\packages\MaxMind.GeoIP2.0.4.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
-      <Private>True</Private>
+      <HintPath>..\packages\MaxMind.GeoIP2.2.1.0.0\lib\net40\MaxMind.GeoIP2.dll</HintPath>
     </Reference>
     <Reference Include="Nest">
       <HintPath>..\packages\NEST.1.3.1\lib\Nest.dll</HintPath>
     </Reference>
     <Reference Include="Newtonsoft.Json, Version=6.0.0.0, Culture=neutral, PublicKeyToken=30ad4fe6b2a6aeed, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Newtonsoft.Json.6.0.4\lib\net40\Newtonsoft.Json.dll</HintPath>
+      <HintPath>..\packages\Newtonsoft.Json.6.0.5\lib\net40\Newtonsoft.Json.dll</HintPath>
     </Reference>
     <Reference Include="NLog">
-      <HintPath>..\packages\NLog.3.1.0.0\lib\net40\NLog.dll</HintPath>
+      <HintPath>..\packages\NLog.3.2.0.0\lib\net40\NLog.dll</HintPath>
     </Reference>
     <Reference Include="RapidRegex.Core">
       <HintPath>..\packages\RapidRegex.Core.1.0.0.2\lib\net40\RapidRegex.Core.dll</HintPath>
     </Reference>
     <Reference Include="RestSharp">
-      <HintPath>..\packages\RestSharp.104.4.0\lib\net4\RestSharp.dll</HintPath>
+      <HintPath>..\packages\RestSharp.105.0.0\lib\net4\RestSharp.dll</HintPath>
     </Reference>
     <Reference Include="System" />
     <Reference Include="System.Core" />
     <Reference Include="System.Linq.Dynamic">
-      <HintPath>..\packages\System.Linq.Dynamic.1.0.3\lib\net40\System.Linq.Dynamic.dll</HintPath>
+      <HintPath>..\packages\System.Linq.Dynamic.1.0.4\lib\net40\System.Linq.Dynamic.dll</HintPath>
     </Reference>
     <Reference Include="System.Xml.Linq" />
     <Reference Include="System.Data.DataSetExtensions" />
     <Reference Include="Microsoft.CSharp" />
     <Reference Include="System.Data" />
     <Reference Include="System.Xml" />
-    <Reference Include="Topshelf, Version=3.1.122.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
+    <Reference Include="Topshelf, Version=3.1.135.0, Culture=neutral, PublicKeyToken=b800c4cfcdeea87b, processorArchitecture=MSIL">
       <SpecificVersion>False</SpecificVersion>
-      <HintPath>..\packages\Topshelf.3.1.3\lib\net40-full\Topshelf.dll</HintPath>
+      <HintPath>..\packages\Topshelf.3.1.4\lib\net40-full\Topshelf.dll</HintPath>
     </Reference>
   </ItemGroup>
   <ItemGroup>
@@ -107,6 +107,7 @@
     <Compile Include="Inputs\TcpInputListener.cs" />
     <Compile Include="Inputs\LogsListener.cs" />
     <Compile Include="Inputs\WindowsEvtInputListener.cs" />
+    <Compile Include="LogErrors.cs" />
     <Compile Include="Manager.cs" />
     <Compile Include="Outputs\Elasticsearch.cs" />
     <Compile Include="Outputs\OutputSender.cs" />
@@ -149,7 +150,9 @@
     <None Include="mdocs\Logs.md" />
     <None Include="mdocs\IISW3CInput.md" />
     <None Include="mdocs\WindowsEvents.md" />
-    <None Include="packages.config" />
+    <None Include="packages.config">
+      <SubType>Designer</SubType>
+    </None>
     <None Include="ReleaseNotes.md" />
   </ItemGroup>
   <ItemGroup>
@@ -160,6 +163,13 @@
   </ItemGroup>
   <ItemGroup />
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
+  <Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
+  <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild">
+    <PropertyGroup>
+      <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them.  For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText>
+    </PropertyGroup>
+    <Error Condition="!Exists('$(SolutionDir)\.nuget\NuGet.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\.nuget\NuGet.targets'))" />
+  </Target>
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
   <Target Name="BeforeBuild">

TimberWinR/mdocs/RedisOutput.md

@@ -8,7 +8,7 @@ The following parameters are allowed when configuring the Redis output.
 | Parameter     |   Type   |  Description                                                | Details               |  Default |
 | :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
 | *threads*     | string   | Location of log files(s) to monitor                         | Number of worker theads to send messages | 1 |
-| *batch_count* | integer  | Sent as a single message                         | Number of messages to aggregate | 10 |
+| *batch_count* | integer  | Sent as a single message                         | Number of messages to aggregate | 200 |
 | *max_batch_count* | integer  | Dynamically adjusted count maximum                      | Increases over time | batch_count*10 |
 | *interval*    | integer  | Interval in milliseconds to sleep during batch sends        | Interval      | 5000 |
 | *index*       | string   | The name of the redis list                                  | logstash index name | logstash |

TimberWinR/packages.config

@@ -1,13 +1,14 @@
 <?xml version="1.0" encoding="utf-8"?>
 <packages>
-  <package id="csredis" version="1.4.7.1" targetFramework="net40" />
+  <package id="csredis" version="3.2.1" targetFramework="net40" />
   <package id="Elasticsearch.Net" version="1.3.1" targetFramework="net40" />
-  <package id="MaxMind.Db" version="0.2.3.0" targetFramework="net40" />
-  <package id="MaxMind.GeoIP2" version="0.4.0.0" targetFramework="net40" />
+  <package id="MaxMind.Db" version="1.0.0.0" targetFramework="net40" />
+  <package id="MaxMind.GeoIP2" version="2.1.0.0" targetFramework="net40" />
   <package id="NEST" version="1.3.1" targetFramework="net40" />
-  <package id="Newtonsoft.Json" version="6.0.4" targetFramework="net40" />
-  <package id="NLog" version="3.1.0.0" targetFramework="net40" />
+  <package id="Newtonsoft.Json" version="6.0.5" targetFramework="net40" />
+  <package id="NLog" version="3.2.0.0" targetFramework="net40" />
   <package id="RapidRegex.Core" version="1.0.0.2" targetFramework="net40" />
-  <package id="RestSharp" version="104.4.0" targetFramework="net40" />
-  <package id="System.Linq.Dynamic" version="1.0.3" targetFramework="net40" />
+  <package id="RestSharp" version="105.0.0" targetFramework="net40" />
+  <package id="System.Linq.Dynamic" version="1.0.4" targetFramework="net40" />
+  <package id="Topshelf" version="3.1.4" targetFramework="net40" />
 </packages>

TimberWix/TimberWinR.Wix.wixproj

@@ -63,9 +63,6 @@
     </WixExtension>
   </ItemGroup>
   <Import Project="$(WixTargetsPath)" />
-  <PropertyGroup>
-    <PreBuildEvent />
-  </PropertyGroup>
   <Target Name="BeforeBuild">
     <!-- Read the version from the to-be-installed .exe -->
     <GetAssemblyIdentity AssemblyFiles="..\TimberWinR.ServiceHost\bin\$(Configuration)\TimberWinR.ServiceHost.exe">
@@ -103,6 +100,9 @@
   <PropertyGroup>
     <PostBuildEvent>$(SolutionDir)\TimberWinR.ExtractID\$(OutDir)\TimberWinR.ExtractID.exe $(TargetDir) $(SolutionDir)chocolateyUninstall.ps1.guid  $(SolutionDir)chocolateyUninstall.ps1.template</PostBuildEvent>
   </PropertyGroup>
+  <PropertyGroup>
+    <PreBuildEvent>cmd.exe /c copy $(SolutionDir)chocolateyUninstall.ps1.template.orig  $(SolutionDir)chocolateyUninstall.ps1.template</PreBuildEvent>
+  </PropertyGroup>
   <!--
 	To modify your build process, add your task inside one of the targets below and uncomment it.
 	Other similar extension points exist, see Wix.targets.

packages/MaxMind.DB.0.2.3.0/lib/net40/MaxMind.Db.xml

@@ -1,279 +0,0 @@
-<?xml version="1.0"?>
-<doc>
-    <assembly>
-        <name>MaxMind.Db</name>
-    </assembly>
-    <members>
-        <member name="T:MaxMind.Db.ObjectType">
-            <summary>
-            Enumeration representing the types of objects read from the database
-            </summary>
-        </member>
-        <member name="T:MaxMind.Db.Result">
-            <summary>
-            A data structure to store an object read from the database
-            </summary>
-        </member>
-        <member name="M:MaxMind.Db.Result.#ctor(Newtonsoft.Json.Linq.JToken,System.Int32)">
-            <summary>
-            Initializes a new instance of the <see cref="T:MaxMind.Db.Result"/> class.
-            </summary>
-            <param name="node">The node.</param>
-            <param name="offset">The offset.</param>
-        </member>
-        <member name="P:MaxMind.Db.Result.Node">
-            <summary>
-            The object read from the database
-            </summary>
-        </member>
-        <member name="P:MaxMind.Db.Result.Offset">
-            <summary>
-            The offset
-            </summary>
-        </member>
-        <member name="T:MaxMind.Db.Decoder">
-            <summary>
-            Given a stream, this class decodes the object graph at a particular location
-            </summary>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.#ctor(System.Threading.ThreadLocal{System.IO.Stream},System.Int32)">
-            <summary>
-            Initializes a new instance of the <see cref="T:MaxMind.Db.Decoder"/> class.
-            </summary>
-            <param name="stream">The stream.</param>
-            <param name="pointerBase">The base address in the stream.</param>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.Decode(System.Int32)">
-            <summary>
-            Decodes the object at the specified offset.
-            </summary>
-            <param name="offset">The offset.</param>
-            <returns>An object containing the data read from the stream</returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.ReadOne(System.Int32)">
-            <summary>
-            Reads the one.
-            </summary>
-            <param name="position">The position.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.ReadMany(System.Int32,System.Int32)">
-            <summary>
-            Reads the many.
-            </summary>
-            <param name="position">The position.</param>
-            <param name="size">The size.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeByType(MaxMind.Db.ObjectType,System.Int32,System.Int32)">
-            <summary>
-            Decodes the type of the by.
-            </summary>
-            <param name="type">The type.</param>
-            <param name="offset">The offset.</param>
-            <param name="size">The size.</param>
-            <returns></returns>
-            <exception cref="T:System.Exception">Unable to handle type!</exception>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.FromControlByte(System.Byte)">
-            <summary>
-            Froms the control byte.
-            </summary>
-            <param name="b">The attribute.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.SizeFromCtrlByte(System.Byte,System.Int32)">
-            <summary>
-            Sizes from control byte.
-            </summary>
-            <param name="ctrlByte">The control byte.</param>
-            <param name="offset">The offset.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeBoolean(System.Int32)">
-            <summary>
-            Decodes the boolean.
-            </summary>
-            <param name="size">The size of the structure.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeDouble(System.Byte[])">
-            <summary>
-            Decodes the double.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeFloat(System.Byte[])">
-            <summary>
-            Decodes the float.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeString(System.Byte[])">
-            <summary>
-            Decodes the string.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeMap(System.Int32,System.Int32)">
-            <summary>
-            Decodes the map.
-            </summary>
-            <param name="size">The size.</param>
-            <param name="offset">The offset.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeLong(System.Byte[])">
-            <summary>
-            Decodes the long.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeIntegerToJValue(System.Byte[])">
-            <summary>
-            Decodes the integer.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeArray(System.Int32,System.Int32)">
-            <summary>
-            Decodes the array.
-            </summary>
-            <param name="size">The size.</param>
-            <param name="offset">The offset.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeUInt64(System.Byte[])">
-            <summary>
-            Decodes the uint64.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeBigInteger(System.Byte[])">
-            <summary>
-            Decodes the big integer.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodePointer(System.Int32,System.Int32,System.Int32@)">
-            <summary>
-            Decodes the pointer.
-            </summary>
-            <param name="ctrlByte">The control byte.</param>
-            <param name="offset">The offset.</param>
-            <param name="outOffset">The resulting offset</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeInteger(System.Byte[])">
-            <summary>
-            Decodes the integer.
-            </summary>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="M:MaxMind.Db.Decoder.DecodeInteger(System.Int32,System.Byte[])">
-            <summary>
-            Decodes the integer.
-            </summary>
-            <param name="baseValue">The base value.</param>
-            <param name="buffer">The buffer.</param>
-            <returns></returns>
-        </member>
-        <member name="T:MaxMind.Db.InvalidDatabaseException">
-            <summary>
-            Thrown when the MaxMind database file is incorrectly formatted
-            </summary>
-        </member>
-        <member name="M:MaxMind.Db.InvalidDatabaseException.#ctor(System.String)">
-            <summary>
-            Initializes a new instance of the <see cref="T:MaxMind.Db.InvalidDatabaseException"/> class.
-            </summary>
-            <param name="message">A message that describes the error.</param>
-        </member>
-        <member name="M:MaxMind.Db.InvalidDatabaseException.#ctor(System.String,System.Exception)">
-            <summary>
-            Initializes a new instance of the <see cref="T:MaxMind.Db.InvalidDatabaseException"/> class.
-            </summary>
-            <param name="message">The error message that explains the reason for the exception.</param>
-            <param name="innerException">The exception that is the cause of the current exception. If the <paramref name="innerException"/> parameter is not a null reference, the current exception is raised in a catch block that handles the inner exception.</param>
-        </member>
-        <member name="T:MaxMind.Db.FileAccessMode">
-            <summary>
-            An enumeration specifying the API to use to read the database
-            </summary>
-        </member>
-        <member name="F:MaxMind.Db.FileAccessMode.MemoryMapped">
-            <summary>
-            Open the file in memory mapped mode. Does not load into real memory.
-            </summary>
-        </member>
-        <member name="F:MaxMind.Db.FileAccessMode.Memory">
-            <summary>
-            Load the file into memory.
-            </summary>
-        </member>
-        <member name="T:MaxMind.Db.Reader">
-            <summary>
-            Given a MaxMind DB file, this class will retrieve information about an IP address
-            </summary>
-        </member>
-        <member name="M:MaxMind.Db.Reader.#ctor(System.String)">
-            <summary>
-            Initializes a new instance of the <see cref="T:MaxMind.Db.Reader"/> class.
-            </summary>
-            <param name="file">The file.</param>
-        </member>
-        <member name="M:MaxMind.Db.Reader.#ctor(System.String,MaxMind.Db.FileAccessMode)">
-            <summary>
-            Initializes a new instance of the <see cref="T:MaxMind.Db.Reader"/> class.
-            </summary>
-            <param name="file">The MaxMind DB file.</param>
-            <param name="mode">The mode by which to access the DB file.</param>
-        </member>
-        <member name="M:MaxMind.Db.Reader.#ctor(System.IO.Stream)">
-            <summary>
-            Initialize with Stream
-            </summary>
-            <param name="stream"></param>
-        </member>
-        <member name="M:MaxMind.Db.Reader.Find(System.String)">
-            <summary>
-            Finds the data related to the specified address.
-            </summary>
-            <param name="ipAddress">The IP address.</param>
-            <returns>An object containing the IP related data</returns>
-        </member>
-        <member name="M:MaxMind.Db.Reader.Find(System.Net.IPAddress)">
-            <summary>
-            Finds the data related to the specified address.
-            </summary>
-            <param name="ipAddress">The IP address.</param>
-            <returns>An object containing the IP related data</returns>
-        </member>
-        <member name="M:MaxMind.Db.Reader.Dispose">
-            <summary>
-            Release resources back to the system.
-            </summary>
-        </member>
-        <member name="P:MaxMind.Db.Reader.Metadata">
-            <summary>
-            Gets the metadata.
-            </summary>
-            <value>
-            The metadata.
-            </value>
-        </member>
-        <member name="T:MaxMind.Db.Metadata">
-            <summary>
-            Data about the database file itself
-            </summary>
-        </member>
-    </members>
-</doc>

packages/NUnit.2.6.3/license.txt

@@ -1,15 +0,0 @@
-Copyright © 2002-2013 Charlie Poole
-Copyright © 2002-2004 James W. Newkirk, Michael C. Two, Alexei A. Vorontsov
-Copyright © 2000-2002 Philip A. Craig
-
-This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment (see the following) in the product documentation is required.
-
-Portions Copyright © 2002-2013 Charlie Poole or Copyright © 2002-2004 James W. Newkirk, Michael C. Two, Alexei A. Vorontsov or Copyright © 2000-2002 Philip A. Craig
-
-2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source distribution.

packages/NUnit.Runners.2.6.3/license.txt

@@ -1,15 +0,0 @@
-Copyright © 2002-2013 Charlie Poole
-Copyright © 2002-2004 James W. Newkirk, Michael C. Two, Alexei A. Vorontsov
-Copyright © 2000-2002 Philip A. Craig
-
-This software is provided 'as-is', without any express or implied warranty. In no event will the authors be held liable for any damages arising from the use of this software.
-
-Permission is granted to anyone to use this software for any purpose, including commercial applications, and to alter it and redistribute it freely, subject to the following restrictions:
-
-1. The origin of this software must not be misrepresented; you must not claim that you wrote the original software. If you use this software in a product, an acknowledgment (see the following) in the product documentation is required.
-
-Portions Copyright © 2002-2013 Charlie Poole or Copyright © 2002-2004 James W. Newkirk, Michael C. Two, Alexei A. Vorontsov or Copyright © 2000-2002 Philip A. Craig
-
-2. Altered source versions must be plainly marked as such, and must not be misrepresented as being the original software.
-
-3. This notice may not be removed or altered from any source distribution.

packages/NUnit.Runners.2.6.3/tools/agent.conf

@@ -1,4 +0,0 @@
-<AgentConfig>
-  <Port>8080</Port>
-  <PathToAssemblies>.</PathToAssemblies>
-</AgentConfig>

packages/NUnit.Runners.2.6.3/tools/agent.log.conf

@@ -1,18 +0,0 @@
-<log4net>
-	<!-- A1 is set to be a ConsoleAppender -->
-	<appender name="A1" type="log4net.Appender.ConsoleAppender">
-
-		<!-- A1 uses PatternLayout -->
-		<layout type="log4net.Layout.PatternLayout">
-			<!-- Print the date in ISO 8601 format -->
-			<conversionPattern value="%-5level %logger - %message%newline" />
-		</layout>
-	</appender>
-	
-	<!-- Set root logger level to DEBUG and its only appender to A1 -->
-	<root>
-		<level value="Info" />
-		<appender-ref ref="A1" />
-	</root>
-
-</log4net>

packages/NUnit.Runners.2.6.3/tools/launcher.log.conf

@@ -1,18 +0,0 @@
-<log4net>
-	<!-- A1 is set to be a ConsoleAppender -->
-	<appender name="A1" type="log4net.Appender.ConsoleAppender">
-
-		<!-- A1 uses PatternLayout -->
-		<layout type="log4net.Layout.PatternLayout">
-			<!-- Print the date in ISO 8601 format -->
-			<conversionPattern value="%-5level %logger - %message%newline" />
-		</layout>
-	</appender>
-	
-	<!-- Set root logger level to DEBUG and its only appender to A1 -->
-	<root>
-		<level value="Info" />
-		<appender-ref ref="A1" />
-	</root>
-
-</log4net>