Move redisHostIndex inc to a finally

It looks like this will just retry connections to the same host if new RedisClient throws.  Move _redisHostIndex++ to a finally.

Should we also be logging when these happen?

README.md

@@ -23,6 +23,9 @@ Please use the TimberWinR Google Group for discussion and support:
 
 https://groups.google.com/forum/#!forum/timberwinr
 
+Latest Build:
+
+![alt tag](https://ci.appveyor.com/api/projects/status/github/Cimpress-MCP/TimberWinR)
 
 ## Inputs
 The current supported Input format sources are:
@@ -33,6 +36,11 @@ The current supported Input format sources are:
  5. [Stdin](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) (Standard Input for Debugging)
  6. [W3C](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/W3CInput.md)(Internet Information Services W3C Advanced/Custom Format)
  7. [Udp](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/UdpInput.md) (listens for UDP on port for JSON messages)
+ 8. [TailFiles](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/TailFiles.md) (Tails log files efficiently *New*)
+
+## Codecs
+The current list of supported codecs are:
+ 1. [Multiline](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)
 
 ## Filters
 The current list of supported filters are:
@@ -62,7 +70,7 @@ A single Json filter using the single tag (this is only provided as a convienien
 ]
 ```
 
-Multiple Json filters must use the jsonFilters and array syntax
+Multiple Json filters must use the jsonFilters and array syntax, also mutateFilters, grokFilters, dateFilters, geoipFilters.
 ```json
 "Filters": [
     {
@@ -200,8 +208,8 @@ Options:
 -configFile:       Specifies the path to the JSON config file, or directory which contains .json file(s).
                    Default is -configFile:default.json
 -diagnosticPort:   Specifies the diagnostic port which can be used to get a health check of the service.
-                   Default Port is 5142, A value of 0 will disable it.  Open a browser
-                      http://localhost:5142
+                   Default Port is 5141, A value of 0 will disable it.  Open a browser
+                      http://localhost:5141
 ```
 #### -configFile
 This may be a single .json file or a directory containing .json file(s).  If it is a directory, all
@@ -213,7 +221,7 @@ If you really just want to try it out, grab the binary distribution, extract the
 into a directory, e.g.  C:\TimberWinR
 
 Grab the [JSON example file](https://raw.githubusercontent.com/efontana/TimberWinR/master/TimberWinR.ServiceHost/default.json) and place it into C:\TimberWinR\default.json. 
-Edit the default.json file and change the Redis instance to match yours, replace 'tstlexiceapp006.vistaprint.svc' with the IP or DNS name
+Edit the default.json file and change the Redis instance to match yours, replace 'tstlexiceapp006.mycompany.svc' with the IP or DNS name
 of the machine running redis.  Fire up the collector, enable the verbose debugging to see some Windows Events.
 
 ```

TimberWinR.ServiceHost/Program.cs

@@ -38,7 +38,8 @@ namespace TimberWinR.ServiceHost
                     serviceConfigurator.WhenStarted(myService => myService.Start());
                     serviceConfigurator.WhenStopped(myService => myService.Stop());
                 });
-               
+
+                hostConfigurator.AddCommandLineDefinition("liveMonitor", c => arguments.LiveMonitor = bool.Parse(c.ToString()));
                 hostConfigurator.AddCommandLineDefinition("configFile", c => arguments.ConfigFile = c);
                 hostConfigurator.AddCommandLineDefinition("logLevel", c => arguments.LogLevel = c);
                 hostConfigurator.AddCommandLineDefinition("logDir", c => arguments.LogfileDir = c);
@@ -60,6 +61,7 @@ namespace TimberWinR.ServiceHost
                         AddServiceParameter("-configFile", arguments.ConfigFile);
                         AddServiceParameter("-logLevel", arguments.LogLevel);
                         AddServiceParameter("-logDir", arguments.LogfileDir);
+                        AddServiceParameter("-liveMonitor", arguments.LiveMonitor);
                         if (arguments.DiagnosticPort > 0)
                             AddServiceParameter("-diagnosticPort", arguments.DiagnosticPort);
                     }
@@ -68,8 +70,7 @@ namespace TimberWinR.ServiceHost
         }
 
         private static void AddServiceParameter(string paramName, string value)
-        {
-         
+        {         
             string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
 
             if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0} ", paramName)))           
@@ -80,8 +81,7 @@ namespace TimberWinR.ServiceHost
         }
 
         private static void AddServiceParameter(string paramName, int value)
-        {
-    
+        {    
             string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
 
             if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
@@ -91,6 +91,16 @@ namespace TimberWinR.ServiceHost
             }
         }
 
+        private static void AddServiceParameter(string paramName, bool value)
+        {
+            string currentValue = Registry.GetValue(KeyPath, KeyName, "").ToString();
+
+            if (!string.IsNullOrEmpty(paramName) && !currentValue.Contains(string.Format("{0}:", paramName)))
+            {
+                currentValue += string.Format(" {0} \"{1}\"", paramName, value.ToString());
+                Registry.SetValue(KeyPath, KeyName, currentValue);
+            }
+        }
     }
 
     internal class Arguments
@@ -99,9 +109,10 @@ namespace TimberWinR.ServiceHost
         public string LogLevel { get; set; }
         public string LogfileDir { get; set; }
         public int DiagnosticPort { get; set; }
-
+        public bool LiveMonitor { get; set; }
         public Arguments()
         {
+            LiveMonitor = false;
             DiagnosticPort = 5141;
             ConfigFile = "default.json";
             LogLevel = "Info";
@@ -147,7 +158,7 @@ namespace TimberWinR.ServiceHost
         /// </summary>
         private void RunService()
         {
-            _manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _cancellationToken);
+            _manager = new TimberWinR.Manager(_args.ConfigFile, _args.LogLevel, _args.LogfileDir, _args.LiveMonitor, _cancellationToken);
             if (_args.DiagnosticPort > 0)
                 _diags = new Diagnostics.Diagnostics(_manager, _cancellationToken, _args.DiagnosticPort);
         }

TimberWinR.ServiceHost/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.19.0")]
-[assembly: AssemblyFileVersion("1.3.19.0")]
+[assembly: AssemblyVersion("1.3.20.0")]
+[assembly: AssemblyFileVersion("1.3.20.0")]

TimberWinR.ServiceHost/config.json

@@ -90,7 +90,7 @@
                     "interval": 5000, 
                     "batch_count":  500,              
                     "host": [
-                        "tstlexiceapp006.vistaprint.svc"
+                        "tstlexiceapp006.mycompany.svc"
                     ]
                 }
             ],            
@@ -99,7 +99,7 @@
                     "threads":  1,   
                     "interval": 5000,                          
                     "host": [
-                        "tstlexiceapp003.vistaprint.svc"
+                        "tstlexiceapp003.mycompany.svc"
                     ]
                 }
             ]                           

TimberWinR.ServiceHost/default.json

@@ -33,7 +33,7 @@
                     "_comment": "Change the host to your Redis instance",
                     "port": 6379,
                     "host": [
-                        "logaggregator.vistaprint.svc"
+                        "logaggregator.mycompany.svc"
                     ]
                 }
             ]

TimberWinR.UnitTests/Configuration.cs

@@ -85,7 +85,7 @@ namespace TimberWinR.UnitTests
                         [{  
                             ""host"": 
                             [
-                                ""logaggregator.vistaprint.svc""
+                                ""logaggregator.mycompany.svc""
                             ]                   
 		                }]
 	                }

TimberWinR.UnitTests/GrokFilterTests.cs

@@ -22,7 +22,7 @@ namespace TimberWinR.UnitTests
                 {"Index", 7},
                 {"Text", null},
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
             };
 
             string grokJson = @"{  
@@ -55,7 +55,7 @@ namespace TimberWinR.UnitTests
             Assert.IsTrue(grok.Apply(json));
 
             // Verify host field added
-            Assert.AreEqual(json["host"].ToString(), "dev.vistaprint.net");
+            Assert.AreEqual(json["host"].ToString(), "dev.mycompany.net");
 
             // Verify two tags added
             Assert.AreEqual(json["tags"][0].ToString(), "rn_7");
@@ -71,7 +71,7 @@ namespace TimberWinR.UnitTests
                 {"Index", 7},
                 {"Text", "crap"},
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
             };
 
             string grokJson = @"{  
@@ -123,7 +123,7 @@ namespace TimberWinR.UnitTests
                     }
                 },
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
             };
 
             JObject json2 = new JObject
@@ -140,7 +140,7 @@ namespace TimberWinR.UnitTests
                     }
                 },
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
             };
 
 
@@ -223,7 +223,7 @@ namespace TimberWinR.UnitTests
                     }
                 },
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
             };
 
             string grokJson1 = @"{  
@@ -311,7 +311,7 @@ namespace TimberWinR.UnitTests
                     }
                 },
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"}
+                {"ComputerName", "dev.mycompany.net"}
             };
 
             string grokJson = @"{  

TimberWinR.UnitTests/JsonFilterTests.cs

@@ -20,14 +20,14 @@ namespace TimberWinR.UnitTests
             JObject jsonInputLine1 = new JObject
             {               
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"},
+                {"ComputerName", "dev.mycompany.net"},
                 {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
             };
 
             JObject jsonInputLine2 = new JObject
             {               
                 {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.vistaprint.net"},
+                {"ComputerName", "dev.mycompany.net"},
                 {"Text", "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
             };
 

TimberWinR.UnitTests/TailFileTests.cs

@@ -0,0 +1,90 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TimberWinR.Inputs;
+using TimberWinR.Parser;
+using Newtonsoft.Json.Linq;
+using System.Diagnostics;
+
+namespace TimberWinR.UnitTests
+{
+    [TestFixture]
+    public class TailFileTests
+    {
+        [Test]
+        public void TestTailFile()
+        {
+            List<JObject> events = new List<JObject>();
+
+            if (File.Exists(".timberwinrdb"))
+                File.Delete(".timberwinrdb");
+
+            var mgr = new Manager();
+            mgr.LogfileDir = ".";
+
+            var tf = new TailFile();
+            var cancelTokenSource = new CancellationTokenSource();
+            tf.Location = "TestTailFile1.log";
+
+   
+            if (File.Exists(tf.Location))
+                File.Delete(tf.Location);
+
+            try
+            {
+                var listener = new TailFileListener(tf, cancelTokenSource.Token);
+
+                listener.OnMessageRecieved += o =>
+                {
+                    events.Add(o);
+                    if (events.Count >= 100)
+                        cancelTokenSource.Cancel();
+                };
+
+                GenerateLogFile(tf.Location);
+
+                bool createdFile = false;
+                while (!listener.Stop && !cancelTokenSource.IsCancellationRequested)
+                {
+                    Thread.Sleep(100);
+                    if (!createdFile)
+                    {
+                        GenerateLogFile(tf.Location);
+                        createdFile = true;
+                    }
+                }
+            }
+            catch (OperationCanceledException oex)
+            {
+                Console.WriteLine("Done!");
+            }
+            catch (Exception ex)
+            {
+                Console.WriteLine(ex.ToString());
+
+            }
+            finally
+            {
+                Assert.AreEqual(100, events.Count);
+            }
+        }
+
+        private static void GenerateLogFile(string fileName)
+        {
+            using (System.IO.StreamWriter file = new System.IO.StreamWriter(fileName))
+            {
+                for (int i = 0; i < 100; i++)
+                {
+                    file.WriteLine("Log Line Number {0}", i);
+                }
+            }
+        }
+    }
+}

TimberWinR.UnitTests/TimberWinR.UnitTests.csproj

@@ -67,6 +67,7 @@
     <Compile Include="MultilineTests.cs" />
     <Compile Include="Parser\ElasticsearchOutputTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
+    <Compile Include="TailFileTests.cs" />
     <Compile Include="TestBase.cs" />
   </ItemGroup>
   <ItemGroup>

TimberWinR.sln

@@ -21,6 +21,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		chocolateyInstall.ps1.template = chocolateyInstall.ps1.template
 		chocolateyUninstall.ps1.guid = chocolateyUninstall.ps1.guid
 		chocolateyUninstall.ps1.template = chocolateyUninstall.ps1.template
+		chocolateyUninstall.ps1.template.orig = chocolateyUninstall.ps1.template.orig
 		LICENSE.txt = LICENSE.txt
 		Performance1.psess = Performance1.psess
 		README.md = README.md

TimberWinR/Configuration.cs

@@ -4,6 +4,8 @@ using System.Data.Odbc;
 using System.Diagnostics;
 using System.Linq;
 using System.Reflection;
+using System.Threading;
+using System.Threading.Tasks;
 using System.Text;
 using System.Xml;
 using System.Xml.Linq;
@@ -16,6 +18,7 @@ using Newtonsoft.Json.Linq;
 using TimberWinR.Inputs;
 using TimberWinR.Filters;
 
+
 using NLog;
 using TimberWinR.Parser;
 using Topshelf.Configurators;
@@ -25,7 +28,12 @@ using WindowsEvent = TimberWinR.Parser.WindowsEvent;
 namespace TimberWinR
 {
     public class Configuration
-    {
+    {       
+        private CancellationToken _cancelToken;
+        private bool _stopService;
+        private FileSystemWatcher _dirWatcher;
+        private Manager _manager;
+ 
         private List<WindowsEvent> _events = new List<WindowsEvent>();
         public IEnumerable<WindowsEvent> Events
         {
@@ -67,7 +75,13 @@ namespace TimberWinR
         public IEnumerable<Log> Logs
         {
             get { return _logs; }
-        }       
+        }
+
+        private List<TailFile> _tails = new List<TailFile>();
+        public IEnumerable<TailFile> TailFiles
+        {
+            get { return _tails; }
+        }    
 
         private List<IISW3CLog> _iisw3clogs = new List<IISW3CLog>();
 
@@ -96,11 +110,88 @@ namespace TimberWinR
             get { return _filters; }
         }
 
+        private void MonitorDirectory(string directoryToWatch, CancellationToken cancelToken, Manager manager)
+        {
+            _manager = manager;
+            _cancelToken = cancelToken;
+            if (_dirWatcher == null)
+            {
+                _dirWatcher = new FileSystemWatcher();
+                _dirWatcher.Path = directoryToWatch;
+                _dirWatcher.NotifyFilter = NotifyFilters.LastAccess | NotifyFilters.LastWrite | NotifyFilters.FileName | NotifyFilters.DirectoryName;
+                // Only watch json files.
+                _dirWatcher.Filter = "*.json";
+                _dirWatcher.Created += DirWatcherOnCreated;
+                _dirWatcher.Changed += DirWatcherOnChanged;
+                _dirWatcher.Renamed += DirWatcherOnRenamed;
+                _dirWatcher.EnableRaisingEvents = true;
+            }       
+        }
 
-        public static Configuration FromDirectory(string jsonDirectory)
+        private void DirWatcherOnRenamed(object sender, RenamedEventArgs e)
+        {
+            // The Renamed file could be a different name from .json
+            FileInfo fi = new FileInfo(e.FullPath);
+            if (fi.Extension == ".json")
+            {
+                LogManager.GetCurrentClassLogger().Info("File: OnRenamed " + e.FullPath + " " + e.ChangeType);
+                ProcessNewJson(e.FullPath);
+            }
+        }
+
+        private void DirWatcherOnCreated(object sender, FileSystemEventArgs e)
+        {
+            FileInfo fi = new FileInfo(e.FullPath);
+            if (fi.Extension == ".json")
+            {
+                LogManager.GetCurrentClassLogger().Info("File: OnCreated " + e.FullPath + " " + e.ChangeType);
+                ProcessNewJson(e.FullPath);
+            }
+        }
+
+        private void DirWatcherOnChanged(object sender, FileSystemEventArgs e)
+        {
+            FileInfo fi = new FileInfo(e.FullPath);
+            if (fi.Extension == ".json")
+            {
+                // Specify what is done when a file is changed, created, or deleted.
+                LogManager.GetCurrentClassLogger()
+                    .Info("File: OnChanged " + e.ChangeType.ToString() + " " + e.FullPath + " " + e.ChangeType);
+                ProcessNewJson(e.FullPath);
+            }
+        }
+      
+        private void ProcessNewJson(string fileName)
+        {
+            try
+            {
+                Configuration c = new Configuration();
+                var config = Configuration.FromFile(fileName, c);
+                _manager.ProcessConfiguration(_cancelToken, config);
+
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger().Error(ex);                         
+            }
+        }
+
+        private void ShutdownDirectoryMonitor()
+        {
+            _stopService = true;
+            _dirWatcher.EnableRaisingEvents = false;
+            LogManager.GetCurrentClassLogger().Info("Stopping Directory Monitor");            
+        }
+
+        private void DirectoryWatcher(string directoryToWatch)
+        {
+            LogManager.GetCurrentClassLogger().Info("Starting Directory Monitor {0}", directoryToWatch);                       
+        }
+
+        public static Configuration FromDirectory(string jsonDirectory, CancellationToken cancelToken, Manager manager)
         {
             Configuration c = null;
-
+      
             foreach (string jsonConfFile in Directory.GetFiles(jsonDirectory, "*.json"))
             {
                 if (!string.IsNullOrEmpty(jsonConfFile))
@@ -109,6 +200,10 @@ namespace TimberWinR
                 }
             }
 
+            // Startup Directory Monitor
+            if (manager.LiveMonitor)           
+                c.MonitorDirectory(jsonDirectory, cancelToken, manager);            
+
             return c;
         }
 
@@ -148,6 +243,8 @@ namespace TimberWinR
                     c._stdins.AddRange(x.TimberWinR.Inputs.Stdins.ToList());
                 if (x.TimberWinR.Inputs.Logs != null)
                     c._logs.AddRange(x.TimberWinR.Inputs.Logs.ToList());
+                if (x.TimberWinR.Inputs.TailFiles != null)
+                    c._tails.AddRange(x.TimberWinR.Inputs.TailFiles.ToList());
                 if (x.TimberWinR.Inputs.Tcps != null)
                     c._tcps.AddRange(x.TimberWinR.Inputs.Tcps.ToList());
                 if (x.TimberWinR.Inputs.Udps != null)

TimberWinR/Inputs/LogsFileDatabase.cs

@@ -0,0 +1,207 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using Newtonsoft.Json;
+using NLog;
+using TimberWinR.Parser;
+
+
+namespace TimberWinR.Inputs
+{
+    //
+    // Maintain persistent state for Log files (to be used across restarts)
+    //
+    public class LogsFileDatabase
+    {
+        private static readonly object _locker = new object();
+        private List<LogsFileDatabaseEntry> Entries { get; set; }
+        private string DatabaseDirectory { get; set; }
+        public string DatabaseFileName
+        {
+            get { return Path.Combine(DatabaseDirectory, ".timberwinrdb"); }
+        }
+
+        public static Manager Manager { get; set; }
+
+        private static LogsFileDatabase instance;
+
+        private bool ExistingFile(string logName)
+        {
+            lock (_locker)
+            {
+                return ExistingFileTest(logName);
+            }
+        }
+
+        private LogsFileDatabaseEntry FindFile(string logName)
+        {
+            lock (_locker)
+            {
+                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+                return existingEntry;
+            }
+        }
+        private bool ExistingFileTest(string logName)
+        {
+            var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+            return existingEntry != null;
+        }
+
+        private void RemoveFileEntry(string logName)
+        {
+            lock (_locker)
+            {
+                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+                if (existingEntry != null)
+                {
+                    Entries.Remove(existingEntry);
+                    WriteDatabaseFileNoLock();
+                }
+            }
+        }
+
+        private LogsFileDatabaseEntry AddFileEntry(string logName)
+        {
+            var de = new LogsFileDatabaseEntry();
+            lock (_locker)
+            {
+                de.NewFile = true;
+                var fi = new FileInfo(logName);
+                de.FileName = logName;
+                de.Size = fi.Length;
+                de.SampleTime = DateTime.UtcNow;
+                de.CreationTimeUtc = fi.CreationTimeUtc;
+                Entries.Add(de);
+                WriteDatabaseFileNoLock();
+            }
+            return de;
+        }
+
+        public static LogsFileDatabaseEntry LookupLogFile(string logName)
+        {
+            LogsFileDatabaseEntry dbe = Instance.FindFile(logName);
+            if (dbe == null)
+                dbe = Instance.AddFileEntry(logName);
+            else
+                dbe.NewFile = false;
+    
+            return dbe;
+        }
+
+        public static void Update(LogsFileDatabaseEntry dbe)
+        {
+            Instance.UpdateEntry(dbe);
+        }
+
+        private void UpdateEntry(LogsFileDatabaseEntry dbe)
+        {
+            lock(_locker)
+            {
+                var fi = new FileInfo(dbe.FileName);
+                dbe.CreationTimeUtc = fi.CreationTimeUtc;
+                dbe.SampleTime = DateTime.UtcNow;
+                dbe.Size = fi.Length;
+                WriteDatabaseFileNoLock();
+            }
+            
+        }
+        public static LogsFileDatabase Instance
+        {
+            get
+            {
+                if (instance == null)
+                {
+                    instance = new LogsFileDatabase(Manager.LogfileDir);
+                    lock (_locker)
+                    {
+                        if (!Directory.Exists(instance.DatabaseDirectory))
+                            Directory.CreateDirectory(instance.DatabaseDirectory);
+                        // If it exists, read the current state, otherwise create an empty database.
+                        if (File.Exists(instance.DatabaseFileName))
+                            instance.ReadDatabaseNoLock();
+                        else
+                            instance.WriteDatabaseFileNoLock();
+                    }
+                }
+                return instance;
+            }
+        }
+
+        private void ReadDatabaseNoLock()
+        {
+            try
+            {
+                var serializer = new JsonSerializer();
+                if (File.Exists(DatabaseFileName))
+                    Entries =
+                        JsonConvert.DeserializeObject<List<LogsFileDatabaseEntry>>(File.ReadAllText(DatabaseFileName));
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger()
+                    .Error("Error reading database '{0}': {1}", DatabaseFileName, ex.ToString());
+                try
+                {
+                    if (File.Exists(DatabaseFileName))
+                        File.Delete(DatabaseFileName);
+                    LogManager.GetCurrentClassLogger().Info("Creating New Database '{0}'", DatabaseFileName);
+                    WriteDatabaseLock();
+                }
+                catch (Exception ex2)
+                {
+                    LogManager.GetCurrentClassLogger().Info("Error Creating New Database '{0}': {1}", DatabaseFileName, ex2.ToString());
+                }              
+            }
+        }
+        private void WriteDatabaseFileNoLock()
+        {
+            try
+            {
+                File.WriteAllText(DatabaseFileName, JsonConvert.SerializeObject(instance.Entries), Encoding.UTF8);
+            }
+            catch (Exception ex)
+            {
+                LogManager.GetCurrentClassLogger()
+                    .Error("Error saving database '{0}': {1}", DatabaseFileName, ex.ToString());
+            }
+        }
+
+
+        private void ReadDatabaseLock()
+        {
+            lock (_locker)
+            {
+                ReadDatabaseNoLock();
+            }
+
+        }
+        private void WriteDatabaseLock()
+        {
+            lock (_locker)
+            {
+                WriteDatabaseFileNoLock();
+            }
+        }
+
+        private LogsFileDatabase(string databaseDirectory)
+        {
+            DatabaseDirectory = databaseDirectory;
+            Entries = new List<LogsFileDatabaseEntry>();
+        }
+
+    }
+
+    public class LogsFileDatabaseEntry
+    {
+        [JsonIgnore]
+        public bool NewFile { get; set; }
+        public string FileName { get; set; }
+        public Int64 MaxRecords { get; set; }
+        public DateTime CreationTimeUtc { get; set; }
+        public DateTime SampleTime { get; set; }
+        public long Size { get; set; }       
+    }
+
+}

TimberWinR/Inputs/LogsListener.cs

@@ -22,146 +22,6 @@ using TimberWinR.Parser;
 
 namespace TimberWinR.Inputs
 {
-    public class LogsFileDatabase
-    {
-        private static readonly object _locker = new object();
-        private List<LogsFileDatabaseEntry> Entries { get; set; }
-        private string DatabaseDirectory { get; set; }
-        public string DatabaseFileName
-        {
-            get { return Path.Combine(DatabaseDirectory, ".timberwinrdb"); }
-        }
-
-        public static Manager Manager { get; set; }
-
-        private static LogsFileDatabase instance;
-
-        private bool ExistingFile(string logName)
-        {
-            lock (_locker)
-            {
-                return ExistingFileTest(logName);
-            }
-        }
-
-        private bool ExistingFileTest(string logName)
-        {
-            var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
-            return existingEntry != null;
-        }
-
-        private void RemoveFileEntry(string logName)
-        {
-            lock (_locker)
-            {
-                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
-                if (existingEntry != null)
-                {
-                    Entries.Remove(existingEntry);
-                    WriteDatabaseFileNoLock();
-                }
-            }
-        }
-
-        private LogsFileDatabaseEntry AddFileEntry(string logName, TextLineInputFormat fmt)
-        {
-            LogsFileDatabaseEntry de = new LogsFileDatabaseEntry();
-            lock (_locker)
-            {
-                var lq = new LogQuery();
-                FileInfo fi = new FileInfo(logName);
-                de.FileName = logName;
-                de.Size = fi.Length;
-                de.SampleTime = DateTime.UtcNow;
-                de.CreationTime = fi.CreationTimeUtc;
-                if (fi.Exists)
-                {
-                    var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
-                    var rcount = lq.Execute(qcount, fmt);
-                    var qr = rcount.getRecord();
-                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
-                    de.MaxRecords = lrn;
-                }
-                Entries.Add(de);
-                WriteDatabaseFileNoLock();
-            }
-            return de;
-        }
-
-        public static LogsFileDatabaseEntry AddLogFile(string logName, TextLineInputFormat fmt)
-        {
-            Instance.RemoveFileEntry(logName); // Remove if already exists, otherwise ignores.
-            return Instance.AddFileEntry(logName, fmt);
-        }
-
-        public static LogsFileDatabase Instance
-        {
-            get
-            {
-                if (instance == null)
-                {
-                    instance = new LogsFileDatabase(Manager.LogfileDir);
-                    lock (_locker)
-                    {
-                        if (!Directory.Exists(instance.DatabaseDirectory))
-                        {
-                            Directory.CreateDirectory(instance.DatabaseDirectory);
-                        }
-                        if (File.Exists(instance.DatabaseFileName))
-                            instance.ReadDatabaseNoLock();
-                        else
-                            instance.WriteDatabaseFileNoLock();
-                    }
-                }
-                return instance;
-            }
-        }
-
-        private void ReadDatabaseNoLock()
-        {
-            JsonSerializer serializer = new JsonSerializer();
-            if (File.Exists(DatabaseFileName))
-                Entries = JsonConvert.DeserializeObject<List<LogsFileDatabaseEntry>>(File.ReadAllText(DatabaseFileName));
-        }
-        private void WriteDatabaseFileNoLock()
-        {
-            File.WriteAllText(DatabaseFileName, JsonConvert.SerializeObject(instance.Entries), Encoding.UTF8);
-        }
-
-
-        private void ReadDatabaseLock()
-        {
-            lock (_locker)
-            {
-                ReadDatabaseNoLock();
-            }
-
-        }
-        private void WriteDatabaseLock()
-        {
-            lock (_locker)
-            {
-                WriteDatabaseFileNoLock();
-            }
-        }
-
-        private LogsFileDatabase(string databaseDirectory)
-        {
-            DatabaseDirectory = databaseDirectory;
-            Entries = new List<LogsFileDatabaseEntry>();
-        }
-
-    }
-
-    public class LogsFileDatabaseEntry
-    {
-        public string FileName { get; set; }
-        public Int64 MaxRecords { get; set; }
-        public DateTime CreationTime { get; set; }
-        public DateTime SampleTime { get; set; }
-        public long Size { get; set; }
-    }
-
     /// <summary>
     /// Tail a file.
     /// </summary>
@@ -338,6 +198,8 @@ namespace TimberWinR.Inputs
                 recurse = _arguments.Recurse
             };
 
+            Dictionary<string, string> _fnfmap = new Dictionary<string, string>();
+
             using (var syncHandle = new ManualResetEventSlim())
             {
                 // Execute the query
@@ -456,13 +318,15 @@ namespace TimberWinR.Inputs
                                 rs = null;
                                 GC.Collect();
                             }
-                            // Sleep 
-                            if (!Stop)
-                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
                         }
                         catch (FileNotFoundException fnfex)
                         {
-                            LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
+                            string fn = fnfex.FileName;
+
+                            if (!_fnfmap.ContainsKey(fn))
+                                LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
+
+                            _fnfmap[fn] = fn;
                         }
                         catch (OperationCanceledException oce)
                         {
@@ -474,7 +338,20 @@ namespace TimberWinR.Inputs
                         }
                         finally
                         {
-                            oLogQuery = null;
+                            try
+                            {
+                                oLogQuery = null;
+                                // Sleep 
+                                if (!Stop)
+                                    syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                            }
+                            catch (OperationCanceledException oce)
+                            {                               
+                            }
+                            catch (Exception ex1)
+                            {
+                                LogManager.GetCurrentClassLogger().Warn(ex1);
+                            }
                         }
                     }
                 }

TimberWinR/Inputs/TailFileListener.cs

@@ -0,0 +1,328 @@
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.Linq;
+using System.Net.Configuration;
+using System.Runtime.InteropServices;
+using System.Text;
+using System.Text.RegularExpressions;
+using System.Threading;
+using System.Threading.Tasks;
+using System.IO;
+using Interop.MSUtil;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using Newtonsoft.Json.Serialization;
+
+using NLog;
+using NLog.LayoutRenderers;
+using TimberWinR.Parser;
+
+namespace TimberWinR.Inputs
+{
+    /// <summary>
+    /// Tail a file.
+    /// </summary>
+    public class TailFileListener : InputListener
+    {
+        private int _pollingIntervalInSeconds;
+        private TimberWinR.Parser.TailFile _arguments;
+        private long _receivedMessages;
+        private Dictionary<string, Int64> _logFileMaxRecords;
+        private Dictionary<string, DateTime> _logFileCreationTimes;
+        private Dictionary<string, DateTime> _logFileSampleTimes;
+        private Dictionary<string, long> _logFileSizes;
+        private Codec _codec;
+        private List<string> _multiline { get; set; }
+
+        public bool Stop { get; set; }
+
+        public TailFileListener(TimberWinR.Parser.TailFile arguments, CancellationToken cancelToken)
+            : base(cancelToken, "Win32-TailLog")
+        {
+            Stop = false;
+
+            _codec = arguments.Codec;
+            _logFileMaxRecords = new Dictionary<string, Int64>();
+            _logFileCreationTimes = new Dictionary<string, DateTime>();
+            _logFileSampleTimes = new Dictionary<string, DateTime>();
+            _logFileSizes = new Dictionary<string, long>();
+
+            _receivedMessages = 0;
+            _arguments = arguments;
+            _pollingIntervalInSeconds = arguments.Interval;
+
+            foreach (string srcFile in _arguments.Location.Split(','))
+            {
+                string file = srcFile.Trim();
+                Task.Factory.StartNew(() => TailFileWatcher(file));
+            }
+        }
+
+        public override void Shutdown()
+        {
+            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
+            Stop = true;
+            base.Shutdown();
+        }
+
+        public override JObject ToJson()
+        {
+            JObject json = new JObject(
+                new JProperty("log",
+                    new JObject(
+                        new JProperty("messages", _receivedMessages),
+                        new JProperty("type", InputType),
+                        new JProperty("location", _arguments.Location),
+                        new JProperty("logSource", _arguments.LogSource),
+                        new JProperty("recurse", _arguments.Recurse),
+
+                        new JProperty("files",
+                            new JArray(from f in _logFileMaxRecords.Keys
+                                       select new JValue(f))),
+                        new JProperty("fileSampleTimes",
+                            new JArray(from f in _logFileSampleTimes.Values
+                                       select new JValue(f))),
+                        new JProperty("fileSizes",
+                            new JArray(from f in _logFileSizes.Values
+                                       select new JValue(f))),
+                        new JProperty("fileIndices",
+                            new JArray(from f in _logFileMaxRecords.Values
+                                       select new JValue(f))),
+                        new JProperty("fileCreationDates",
+                            new JArray(from f in _logFileCreationTimes.Values
+                                       select new JValue(f)))
+                        )));
+
+
+            if (_codec != null)
+            {
+                var cp = new JProperty("codec",
+                    new JArray(
+                        new JObject(
+                            new JProperty("type", _codec.Type.ToString()),
+                            new JProperty("what", _codec.What.ToString()),
+                            new JProperty("negate", _codec.Negate),
+                            new JProperty("multilineTag", _codec.MultilineTag),
+                            new JProperty("pattern", _codec.Pattern))));
+                json.Add(cp);
+            }
+
+
+            return json;
+        }
+
+        // return true to cancel codec
+        private void applyMultilineCodec(string msg)
+        {
+            if (_codec.Re == null)
+                _codec.Re = new Regex(_codec.Pattern);
+
+            Match match = _codec.Re.Match(msg);
+
+            bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
+
+            switch (_codec.What)
+            {
+                case Codec.WhatType.previous:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+
+                        _multiline.Add(msg);
+                    }
+                    else // No Match
+                    {
+                        if (_multiline != null)
+                        {
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codec.MultilineTag));
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);
+                            _receivedMessages++;
+                        }
+                        _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    break;
+                case Codec.WhatType.next:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    else // No match
+                    {
+                        if (_multiline != null)
+                        {
+                            _multiline.Add(msg);
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codec.MultilineTag));
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);
+                            _receivedMessages++;
+                        }
+                        else
+                        {
+                            JObject jo = new JObject();
+                            jo["message"] = msg;
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);
+                            _receivedMessages++;
+                        }
+                    }
+                    break;
+            }
+        }
+
+        private void TailFileContents(string fileName, long offset)
+        {
+            using (StreamReader reader = new StreamReader(new FileStream(fileName,
+                     FileMode.Open, FileAccess.Read, FileShare.ReadWrite)))
+            {
+                //start at the end of the file
+                long lastMaxOffset = offset;
+
+                //if the file size has not changed, idle
+                if (reader.BaseStream.Length == lastMaxOffset)
+                    return;
+
+                //seek to the last max offset
+                reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin);
+
+                //read out of the file until the EOF
+                string line = "";
+                long lineOffset = 0;
+                while ((line = reader.ReadLine()) != null)
+                {
+                    if (string.IsNullOrEmpty(line))
+                        continue;
+
+                    long index = lastMaxOffset + lineOffset;
+                    string text = line;
+                    string logFileName = fileName;
+                    var json = new JObject();
+
+                    if (json["logSource"] == null)
+                    {
+                        if (string.IsNullOrEmpty(_arguments.LogSource))
+                            json.Add(new JProperty("logSource", fileName));
+                        else
+                            json.Add(new JProperty("logSource", _arguments.LogSource));
+                    }
+                    json["Text"] = line;
+                    json["Index"] = index;
+                    json["LogFileName"] = fileName;
+
+                    if (_codec != null && _codec.Type == Codec.CodecType.multiline)
+                        applyMultilineCodec(line);
+                    else
+                    {
+                        ProcessJson(json);
+                        Interlocked.Increment(ref _receivedMessages);
+                    }
+                    lineOffset += line.Length;
+                    // Console.WriteLine("File: {0}:{1}: {2}", fileName, reader.BaseStream.Position, line);
+                }
+                //update the last max offset
+                lastMaxOffset = reader.BaseStream.Position;
+            }
+        }
+        // One thread for each kind of file to watch, i.e. "*.log,*.txt" would be two separate
+        // threads.
+        private void TailFileWatcher(string fileToWatch)
+        {
+            Dictionary<string, string> _fnfmap = new Dictionary<string, string>();
+
+            using (var syncHandle = new ManualResetEventSlim())
+            {
+                // Execute the query
+                while (!Stop && !CancelToken.IsCancellationRequested)
+                {
+                    try
+                    {
+                        if (!CancelToken.IsCancellationRequested)
+                        {
+                            string path = Path.GetDirectoryName(fileToWatch);
+                            string name = Path.GetFileName(fileToWatch);
+                            if (string.IsNullOrEmpty(path))
+                                path = ".";
+
+                            // Ok, we have a potential file filter here as 'fileToWatch' could be foo.log or *.log
+
+                            SearchOption so = SearchOption.TopDirectoryOnly;
+                            if (_arguments.Recurse == -1)
+                                so = SearchOption.AllDirectories;
+
+                            foreach (string fileName in Directory.GetFiles(path, name, so))
+                            {
+                                var dbe = LogsFileDatabase.LookupLogFile(fileName);
+                                FileInfo fi = new FileInfo(dbe.FileName);
+                                //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);
+                                long length = fi.Length;
+                                bool logHasRolled = false;
+                                if (fi.Length < dbe.Size || fi.CreationTimeUtc != dbe.CreationTimeUtc)
+                                {
+                                    LogManager.GetCurrentClassLogger().Info("Log has Rolled: {0}", dbe.FileName);
+                                    logHasRolled = true;
+                                }
+                                bool processWholeFile = logHasRolled || dbe.NewFile;
+                                if (processWholeFile)
+                                {
+                                    LogManager.GetCurrentClassLogger().Info("Process Whole File: {0}", dbe.FileName);
+                                    TailFileContents(dbe.FileName, 0);
+                                }
+                                else
+                                {
+                                    TailFileContents(dbe.FileName, dbe.Size);
+                                }
+                                LogsFileDatabase.Update(dbe);
+                            }
+                        }
+                    }
+                    catch (FileNotFoundException fnfex)
+                    {
+                        string fn = fnfex.FileName;
+
+                        if (!_fnfmap.ContainsKey(fn))
+                            LogManager.GetCurrentClassLogger().Warn(fnfex.Message);
+                        _fnfmap[fn] = fn;
+                    }
+                    catch (OperationCanceledException oce)
+                    {
+                        break;
+                    }
+                    catch (Exception ex)
+                    {
+                        LogManager.GetCurrentClassLogger().Error(ex);
+                    }
+                    finally
+                    {
+                        try
+                        {
+                            if (!Stop)
+                                syncHandle.Wait(TimeSpan.FromSeconds(_pollingIntervalInSeconds), CancelToken);
+                        }
+                        catch (OperationCanceledException)
+                        {
+                        }
+                        catch (Exception ex1)
+                        {
+                            LogManager.GetCurrentClassLogger().Warn(ex1);
+                        }                      
+                    }
+                }
+            }
+            Finished();
+        }
+    }
+}
+

TimberWinR/Inputs/UdpInputListener.cs

@@ -21,6 +21,7 @@ namespace TimberWinR.Inputs
 
         private readonly int _port;
         private long _receivedMessages;
+        private long _parsedErrors;
 
         private struct listenProfile
         {
@@ -34,6 +35,7 @@ namespace TimberWinR.Inputs
                 new JProperty("udp",
                     new JObject(
                         new JProperty("port", _port),
+                        new JProperty("errors", _parsedErrors),
                         new JProperty("messages", _receivedMessages)
                         )));
 
@@ -71,22 +73,25 @@ namespace TimberWinR.Inputs
         private void StartListener(object useProfile)
         {
             var profile = (listenProfile)useProfile;
-
+            string lastMessage = "";
             try
             {
                 while (!CancelToken.IsCancellationRequested)
                 {
-                    byte[] bytes = profile.client.Receive(ref profile.endPoint);
                     try
                     {
-                        var data = Encoding.ASCII.GetString(bytes, 0, bytes.Length);
+                        byte[] bytes = profile.client.Receive(ref profile.endPoint);  
+                        var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
+                        lastMessage = data;
                         JObject json = JObject.Parse(data);
                         ProcessJson(json);
                         _receivedMessages++;
                     }
                     catch (Exception ex1)
                     {
+                        LogManager.GetCurrentClassLogger().Warn("Bad JSON: {0}", lastMessage);
                         LogManager.GetCurrentClassLogger().Warn(ex1);
+                        _parsedErrors++;
                     }
                 }
                 _udpListener.Close();

TimberWinR/Manager.cs

@@ -26,6 +26,8 @@ namespace TimberWinR
         public List<TcpInputListener> Tcps { get; set; }
         public List<TcpInputListener> Udps { get; set; }
         public List<InputListener> Listeners { get; set; }
+        public bool LiveMonitor { get; set; }
+
         public DateTime StartedOn { get; set; }
         public string JsonConfig { get; set; }
         public string LogfileDir { get; set; }
@@ -60,11 +62,17 @@ namespace TimberWinR
             Interlocked.Add(ref numMessages, count);
         }
 
-        public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken)
+        public Manager()
+        {
+            LogsFileDatabase.Manager = this;                   
+        }
+
+        public Manager(string jsonConfigFile, string logLevel, string logfileDir, bool liveMonitor, CancellationToken cancelToken)
         {
             LogsFileDatabase.Manager = this;           
   
             StartedOn = DateTime.UtcNow;
+            LiveMonitor = liveMonitor;
 
             var vfi = new FileInfo(jsonConfigFile);
 
@@ -105,7 +113,6 @@ namespace TimberWinR
             LogManager.GetCurrentClassLogger()
                 .Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
 
-
             try
             {
                 // Is it a directory?
@@ -113,7 +120,7 @@ namespace TimberWinR
                 {
                     DirectoryInfo di = new DirectoryInfo(jsonConfigFile);
                     LogManager.GetCurrentClassLogger().Info("Initialized, Reading Configurations From {0}", di.FullName);
-                    Config = Configuration.FromDirectory(jsonConfigFile);
+                    Config = Configuration.FromDirectory(jsonConfigFile, cancelToken, this);
                 }
                 else
                 {
@@ -139,36 +146,40 @@ namespace TimberWinR
             LogManager.GetCurrentClassLogger().Info("Log Directory {0}", logfileDir);
             LogManager.GetCurrentClassLogger().Info("Logging Level: {0}", LogManager.GlobalThreshold);
 
-            // Read the Configuration file
-            if (Config != null)
+            ProcessConfiguration(cancelToken, Config);
+        }
+
+        public void ProcessConfiguration(CancellationToken cancelToken, Configuration config)
+        {
+// Read the Configuration file
+            if (config != null)
             {
-                if (Config.RedisOutputs != null)
+                if (config.RedisOutputs != null)
                 {
-                    foreach (var ro in Config.RedisOutputs)
+                    foreach (var ro in config.RedisOutputs)
                     {
                         var redis = new RedisOutput(this, ro, cancelToken);
                         Outputs.Add(redis);
                     }
-
                 }
-                if (Config.ElasticsearchOutputs != null)
+                if (config.ElasticsearchOutputs != null)
                 {
-                    foreach (var ro in Config.ElasticsearchOutputs)
+                    foreach (var ro in config.ElasticsearchOutputs)
                     {
                         var els = new ElasticsearchOutput(this, ro, cancelToken);
                         Outputs.Add(els);
                     }
                 }
-                if (Config.StdoutOutputs != null)
+                if (config.StdoutOutputs != null)
                 {
-                    foreach (var ro in Config.StdoutOutputs)
+                    foreach (var ro in config.StdoutOutputs)
                     {
                         var stdout = new StdoutOutput(this, ro, cancelToken);
                         Outputs.Add(stdout);
                     }
                 }
 
-                foreach (Parser.IISW3CLog iisw3cConfig in Config.IISW3C)
+                foreach (Parser.IISW3CLog iisw3cConfig in config.IISW3C)
                 {
                     var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
                     Listeners.Add(elistner);
@@ -176,7 +187,7 @@ namespace TimberWinR
                         output.Connect(elistner);
                 }
 
-                foreach (Parser.W3CLog iisw3cConfig in Config.W3C)
+                foreach (Parser.W3CLog iisw3cConfig in config.W3C)
                 {
                     var elistner = new W3CInputListener(iisw3cConfig, cancelToken);
                     Listeners.Add(elistner);
@@ -184,7 +195,7 @@ namespace TimberWinR
                         output.Connect(elistner);
                 }
 
-                foreach (Parser.WindowsEvent eventConfig in Config.Events)
+                foreach (Parser.WindowsEvent eventConfig in config.Events)
                 {
                     var elistner = new WindowsEvtInputListener(eventConfig, cancelToken);
                     Listeners.Add(elistner);
@@ -192,7 +203,7 @@ namespace TimberWinR
                         output.Connect(elistner);
                 }
 
-                foreach (var logConfig in Config.Logs)
+                foreach (var logConfig in config.Logs)
                 {
                     var elistner = new LogsListener(logConfig, cancelToken);
                     Listeners.Add(elistner);
@@ -200,7 +211,15 @@ namespace TimberWinR
                         output.Connect(elistner);
                 }
 
-                foreach (var tcp in Config.Tcps)
+                foreach (var logConfig in config.TailFiles)
+                {
+                    var elistner = new TailFileListener(logConfig, cancelToken);
+                    Listeners.Add(elistner);
+                    foreach (var output in Outputs)
+                        output.Connect(elistner);
+                }
+
+                foreach (var tcp in config.Tcps)
                 {
                     var elistner = new TcpInputListener(cancelToken, tcp.Port);
                     Listeners.Add(elistner);
@@ -208,7 +227,7 @@ namespace TimberWinR
                         output.Connect(elistner);
                 }
 
-                foreach (var udp in Config.Udps)
+                foreach (var udp in config.Udps)
                 {
                     var elistner = new UdpInputListener(cancelToken, udp.Port);
                     Listeners.Add(elistner);
@@ -216,7 +235,7 @@ namespace TimberWinR
                         output.Connect(elistner);
                 }
 
-                foreach (var stdin in Config.Stdins)
+                foreach (var stdin in config.Stdins)
                 {
                     var elistner = new StdinListener(stdin, cancelToken);
                     Listeners.Add(elistner);
@@ -225,28 +244,28 @@ namespace TimberWinR
                 }
 
                 var computerName = System.Environment.MachineName + "." +
-                       Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
-                           @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
-                           .GetValue("Domain", "")
-                           .ToString();
+                                   Microsoft.Win32.Registry.LocalMachine.OpenSubKey(
+                                       @"SYSTEM\CurrentControlSet\services\Tcpip\Parameters")
+                                       .GetValue("Domain", "")
+                                       .ToString();
 
                 foreach (var output in Outputs)
                 {
                     var name = Assembly.GetExecutingAssembly().GetName();
                     JObject json = new JObject(
-                     new JProperty("TimberWinR",
-                         new JObject(
-                             new JProperty("version", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
-                             new JProperty("host", computerName),
-                             new JProperty("output", output.Name),
-                             new JProperty("initialized", DateTime.UtcNow)
-                             )));
+                        new JProperty("TimberWinR",
+                            new JObject(
+                                new JProperty("version",
+                                    GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString()),
+                                new JProperty("host", computerName),
+                                new JProperty("output", output.Name),
+                                new JProperty("initialized", DateTime.UtcNow)
+                                )));
                     json.Add(new JProperty("type", "Win32-TimberWinR"));
                     json.Add(new JProperty("host", computerName));
                     output.Startup(json);
                 }
             }
-
         }
 
 

TimberWinR/Outputs/Redis.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics.Eventing.Reader;
 using System.Linq;
 using System.Linq.Expressions;
 using System.Net.Sockets;
@@ -23,26 +24,29 @@ namespace TimberWinR.Outputs
         private readonly int _port;
         private readonly int _timeout;
         private readonly object _locker = new object();
-        private readonly List<string> _jsonQueue;
-        // readonly Task _consumerTask;
+        private readonly List<string> _jsonQueue;       
         private readonly string[] _redisHosts;
         private int _redisHostIndex;
         private TimberWinR.Manager _manager;
         private readonly int _batchCount;
+        private int _currentBatchCount;
+        private readonly int _maxBatchCount;       
         private readonly int _interval;
         private readonly int _numThreads;
-
+        private readonly long[] _sampleQueueDepths;
+        private int _sampleCountIndex;
         private long _sentMessages;
         private long _errorCount;
         private long _redisDepth;
-
-        private int _maxQueueSize;
-        private bool _queueOverflowDiscardOldest;
+        private DateTime? _lastErrorTime;
+        private const int QUEUE_SAMPLE_SIZE = 30; // 30 samples over 2.5 minutes (default)
+        private readonly int _maxQueueSize;
+        private readonly bool _queueOverflowDiscardOldest;
 
         public bool Stop { get; set; }
 
         /// <summary>
-        /// Get the next client
+        /// Get the next client from the list of hosts.
         /// </summary>
         /// <returns></returns>
         private RedisClient getClient()
@@ -56,16 +60,17 @@ namespace TimberWinR.Outputs
                 try
                 {
                     RedisClient client = new RedisClient(_redisHosts[_redisHostIndex], _port, _timeout);
-
-                    _redisHostIndex++;
-                    if (_redisHostIndex >= _redisHosts.Length)
-                        _redisHostIndex = 0;
-
                     return client;
                 }
                 catch (Exception)
                 {
                 }
+                finally
+                {
+                    _redisHostIndex++;
+                    if (_redisHostIndex >= _redisHosts.Length)
+                        _redisHostIndex = 0;
+                }
                 numTries++;
             }
 
@@ -79,6 +84,7 @@ namespace TimberWinR.Outputs
                     new JObject(
                         new JProperty("host", string.Join(",", _redisHosts)),
                         new JProperty("errors", _errorCount),
+                        new JProperty("lastErrorTime", _lastErrorTime),
                         new JProperty("redis_depth", _redisDepth),
                         new JProperty("sent_messages", _sentMessages),
                         new JProperty("queued_messages", _jsonQueue.Count),
@@ -88,6 +94,10 @@ namespace TimberWinR.Outputs
                         new JProperty("interval", _interval),
                         new JProperty("threads", _numThreads),
                         new JProperty("batchcount", _batchCount),
+                        new JProperty("currentBatchCount", _currentBatchCount),
+                        new JProperty("maxBatchCount", _maxBatchCount),
+                        new JProperty("averageQueueDepth", AverageQueueDepth()),   
+                        new JProperty("queueSamples", new JArray(_sampleQueueDepths)),
                         new JProperty("index", _logstashIndexName),
                         new JProperty("hosts",
                             new JArray(
@@ -100,11 +110,19 @@ namespace TimberWinR.Outputs
         public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken)
             : base(cancelToken, "Redis")
         {
+            // Last QUEUE_SAMPLE_SIZE queue samples (spans timestamp * 10)
+            _sampleQueueDepths = new long[QUEUE_SAMPLE_SIZE];
+            _sampleCountIndex = 0;
             _redisDepth = 0;
             _batchCount = ro.BatchCount;
+            _maxBatchCount = ro.MaxBatchCount;
+            // Make sure maxBatchCount is larger than batchCount
+            if (_maxBatchCount < _batchCount)
+                _maxBatchCount = _batchCount*10;
+            _currentBatchCount = _batchCount;
             _manager = manager;
             _redisHostIndex = 0;
-            _redisHosts = ro.Host;
+            _redisHosts = ro.Host;          
             _jsonQueue = new List<string>();
             _port = ro.Port;
             _timeout = ro.Timeout;
@@ -112,6 +130,7 @@ namespace TimberWinR.Outputs
             _interval = ro.Interval;
             _numThreads = ro.NumThreads;
             _errorCount = 0;
+            _lastErrorTime = null;
             _maxQueueSize = ro.MaxQueueSize;
             _queueOverflowDiscardOldest = ro.QueueOverflowDiscardOldest;
 
@@ -183,6 +202,18 @@ namespace TimberWinR.Outputs
             return drop;
         }
 
+
+        // 
+        // What is average queue depth?
+        //
+        private int AverageQueueDepth()
+        {
+            lock(_locker)
+            {
+                return (int)_sampleQueueDepths.Average();
+            }
+        }
+
         // 
         // Pull off messages from the Queue, batch them up and send them all across
         // 
@@ -198,10 +229,16 @@ namespace TimberWinR.Outputs
                         try
                         {
                             string[] messages;
+                            // Exclusively
                             lock (_locker)
-                            {
-                                messages = _jsonQueue.Take(_batchCount).ToArray();
+                            {           
+                                // Take a sample of the queue depth
+                                if (_sampleCountIndex >= QUEUE_SAMPLE_SIZE)
+                                    _sampleCountIndex = 0;     
+                                _sampleQueueDepths[_sampleCountIndex++] = _jsonQueue.Count;                                                                                    
+                                messages = _jsonQueue.Take(_currentBatchCount).ToArray();
                                 _jsonQueue.RemoveRange(0, messages.Length);
+                                var remainingCount = _jsonQueue.Count;
                                 if (messages.Length > 0)
                                     _manager.IncrementMessageCount(messages.Length);
                             }
@@ -209,6 +246,7 @@ namespace TimberWinR.Outputs
                             if (messages.Length > 0)
                             {
                                 int numHosts = _redisHosts.Length;
+                                bool sentSuccessfully = false;
                                 while (numHosts-- > 0)
                                 {
                                     try
@@ -225,17 +263,22 @@ namespace TimberWinR.Outputs
                                                 try
                                                 {
                                                     _redisDepth = client.RPush(_logstashIndexName, messages);
-                                                    _sentMessages += messages.Length;
+                                                    _sentMessages += messages.Length;                                                  
+                                                    client.EndPipe();
+                                                    sentSuccessfully = true;
                                                 }
                                                 catch (SocketException ex)
                                                 {
                                                     LogManager.GetCurrentClassLogger().Warn(ex);
                                                     Interlocked.Increment(ref _errorCount);
+                                                    _lastErrorTime = DateTime.UtcNow;
                                                 }
-                                                finally
+                                                catch (Exception ex)
                                                 {
-                                                    client.EndPipe();
-                                                }
+                                                    LogManager.GetCurrentClassLogger().Error(ex);
+                                                    Interlocked.Increment(ref _errorCount);
+                                                    _lastErrorTime = DateTime.UtcNow;
+                                                }                                               
                                                 break;
                                             }
                                             else
@@ -244,6 +287,7 @@ namespace TimberWinR.Outputs
                                                 LogManager.GetCurrentClassLogger()
                                                     .Fatal("Unable to connect with any Redis hosts, {0}",
                                                         String.Join(",", _redisHosts));
+                                                _lastErrorTime = DateTime.UtcNow;
                                             }
                                         }
                                     }
@@ -251,6 +295,18 @@ namespace TimberWinR.Outputs
                                     {
                                         LogManager.GetCurrentClassLogger().Error(ex);
                                         Interlocked.Increment(ref _errorCount);
+                                        _lastErrorTime = DateTime.UtcNow;                                
+                                    }
+                                } // No more hosts to try.
+
+                                // Re-compute current batch size
+                                updateCurrentBatchCount();
+
+                                if (!sentSuccessfully)                                
+                                {
+                                    lock (_locker)
+                                    {
+                                        _jsonQueue.InsertRange(0, messages);
                                     }
                                 }
                             }
@@ -264,12 +320,26 @@ namespace TimberWinR.Outputs
                         }
                         catch (Exception ex)
                         {
-
-                            throw;
+                            _lastErrorTime = DateTime.UtcNow;
+                            Interlocked.Increment(ref _errorCount);
+                            LogManager.GetCurrentClassLogger().Error(ex);                         
                         }
                     }
                 }
             }
         }
+
+        // Sample the queue and adjust the batch count if needed (ramp up slowly)
+        private void updateCurrentBatchCount()
+        {
+            if (_currentBatchCount < _maxBatchCount && AverageQueueDepth() > _currentBatchCount)
+            {
+                _currentBatchCount += _maxBatchCount/QUEUE_SAMPLE_SIZE;
+            }
+            else // Reset to default
+            {
+                _currentBatchCount = _batchCount;
+            }
+        }
     }
 }

TimberWinR/Parser.cs

@@ -297,6 +297,36 @@ namespace TimberWinR.Parser
         }
     }
 
+    public class TailFile : IValidateSchema
+    {
+        [JsonProperty(PropertyName = "location")]
+        public string Location { get; set; }       
+        [JsonProperty(PropertyName = "recurse")]
+        public int Recurse { get; set; }        
+        [JsonProperty(PropertyName = "fields")]
+        public List<Field> Fields { get; set; }
+        [JsonProperty(PropertyName = "interval")]
+        public int Interval { get; set; }
+        [JsonProperty(PropertyName = "logSource")]
+        public string LogSource { get; set; }
+        [JsonProperty(PropertyName = "codec")]
+        public Codec Codec { get; set; }
+
+        public TailFile()
+        {
+            Fields = new List<Field>();
+            Fields.Add(new Field("LogFilename", "string"));
+            Fields.Add(new Field("Index", "integer"));
+            Fields.Add(new Field("Text", "string"));
+            Interval = 30;
+        }
+
+        public void Validate()
+        {
+
+        }
+    }
+
     public class Log : IValidateSchema
     {
         [JsonProperty(PropertyName = "location")]
@@ -546,6 +576,8 @@ namespace TimberWinR.Parser
         public int Timeout { get; set; }
         [JsonProperty(PropertyName = "batch_count")]
         public int BatchCount { get; set; }
+        [JsonProperty(PropertyName = "max_batch_count")]
+        public int MaxBatchCount { get; set; }
         [JsonProperty(PropertyName = "threads")]
         public int NumThreads { get; set; }
         [JsonProperty(PropertyName = "interval")]
@@ -562,6 +594,7 @@ namespace TimberWinR.Parser
             Host = new string[] { "localhost" };
             Timeout = 10000;
             BatchCount = 10;
+            MaxBatchCount = BatchCount*10;
             NumThreads = 1;
             Interval = 5000;
             QueueOverflowDiscardOldest = true;
@@ -600,6 +633,9 @@ namespace TimberWinR.Parser
         [JsonProperty("Logs")]
         public Log[] Logs { get; set; }
 
+        [JsonProperty("TailFiles")]
+        public TailFile[] TailFiles { get; set; }
+
         [JsonProperty("Tcp")]
         public Tcp[] Tcps { get; set; }
 

TimberWinR/ReleaseNotes.md

@@ -3,10 +3,18 @@
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
 
 Version History
+### 1.3.20.0 - 03/03/2015
+1. Added new Redis parameter _max\_batch\_count_ which increases the _batch\_count_ dynamically over time 
+   to handle input flooding.   Default is _batch\_count_ * 10 
 
-### 1.3.19.0 - 01/12/2015
+### 1.3.19.0 - 02/26/2015
 
-1. Added support for Multiline codecs for Stdin and Logs listeners, addresses issue #23
+1. Added support for Multiline codecs for Stdin and Logs listeners, closes issue [#23](https://github.com/Cimpress-MCP/TimberWinR/issues/23)
+2. Added new TailFiles input type which uses a native implementation (more-efficient) than using LogParser's Log
+3. Updated Udp input listner to use UTF8 Encoding rather than ASCII
+4. Reduced noisy complaint about missing log files for Logs listener
+5. Fixed bug when tailing non-existent log files which resulted in high cpu-usage.
+6. Added feature to watch the configuration directory
 
 ### 1.3.18.0 - 12/22/2014
 

TimberWinR/TimberWinR.csproj

@@ -87,6 +87,8 @@
     <Compile Include="Filters\MutateFilter.cs" />
     <Compile Include="Inputs\FieldDefinitions.cs" />
     <Compile Include="Inputs\IISW3CRowReader.cs" />
+    <Compile Include="Inputs\LogsFileDatabase.cs" />
+    <Compile Include="Inputs\TailFileListener.cs" />
     <Compile Include="Inputs\UdpInputListener.cs" />
     <Compile Include="Inputs\W3CInputListener.cs" />
     <Compile Include="Inputs\IISW3CInputListener.cs" />
@@ -126,6 +128,7 @@
     <None Include="mdocs\DateFilter.md" />
     <None Include="mdocs\Filters.md" />
     <None Include="mdocs\GeoIPFilter.md" />
+    <None Include="mdocs\TailFiles.md" />
     <None Include="mdocs\UdpInput.md" />
     <None Include="mdocs\W3CInput.md" />
     <None Include="mdocs\JsonFilter.md" />

TimberWinR/mdocs/Codec.md

@@ -8,6 +8,10 @@ The following parameters are allowed when configuring the Codec.
 | *type*            | enum           |Codec type 'multiline'                                                    | Must be 'multiline'           |     |
 | *pattern*         | regex          |Regular expression to be matched                                          | Must be legal .NET Regex      |     |
 | *what*            | enum           |Value can be previous or next                                             | If the pattern matched, does event belong to the next or previous event? |     |
+| *negate*          | bool           |Inverts the pattern sense                                                 | If true, a message not matching the pattern will constitute a match of the multiline filter and the what will be applied. (vice-versa is also true) |  false   |
+| *multiline_tag*   | string         |Tag to be added when multiline conversion is applied                      |                               |  multiline | 
+
+This codec applies to [Logs](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) and [Stdin](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdinInput.md) only.
 
 Example Input: Mutliline input log file
 
@@ -20,9 +24,10 @@ Example Input: Mutliline input log file
                 "location": "C:\\Logs1\\multiline.log",
                 "recurse": -1,
                 "codec": {
-                "type": "multiline",
-                "pattern": "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)",
-                "what": "previous"
+                  "negate": false,
+                  "type": "multiline",
+                  "pattern": "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)",
+                  "what": "previous"
                 }
             }
 		 }

TimberWinR/mdocs/ElasticsearchOutput.md

@@ -26,7 +26,7 @@ Example Input:
                     "threads":  1,   
                     "interval": 5000,                             
                     "host": [
-                        "tstlexiceapp006.vistaprint.svc"
+                        "tstlexiceapp006.mycompany.svc"
                     ]
                 }
             ]

TimberWinR/mdocs/JsonFilter.md

@@ -61,7 +61,7 @@ The resulting output would be:
 ```
   {
     "type": "Win32-FileLog",
-    "ComputerName": "dev.vistaprint.net",
+    "ComputerName": "dev.mycompany.net",
     "Text": "{\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}",
     "stuff": {
       "Email": "james@example.com",

TimberWinR/mdocs/Logs.md

@@ -8,6 +8,7 @@ The following parameters are allowed when configuring WindowsEvents.
 | Parameter         |     Type       |  Description                                                             | Details               |  Default |
 | :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
 | *location*        | string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
+| *logSource*       | string  |Source name                                  | Used for conditions |     |
 | *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
 | *splitLongLines*  | boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false |
 | *iCodepage*       | integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
@@ -21,6 +22,7 @@ Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.
         "Inputs": {
             "Logs": [
                 {
+                    "logSource": "log files",
                     "location": "C:\\Logs1\\*.log",
                     "recurse": -1
                 }

TimberWinR/mdocs/RedisOutput.md

@@ -8,6 +8,8 @@ The following parameters are allowed when configuring the Redis output.
 | Parameter     |   Type   |  Description                                                | Details               |  Default |
 | :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
 | *threads*     | string   | Location of log files(s) to monitor                         | Number of worker theads to send messages | 1 |
+| *batch_count* | integer  | Sent as a single message                         | Number of messages to aggregate | 10 |
+| *max_batch_count* | integer  | Dynamically adjusted count maximum                      | Increases over time | batch_count*10 |
 | *interval*    | integer  | Interval in milliseconds to sleep during batch sends        | Interval      | 5000 |
 | *index*       | string   | The name of the redis list                                  | logstash index name | logstash |
 | *host*        | [string] | The hostname(s) of your Redis server(s) | IP or DNS name |  |
@@ -26,7 +28,7 @@ Example Input:
                     "interval": 5000, 
                     "batch_count":  500,              
                     "host": [
-                        "tstlexiceapp006.vistaprint.svc"
+                        "tstlexiceapp006.mycompany.svc"
                     ]
                 }
             ]

TimberWinR/mdocs/StdinInput.md

@@ -3,7 +3,12 @@
 The Stdin Input will read from the console (Console.ReadLine) and build a simple message for testing.
 
 ## Parameters
-There are no Parameters at this time.
+The following parameters are allowed when configuring WindowsEvents.
+
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
+
 
 ```json
 {
@@ -26,5 +31,3 @@ A field: "type": "Win32-Stdin" is automatically appended, and the entire JSON is
 | ---- |:-----| :-----------------------------------------------------------------------|
 | type | STRING |Win32-Stdin |
 | message | STRING | The message typed in  |
-| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
-

TimberWinR/mdocs/TailFiles.md

@@ -0,0 +1,41 @@
+# Input: TailFiles
+
+The TailFiles input will monitor a log (text) file similar to how a Linux "tail -f" command works.  This uses
+a native implementation rather than uses LogParser
+
+## Parameters
+The following parameters are allowed when configuring WindowsEvents.
+
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *location*        | string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
+| *logSource*       | string  |Source name                                  | Used for conditions |     |
+| *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
+| *interval*        | integer |Polling interval in seconds                                     | Defaults every 60 seconds | 60 |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
+
+Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern.  I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
+
+```json
+{
+    "TimberWinR": {
+        "Inputs": {
+            "TailFiles": [
+                {
+                    "logSource": "log files",
+                    "location": "C:\\Logs1\\*.log",
+                    "recurse": -1
+                }
+            ]
+		}
+	}
+}
+```
+## Fields
+After a successful parse of an event, the following fields are added:
+
+| Name | Type | Description |
+| ---- |:-----| :-----------|
+| LogFilename | STRING |Full path of the file containing this line | 
+| Index | INTEGER | Line number |
+| Text | STRING | Text line content  |

chocolateyUninstall.ps1.template

@@ -1,7 +1,7 @@
 $packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
 $installerType = 'msi' #only one of these: exe, msi, msu
 $url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
-$silentArgs = '${PROJECTGUID} /quiet' 
+$silentArgs = '{593EF0C4-54E0-40D5-A3E3-922CD1C25B9E} /quiet' 
 $validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
 UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes
 

chocolateyUninstall.ps1.template.orig

@@ -0,0 +1,8 @@
+$packageName = 'TimberWinR-${version}' # arbitrary name for the package, used in messages
+$installerType = 'msi' #only one of these: exe, msi, msu
+$url = 'http://www.ericfontana.com/TimberWinR/TimberWinR-${version}.0.msi' # download url
+$silentArgs = '${PROJECTGUID} /quiet' 
+$validExitCodes = @(0) #please insert other valid exit codes here, exit codes for ms http://msdn.microsoft.com/en-us/library/aa368542(VS.85).aspx
+UnInstall-ChocolateyPackage "$packageName" "$installerType" "$silentArgs" "$url" -validExitCodes $validExitCodes
+
+