Removed some cleanup

README.md

@@ -107,7 +107,6 @@ represented as a JSON Property or Array.
 1. [Redis](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/RedisOutput.md)
 2. [Elasticsearch](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/ElasticsearchOutput.md)
 3. [Stdout](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/StdoutOutput.md)
-4. [File](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/FileOutput.md)
 
 ## Sample Configuration
 TimberWinR reads a JSON configuration file, an example file is shown here:

TimberWinR.ServiceHost/Properties/AssemblyInfo.cs

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.25.0")]
-[assembly: AssemblyFileVersion("1.3.25.0")]
+[assembly: AssemblyVersion("1.3.24.0")]
+[assembly: AssemblyFileVersion("1.3.24.0")]

TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj

@@ -117,15 +117,6 @@
     <Content Include="test4.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
-    <Content Include="test5-twconfig.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-    <Content Include="test5.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
-    <Content Include="results5.json">
-      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
-    </Content>
   </ItemGroup>
   <ItemGroup>
     <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

TimberWinR.TestGenerator/UdpTestGenerator.cs

@@ -1,6 +1,4 @@
-using System.Security.Cryptography;
-using System.Threading;
-using Newtonsoft.Json;
+using System.Threading;
 using Newtonsoft.Json.Linq;
 using NLog;
 using NLog.Config;
@@ -26,7 +24,7 @@ namespace TimberWinR.TestGenerator
             NumMessages = 100;
             Port = 6379;
             Host = "localhost";
-            SleepTimeMilliseconds = 1;
+            SleepTimeMilliseconds = 10;
         }
     }
 
@@ -50,37 +48,16 @@ namespace TimberWinR.TestGenerator
             {
                 JObject o = new JObject
                 {
-                    {"Application", "udp-generator"},  
-                    {"Executable", "VP.Common.SvcFrm.Services.Host, Version=29.7.0.0, Culture=neutral, PublicKeyToken=null"},
-                    {"RenderedMessage", "Responding to RequestSchedule message from 10.1.230.36 with Ack because: PRJ byte array is null."},
-                    {"Team", "Manufacturing Software"},   
-                    {"RecordNumber", i},
-                    {"Host", hostName},                   
+                    {"Application", "udp-generator"},               
+                    {"Host", hostName},
                     {"UtcTimestamp", DateTime.UtcNow.ToString("o")},
-                    {"Type", "VP.Fulfillment.Direct.Initialization.LogWrapper"},                
+                    {"Type", "udp"},                
                     {"Message", "Testgenerator udp message " + DateTime.UtcNow.ToString("o")},
                     {"Index", "logstash"}
                 };
-
-                string hashedString = "";
-                foreach(var key in o)
-                {
-                    hashedString += key.ToString();
-                }
-
-                var source = ASCIIEncoding.ASCII.GetBytes(hashedString);
-                var md5 = new MD5CryptoServiceProvider().ComputeHash(source);
-                var hash = string.Concat(md5.Select(x => x.ToString("X2")));
-
-                o["md5"] = hash;
-
-                byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString(Formatting.None));
+                byte[] sendbuf = Encoding.UTF8.GetBytes(o.ToString());
                 IPEndPoint ep = new IPEndPoint(broadcast, parms.Port);
                 s.SendTo(sendbuf, ep);
-
-                if (i % 1000 == 0)
-                    LogManager.GetCurrentClassLogger().Info("Sent {0} of {1} messages", i, parms.NumMessages);
-
                 Thread.Sleep(parms.SleepTimeMilliseconds);
             }
 

TimberWinR.TestGenerator/default.json

@@ -36,7 +36,7 @@
                     "batch_count": 500,
                     "threads":  2,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/results5.json

@@ -1,13 +0,0 @@
-{
-    "Results": {
-        "Inputs": [           
-            {
-                "udp": {
-                    "test1: message sent count": "[messages] == 80000",
-                    "test2: average cpu": "[avgCpuUsage] <= 30",
-                    "test3: maximum memory": "[maxMemUsage] <= 30"              
-                }
-            }
-        ]
-    }
-}

TimberWinR.TestGenerator/test1-twconfig.json

@@ -39,9 +39,9 @@
                     "_comment": "Change the host to your Redis instance",
                     "port": 6379,
                     "batch_count": 500,
-                    "threads": 1,
+                    "threads": 2,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/test2-tw.json

@@ -36,7 +36,7 @@
                     "batch_count": 500,
                     "threads":  2,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/test3-tw.json

@@ -41,7 +41,7 @@
                     "batch_count": 500,
                     "threads": 2,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/test4-tw.json

@@ -41,7 +41,7 @@
                     "batch_count": 500,
                     "threads": 2,
                     "host": [
-                        "tstlexiceapp006.mycompany.svc"
+                        "tstlexiceapp006.vistaprint.svc"
                     ]
                 }
             ]

TimberWinR.TestGenerator/test5-twconfig.json

@@ -1,36 +0,0 @@
-{
-    "TimberWinR": {
-        "Inputs": {
-            "Udp": [
-                {
-                    "_comment": "Output from NLog",
-                    "port": 5140,
-                    "add_field": [
-                        "Environment",
-                        "PLANT_TST_TIMBERWINR"
-                    ],
-                    "rename": [
-                        "Type",
-                        "type"
-                    ]
-                }
-            ],
-            "TailFiles": [
-                {
-                    "interval": 5,
-                    "logSource": "log files",
-                    "location": "*.jlog",
-                    "recurse": -1
-                }
-            ]
-        },
-        "Outputs": {
-            "File": [
-                {
-                    "format":  "indented",
-                    "file_name": "test5_output.txt"
-                }
-            ]
-        }
-    }
-}

TimberWinR.TestGenerator/test5.json

@@ -1,15 +0,0 @@
-{
-    "test": "Test 5",
-    "arguments": {
-        "--start":  "",
-        "--testFile": "test5.json",
-        "--testDir": "test5",
-        "--timberWinRConfig": "test5-twconfig.json",
-        "--numMessages": 80000,
-        "--logLevel": "debug",
-        "--udp-host":  "localhost",
-        "--udp": "5140",  
-        "--udp-rate":  1,            
-        "--resultsFile": "results5.json"
-    }
-}

TimberWinR.UnitTests/JsonFilterTests.cs

@@ -77,56 +77,5 @@ namespace TimberWinR.UnitTests
             Assert.IsNull(nostuff);
             Assert.AreEqual(6, jsonInputLine2.Count);            
         }
-
-        [Test]
-        public void TestRenameAndAdds()
-        {
-            JObject jsonInputLine1 = new JObject
-            {               
-                {"type", "Win32-FileLog"},
-                {"ComputerName", "dev.mycompany.net"},
-                {"Text", "{\"log4net:Username\" : \"NT AUTHORITY\",\"Email\":\"james@example.com\",\"Active\":true,\"CreatedDate\":\"2013-01-20T00:00:00Z\",\"Roles\":[\"User\",\"Admin\"]}"}
-            };
-
-
-            string jsonFilter = @"{  
-            ""TimberWinR"":{        
-                ""Filters"":[  
-	                {  
-		            ""json"":{  
-		                ""type"": ""Win32-FileLog"",		               
-                        ""source"": ""Text"",
-                        ""promote"": ""Text"",
-                        ""add_field"":[  
-                            ""test1"",
-                            ""value1"",
-                            ""test2"",
-                            ""value2""
-                        ],       
-                        ""rename"":[  
-                            ""Active"",
-                            ""active"",
-                            ""log4net:Username"",
-                            ""lusername""
-                        ]          
-		            }
-	              }]
-                }
-            }";
-            
-
-            // Positive Tests
-            Configuration c = Configuration.FromString(jsonFilter);
-            Json jf = c.Filters.First() as Json;
-            Assert.IsTrue(jf.Apply(jsonInputLine1));
-
-            Assert.AreEqual("NT AUTHORITY", jsonInputLine1["lusername"].ToString());
-            Assert.AreEqual("True", jsonInputLine1["active"].ToString());
-            Assert.IsNotNull(jsonInputLine1["test1"]);
-            Assert.IsNotNull(jsonInputLine1["test2"]);
-            Assert.AreEqual("value1", jsonInputLine1["test1"].ToString());
-            Assert.AreEqual("value2", jsonInputLine1["test2"].ToString());
-        }
- 
     }
 }

TimberWinR.sln

@@ -116,7 +116,4 @@ Global
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
 	EndGlobalSection
-	GlobalSection(Performance) = preSolution
-		HasPerformanceSessions = true
-	EndGlobalSection
 EndGlobal

TimberWinR/Configuration.cs

@@ -58,12 +58,6 @@ namespace TimberWinR
             get { return _stdoutOutputs; }
         }
 
-        private List<FileOutputParameters> _fileOutputs = new List<FileOutputParameters>();
-        public IEnumerable<FileOutputParameters> FileOutputs
-        {
-            get { return _fileOutputs; }
-        }
-
         private List<TcpParameters> _tcps = new List<TcpParameters>();
         public IEnumerable<TcpParameters> Tcps
         {
@@ -271,8 +265,6 @@ namespace TimberWinR
                     c._elasticsearchOutputs.AddRange(x.TimberWinR.Outputs.Elasticsearch.ToList());
                 if (x.TimberWinR.Outputs.Stdout != null)
                     c._stdoutOutputs.AddRange(x.TimberWinR.Outputs.Stdout.ToList());
-                if (x.TimberWinR.Outputs.File != null)
-                    c._fileOutputs.AddRange(x.TimberWinR.Outputs.File.ToList());
             }
 
             if (x.TimberWinR.Filters != null)
@@ -310,7 +302,6 @@ namespace TimberWinR
             _redisOutputs = new List<RedisOutputParameters>();
             _elasticsearchOutputs = new List<ElasticsearchOutputParameters>();
             _stdoutOutputs = new List<StdoutOutputParameters>();
-            _fileOutputs = new List<FileOutputParameters>();
             _tcps = new List<TcpParameters>();
             _udps = new List<UdpParameters>();
         }

TimberWinR/Filters/GrokFilter.cs

@@ -30,6 +30,8 @@ namespace TimberWinR.Parser
         }
     }
 
+
+
     public partial class Grok : LogstashFilter
     {
         public override JObject ToJson()
@@ -90,37 +92,35 @@ namespace TimberWinR.Parser
             return true;
         }
 
-        // Test for any true matching condition(s)
         private bool Matches(Newtonsoft.Json.Linq.JObject json)
         {
-            for (int i = 0; i < Match.Length; i += 2)
-            {
-                string field = Match[i];
-                string expr = Match[i + 1];
+            string field = Match[0];
+            string expr = Match[1];
 
-                JToken token = null;
-                if (json.TryGetValue(field, out token))
+            JToken token = null;
+            if (json.TryGetValue(field, out token))
+            {
+                string text = token.ToString();
+                if (!string.IsNullOrEmpty(text))
                 {
-                    string text = token.ToString();
-                    if (!string.IsNullOrEmpty(text))
+                    var resolver = new RegexGrokResolver();
+                    var pattern = resolver.ResolveToRegex(expr);
+                    var match = Regex.Match(text, pattern);
+                    if (match.Success)
                     {
-                        var resolver = new RegexGrokResolver();
-                        var pattern = resolver.ResolveToRegex(expr);
-                        var match = Regex.Match(text, pattern);
-                        if (match.Success)
+                        var regex = new Regex(pattern);
+                        var namedCaptures = regex.MatchNamedCaptures(text);
+                        foreach (string fieldName in namedCaptures.Keys)
                         {
-                            var regex = new Regex(pattern);
-                            var namedCaptures = regex.MatchNamedCaptures(text);
-                            foreach (string fieldName in namedCaptures.Keys)
-                            {
-                                AddOrModify(json, fieldName, namedCaptures[fieldName]);
-                            }
-                            return true; // Yes!
+                            AddOrModify(json, fieldName, namedCaptures[fieldName]);
                         }
+                        return true; // Yes!
                     }
-                    if (string.IsNullOrEmpty(expr))
-                        return true; // Empty field is no match                 
                 }
+                if (string.IsNullOrEmpty(expr))
+                    return true; // Empty field is no match
+                else
+                    return false;
             }
             return false; // Not specified is failure
         }
@@ -136,7 +136,7 @@ namespace TimberWinR.Parser
                     AddOrModify(json, fieldName, fieldValue);
                 }
             }
-        }       
+        }
 
         private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
         {

TimberWinR/Filters/JsonFilter.cs

@@ -92,12 +92,9 @@ namespace TimberWinR.Parser
 
                     if (Rename != null && Rename.Length > 0)
                     {
-                        for (int i = 0; i < Rename.Length; i += 2)
-                        {
-                            string oldName = ExpandField(Rename[i], json);
-                            string newName = ExpandField(Rename[i+1], json);
-                            RenameProperty(json, oldName, newName);
-                        }
+                        string oldName = ExpandField(Rename[0], json);
+                        string newName = ExpandField(Rename[1], json);
+                        RenameProperty(json, oldName, newName);
                     }
 
                     if (RemoveSource)

TimberWinR/Inputs/InputListener.cs

@@ -44,11 +44,6 @@ namespace TimberWinR.Inputs
                              .ToString();    
         }
 
-        public void SetTypeName(string newType)
-        {
-            _typeName = newType;
-        }
-
         public bool HaveSeenFile(string fileName)
         {
             return Files.Contains(fileName);
@@ -101,34 +96,6 @@ namespace TimberWinR.Inputs
                 LogManager.GetCurrentClassLogger().Error(ex);
             }          
         }
-       
-        protected void AddOrModify(JObject json, string fieldName, string fieldValue)
-        {
-            if (json[fieldName] == null)
-                json.Add(fieldName, fieldValue);
-            else
-                json[fieldName] = fieldValue;
-        }
-
-        protected void RenameProperty(JObject json, string oldName, string newName)
-        {
-            JToken token = json[oldName];
-            if (token != null)
-            {
-                json.Add(newName, token.DeepClone());
-                json.Remove(oldName);
-            }
-        }
-
-        protected string ExpandField(string fieldName, JObject json)
-        {
-            foreach (var token in json.Children().ToList())
-            {
-                string replaceString = "%{" + token.Path + "}";
-                fieldName = fieldName.Replace(replaceString, json[token.Path].ToString());
-            }
-            return fieldName;
-        }
 
         protected void EnsureRollingCaught()
         {
@@ -156,7 +123,6 @@ namespace TimberWinR.Inputs
             }           
         }
 
-
         public virtual void AddDefaultFields(JObject json)
         {
             if (json["type"] == null)

TimberWinR/Inputs/LogsFileDatabase.cs

@@ -30,18 +30,10 @@ namespace TimberWinR.Inputs
 
         private static LogsFileDatabase instance;
 
-        private bool ExistingFile(string logName)
-        {
-            lock (_locker)
-            {
-                return ExistingFileTest(logName);
-            }
-        }
-       
         //
         // Lookup the database entry for this log file, returns null if there isnt one.
         //
-        private LogsFileDatabaseEntry FindFile(string logName)
+        private LogsFileDatabaseEntry FindFileWithLock(string logName)
         {
             lock (_locker)
             {
@@ -69,12 +61,12 @@ namespace TimberWinR.Inputs
             }
         }
 
-        private LogsFileDatabaseEntry AddFileEntry(string logName)
+        private LogsFileDatabaseEntry AddFileEntryWithLock(string logName)
         {
             var de = new LogsFileDatabaseEntry();
             lock (_locker)
             {
-                var fi = new FileInfo(logName);             
+                var fi = new FileInfo(logName);
                 de.FileName = logName;
                 de.LogFileExists = fi.Exists;
                 de.Previous = "";
@@ -82,21 +74,21 @@ namespace TimberWinR.Inputs
                 de.ProcessedFile = false;
                 de.LastPosition = fi.Length;
                 de.SampleTime = DateTime.UtcNow;
-                de.CreationTimeUtc = fi.CreationTimeUtc;               
+                de.CreationTimeUtc = fi.CreationTimeUtc;
 
                 Entries.Add(de);
                 WriteDatabaseFileNoLock();
             }
             return de;
-        }      
+        }
 
         public static LogsFileDatabaseEntry LookupLogFile(string logName)
         {
-            LogsFileDatabaseEntry dbe = Instance.FindFile(logName);
+            LogsFileDatabaseEntry dbe = Instance.FindFileWithLock(logName);
             if (dbe == null)
-                dbe = Instance.AddFileEntry(logName);
+                dbe = Instance.AddFileEntryWithLock(logName);
 
-            FileInfo fi = new FileInfo(logName);          
+            FileInfo fi = new FileInfo(logName);
 
             dbe.LogFileExists = fi.Exists;
             var creationTime = fi.CreationTimeUtc;
@@ -107,18 +99,18 @@ namespace TimberWinR.Inputs
                 dbe.Previous = "";
             }
             dbe.CreationTimeUtc = creationTime;
-      
+
             return dbe;
         }
 
         // Find all the non-existent entries and remove them.
-        private void PruneFiles()
+        private void PruneFilesWithLock()
         {
             lock (_locker)
             {
-                foreach(var entry in Entries.ToList())
+                foreach (var entry in Entries.ToList())
                 {
-                    FileInfo fi = new FileInfo(entry.FileName);
+                    var fi = new FileInfo(entry.FileName);
                     if (!fi.Exists)
                         Entries.Remove(entry);
                 }
@@ -130,7 +122,7 @@ namespace TimberWinR.Inputs
         {
             dbe.ProcessedFile = processedFile;
             dbe.LogFileExists = File.Exists(dbe.FileName);
-            Instance.UpdateEntry(dbe, lastOffset);           
+            Instance.UpdateEntryWithLock(dbe, lastOffset);
         }
 
         public static void Roll(LogsFileDatabaseEntry dbe)
@@ -138,20 +130,20 @@ namespace TimberWinR.Inputs
             dbe.ProcessedFile = false;
             dbe.LastPosition = 0;
             dbe.Previous = "";
-            Instance.UpdateEntry(dbe, 0);
-            dbe.NewFile = true;           
+            Instance.UpdateEntryWithLock(dbe, 0);
+            dbe.NewFile = true;
         }
 
-        private void UpdateEntry(LogsFileDatabaseEntry dbe, long lastOffset)
+        private void UpdateEntryWithLock(LogsFileDatabaseEntry dbe, long lastOffset)
         {
             lock (_locker)
             {
-                var fi = new FileInfo(dbe.FileName);                
+                var fi = new FileInfo(dbe.FileName);
                 dbe.NewFile = !fi.Exists;
                 dbe.CreationTimeUtc = fi.CreationTimeUtc;
-                dbe.SampleTime = DateTime.UtcNow;                 
+                dbe.SampleTime = DateTime.UtcNow;
                 dbe.LastPosition = lastOffset;
-              
+
                 WriteDatabaseFileNoLock();
             }
         }
@@ -175,7 +167,7 @@ namespace TimberWinR.Inputs
                         if (instance.Entries == null)
                             instance.Entries = new List<LogsFileDatabaseEntry>();
 
-                        instance.PruneFiles();
+                        instance.PruneFilesWithLock();
                     }
                 }
                 return instance;
@@ -195,18 +187,17 @@ namespace TimberWinR.Inputs
             }
             catch (Exception ex)
             {
-                LogManager.GetCurrentClassLogger()
-                    .Error("Error reading database '{0}': {1}", DatabaseFileName, ex.ToString());
+                LogManager.GetCurrentClassLogger().Error("Error reading database '{0}': {1}", DatabaseFileName, ex.ToString());
                 try
                 {
                     if (File.Exists(DatabaseFileName))
                         File.Delete(DatabaseFileName);
-                    LogManager.GetCurrentClassLogger().Info("Creating New Database '{0}'", DatabaseFileName);
+                    LogManager.GetCurrentClassLogger().Error("Creating New Database '{0}'", DatabaseFileName);
                     WriteDatabaseLock();
                 }
                 catch (Exception ex2)
                 {
-                    LogManager.GetCurrentClassLogger().Info("Error Creating New Database '{0}': {1}", DatabaseFileName, ex2.ToString());
+                    LogManager.GetCurrentClassLogger().Error("Error Creating New Database '{0}': {1}", DatabaseFileName, ex2.ToString());
                 }
             }
         }
@@ -258,7 +249,7 @@ namespace TimberWinR.Inputs
         public bool NewFile { get; set; }
         public bool ProcessedFile { get; set; }
         public bool LogFileExists { get; set; }
-        public string FileName { get; set; }       
+        public string FileName { get; set; }
         public DateTime CreationTimeUtc { get; set; }
         public DateTime SampleTime { get; set; }
         public long LastPosition { get; set; }

TimberWinR/Inputs/LogsListener.cs

@@ -53,9 +53,6 @@ namespace TimberWinR.Inputs
             if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                 _codec = new Multiline(_codecArguments);
 
-            if (!string.IsNullOrEmpty(arguments.Type))
-                SetTypeName(arguments.Type);
-
             _receivedMessages = 0;
             _arguments = arguments;
             _pollingIntervalInSeconds = arguments.Interval;

TimberWinR/Inputs/TailFileListener.cs

@@ -48,9 +48,6 @@ namespace TimberWinR.Inputs
             if (_codecArguments != null && _codecArguments.Type == CodecArguments.CodecType.multiline)
                 _codec = new Multiline(_codecArguments);
 
-            if (!string.IsNullOrEmpty(arguments.Type))
-                SetTypeName(arguments.Type);
-
             _receivedMessages = 0;
             _arguments = arguments;
             _pollingIntervalInSeconds = arguments.Interval;
@@ -113,7 +110,7 @@ namespace TimberWinR.Inputs
             const int bufSize = 16535;
             long prevLen = offset;
 
-            FileInfo fi = new FileInfo(fileName);
+            var fi = new FileInfo(fileName);
             if (!fi.Exists)
                 return;
 
@@ -123,9 +120,9 @@ namespace TimberWinR.Inputs
             {
                 stream.Seek(prevLen, SeekOrigin.Begin);
 
-                char[] buffer = new char[bufSize];
-                StringBuilder current = new StringBuilder();
-                using (StreamReader sr = new StreamReader(stream))
+                var buffer = new char[bufSize];
+                var current = new StringBuilder();
+                using (var sr = new StreamReader(stream))
                 {
                     int nRead;
                     do

TimberWinR/Inputs/TcpInputListener.cs

@@ -16,7 +16,7 @@ namespace TimberWinR.Inputs
         private Thread _listenThreadV4;
         private Thread _listenThreadV6;
         private readonly int _port;
-        private TimberWinR.Parser.TcpParameters _arguments;
+       
         private long _receivedMessages;
         private long _errorCount;
 
@@ -32,19 +32,15 @@ namespace TimberWinR.Inputs
             return json;
         }
 
-        public TcpInputListener(TimberWinR.Parser.TcpParameters arguments, CancellationToken cancelToken, int port = 5140)
+        public TcpInputListener(CancellationToken cancelToken, int port = 5140)
             : base(cancelToken, "Win32-Tcp")
         {
             _port = port;
-            _arguments = arguments;
 
             LogManager.GetCurrentClassLogger().Info("Tcp Input(v4/v6) on Port {0} Ready", _port);
 
-            if (!string.IsNullOrEmpty(arguments.Type))
-                SetTypeName(arguments.Type);
-
             _receivedMessages = 0;
-           
+
             _tcpListenerV6 = new System.Net.Sockets.TcpListener(IPAddress.IPv6Any, port);
             _tcpListenerV4 = new System.Net.Sockets.TcpListener(IPAddress.Any, port);
 
@@ -95,33 +91,6 @@ namespace TimberWinR.Inputs
             }
         }
 
-        //
-        // Renames, and AddFields
-        //
-        private void ApplyFilters(JObject json)
-        {
-            if (_arguments.Renames != null)
-            {
-                for (int i = 0; i < _arguments.Renames.Length; i += 2)
-                {
-                    var oldName = ExpandField(_arguments.Renames[i], json);
-                    var newName = ExpandField(_arguments.Renames[i + 1], json);
-                    RenameProperty(json, oldName, newName);
-                }
-            }
-
-            if (_arguments.AddFields != null)
-            {
-                for (int i = 0; i < _arguments.AddFields.Length; i += 2)
-                {
-                    var fieldName = ExpandField(_arguments.AddFields[i], json);
-                    var fieldValue = ExpandField(_arguments.AddFields[i + 1], json);
-                    AddOrModify(json, fieldName, fieldValue);
-                }
-            }
-        }
-
-
         private void HandleNewClient(object client)
         {
             var tcpClient = (TcpClient)client;
@@ -140,7 +109,6 @@ namespace TimberWinR.Inputs
                             try
                             {
                                 JObject json = JObject.Load(reader);
-                                ApplyFilters(json);
                                 ProcessJson(json);
                                 Interlocked.Increment(ref _receivedMessages);
                             }

TimberWinR/Inputs/UdpInputListener.cs

@@ -20,7 +20,6 @@ namespace TimberWinR.Inputs
         private long _parseErrors;
         private long _receiveErrors;
         private long _parsedMessages;
-        private TimberWinR.Parser.UdpParameters _arguments;
 
         public override JObject ToJson()
         {
@@ -36,14 +35,10 @@ namespace TimberWinR.Inputs
             return json;
         }
 
-        public UdpInputListener(TimberWinR.Parser.UdpParameters arguments, CancellationToken cancelToken, int port = 5140) : base(cancelToken, "Win32-Udp")
+        public UdpInputListener(CancellationToken cancelToken, int port = 5140) : base(cancelToken, "Win32-Udp")
         {
             _port = port;
             _receivedMessages = 0;
-            _arguments = arguments;
-
-            if (!string.IsNullOrEmpty(arguments.Type))
-                SetTypeName(arguments.Type);
 
             // setup raw data processor
             _unprocessedRawData = new BlockingCollection<byte[]>();
@@ -146,32 +141,6 @@ namespace TimberWinR.Inputs
             Finished();
         }
 
-        //
-        // Renames, and AddFields
-        //
-        private void ApplyFilters(JObject json)
-        {      
-            if (_arguments.Renames != null)
-            {
-                for (int i=0; i<_arguments.Renames.Length; i += 2)
-                {
-                    var oldName = ExpandField(_arguments.Renames[i], json);
-                    var newName = ExpandField(_arguments.Renames[i + 1], json);
-                    RenameProperty(json, oldName, newName);
-                }
-            }
-
-            if (_arguments.AddFields != null)
-            {
-                for (int i = 0; i < _arguments.AddFields.Length; i += 2)
-                {
-                    var fieldName = ExpandField(_arguments.AddFields[i], json);
-                    var fieldValue = ExpandField(_arguments.AddFields[i + 1], json);
-                    AddOrModify(json, fieldName, fieldValue);
-                }
-            }
-        }
-
         private void ProcessData(byte[] bytes)
         {
             var data = Encoding.UTF8.GetString(bytes, 0, bytes.Length);
@@ -179,8 +148,8 @@ namespace TimberWinR.Inputs
             try
             {
                 var json = JObject.Parse(data);
-                ApplyFilters(json);
                 ProcessJson(json);
+
                 _parsedMessages++;
             }
             catch (Exception ex)

TimberWinR/Manager.cs

@@ -190,15 +190,6 @@ namespace TimberWinR
                     }
                 }
 
-                if (config.FileOutputs != null)
-                {
-                    foreach (var ro in config.FileOutputs)
-                    {
-                        var output = new FileOutput(this, ro, cancelToken);
-                        Outputs.Add(output);
-                    }
-                }
-
                 foreach (Parser.IISW3CLogParameters iisw3cConfig in config.IISW3C)
                 {
                     var elistner = new IISW3CInputListener(iisw3cConfig, cancelToken);
@@ -241,7 +232,7 @@ namespace TimberWinR
 
                 foreach (var tcp in config.Tcps)
                 {
-                    var elistner = new TcpInputListener(tcp, cancelToken, tcp.Port);
+                    var elistner = new TcpInputListener(cancelToken, tcp.Port);
                     Listeners.Add(elistner);
                     foreach (var output in Outputs)
                         output.Connect(elistner);
@@ -249,7 +240,7 @@ namespace TimberWinR
 
                 foreach (var udp in config.Udps)
                 {
-                    var elistner = new UdpInputListener(udp, cancelToken, udp.Port);
+                    var elistner = new UdpInputListener(cancelToken, udp.Port);
                     Listeners.Add(elistner);
                     foreach (var output in Outputs)
                         output.Connect(elistner);

TimberWinR/Outputs/File.cs

@@ -1,140 +0,0 @@
-using System.IO;
-using System.Text;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using NLog;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-
-namespace TimberWinR.Outputs
-{
-    public class FileOutput : OutputSender
-    {
-        private TimberWinR.Manager _manager;
-        private readonly int _interval;
-        private readonly object _locker = new object();
-        private readonly List<JObject> _jsonQueue;
-        private long _sentMessages;     
-        private Parser.FileOutputParameters _arguments;
-        public bool Stop { get; set; }
-
-        public FileOutput(TimberWinR.Manager manager, Parser.FileOutputParameters arguments, CancellationToken cancelToken)
-            : base(cancelToken, "File")
-        {
-            _arguments = arguments;
-            _sentMessages = 0;
-            _manager = manager;
-            _interval = arguments.Interval;
-            _jsonQueue = new List<JObject>();
-
-            var elsThread = new Task(FileSender, cancelToken);
-            elsThread.Start();
-        }
-
-        public override JObject ToJson()
-        {
-            JObject json = new JObject(
-                new JProperty("file-output",
-                    new JObject(
-                        new JProperty("queuedMessageCount", _jsonQueue.Count),
-                        new JProperty("sentMessageCount", _sentMessages))));
-
-            return json;
-        }
-
-        // 
-        // Pull off messages from the Queue, batch them up and send them all across
-        // 
-        private void FileSender()
-        {
-           
-            using (var syncHandle = new ManualResetEventSlim())
-            {
-                var fi = new FileInfo(_arguments.FileName);
-                if (File.Exists(_arguments.FileName))
-                    File.Delete(_arguments.FileName);
-
-                LogManager.GetCurrentClassLogger().Info("File Output Sending To: {0}", fi.FullName);
-       
-                using (StreamWriter sw = File.AppendText(_arguments.FileName))
-                {
-                    // Execute the query
-                    while (!Stop)
-                    {
-                        if (!CancelToken.IsCancellationRequested)
-                        {
-                            try
-                            {
-                                JObject[] messages;
-                                lock (_locker)
-                                {
-                                    messages = _jsonQueue.Take(_jsonQueue.Count).ToArray();
-                                    _jsonQueue.RemoveRange(0, messages.Length);
-                                }
-
-                                if (messages.Length > 0)
-                                {
-                                    try
-                                    {
-                                        foreach (JObject obj in messages)
-                                        {                                           
-                                            sw.WriteLine(obj.ToString(_arguments.ToFormat()));                                        
-                                            _sentMessages++;
-                                        }
-                                    }
-                                    catch (Exception ex)
-                                    {
-                                        LogManager.GetCurrentClassLogger().Error(ex);
-                                    }
-                                }
-                                if (!Stop)
-                                    syncHandle.Wait(TimeSpan.FromMilliseconds(_interval), CancelToken);
-                            }
-                            catch (OperationCanceledException)
-                            {
-                                break;
-                            }
-                            catch (Exception)
-                            {
-                            }
-                        }
-                    }
-                }
-            }
-        }
-
-        protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage)
-        {
-            if (_manager.Config.Filters != null)
-            {
-                if (ApplyFilters(jsonMessage))
-                    return;
-            }
-
-            var message = jsonMessage.ToString();
-   
-            lock (_locker)
-            {
-                _jsonQueue.Add(jsonMessage);
-            }
-        }
-
-        private bool ApplyFilters(JObject json)
-        {
-            bool drop = false;
-
-            foreach (var filter in _manager.Config.Filters)
-            {
-                if (!filter.Apply(json))
-                    drop = true;
-            }
-
-            return drop;
-        }
-
-    }
-}
-

TimberWinR/Outputs/Redis.cs

@@ -79,11 +79,9 @@ namespace TimberWinR.Outputs
         // Sample the queue and adjust the batch count if needed (ramp up slowly)
         public int UpdateCurrentBatchCount(int queueSize, int currentBatchCount)
         {
-            var avgQueueDepth = AverageQueueDepth();
-        
-            if (currentBatchCount < _maxBatchCount && currentBatchCount < queueSize && avgQueueDepth > currentBatchCount)
+            if (currentBatchCount < _maxBatchCount && currentBatchCount < queueSize && AverageQueueDepth() > currentBatchCount)
             {
-                currentBatchCount += Math.Max(avgQueueDepth / _batchCount, _batchCount / 5);
+                currentBatchCount += Math.Max(_maxBatchCount / _batchCount, 1);
                 if (currentBatchCount >= _maxBatchCount && !_warnedReachedMax)
                 {
                     LogManager.GetCurrentClassLogger().Warn("Maximum Batch Count of {0} reached.", currentBatchCount);
@@ -316,9 +314,6 @@ namespace TimberWinR.Outputs
                             {
                                 _batchCounter.SampleQueueDepth(_jsonQueue.Count);
                                 // Re-compute current batch size
-
-                                LogManager.GetCurrentClassLogger().Trace("{0}: Average Queue Depth: {1}, Current Length: {2}", Thread.CurrentThread.ManagedThreadId, _batchCounter.AverageQueueDepth(), _jsonQueue.Count); 
-       
                                 _currentBatchCount = _batchCounter.UpdateCurrentBatchCount(_jsonQueue.Count, _currentBatchCount);
 
                                 messages = _jsonQueue.Take(_currentBatchCount).ToArray();

TimberWinR/Parser.cs

@@ -12,7 +12,6 @@ using Microsoft.SqlServer.Server;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
-using NLog.Config;
 using TimberWinR.Outputs;
 using System.CodeDom.Compiler;
 
@@ -44,8 +43,10 @@ namespace TimberWinR.Parser
             JToken token = json[oldName];
             if (token != null)
             {
-                json.Add(newName, token.DeepClone());
                 json.Remove(oldName);
+                JToken newToken = json[newName];
+                if (newToken == null)
+                    json.Add(newName, token);
             }
         }
 
@@ -268,7 +269,7 @@ namespace TimberWinR.Parser
         public string Type { get; set; }
 
         [JsonProperty(PropertyName = "codec")]
-        public CodecArguments CodecArguments { get; set; }
+        public CodecArguments CodecArguments { get; set; }       
 
         [JsonProperty(PropertyName = "message")]
         public string Message { get; set; }
@@ -280,7 +281,7 @@ namespace TimberWinR.Parser
         public int Rate { get; set; }
 
         public void Validate()
-        {
+        {                      
         }
 
         public GeneratorParameters()
@@ -332,12 +333,10 @@ namespace TimberWinR.Parser
 
     public class TailFileArguments : IValidateSchema
     {
-        [JsonProperty(PropertyName = "type")]
-        public string Type { get; set; }
         [JsonProperty(PropertyName = "location")]
-        public string Location { get; set; }
+        public string Location { get; set; }       
         [JsonProperty(PropertyName = "recurse")]
-        public int Recurse { get; set; }
+        public int Recurse { get; set; }        
         [JsonProperty(PropertyName = "fields")]
         public List<Field> Fields { get; set; }
         [JsonProperty(PropertyName = "interval")]
@@ -364,8 +363,6 @@ namespace TimberWinR.Parser
 
     public class LogParameters : IValidateSchema
     {
-        [JsonProperty(PropertyName = "type")]
-        public string Type { get; set; }
         [JsonProperty(PropertyName = "location")]
         public string Location { get; set; }
         [JsonProperty(PropertyName = "iCodepage")]
@@ -402,21 +399,15 @@ namespace TimberWinR.Parser
     {
         [JsonProperty(PropertyName = "port")]
         public int Port { get; set; }
-        [JsonProperty(PropertyName = "type")]
-        public string Type { get; set; }
-        [JsonProperty("add_field")]
-        public string[] AddFields { get; set; }
-        [JsonProperty("rename")]
-        public string[] Renames { get; set; }
 
         public TcpParameters()
         {
             Port = 5140;
-            Type = "Win32-Tcp";
         }
 
         public void Validate()
         {
+
         }
     }
 
@@ -425,21 +416,15 @@ namespace TimberWinR.Parser
     {
         [JsonProperty(PropertyName = "port")]
         public int Port { get; set; }
-        [JsonProperty(PropertyName = "type")]
-        public string Type { get; set; }
-        [JsonProperty("add_field")]
-        public string[] AddFields { get; set; }
-        [JsonProperty("rename")]
-        public string[] Renames { get; set; }
 
         public UdpParameters()
         {
             Port = 5142;
-            Type = "Win32-Udp";
         }
 
         public void Validate()
         {
+
         }
     }
     public class W3CLogParameters : IValidateSchema
@@ -568,9 +553,9 @@ namespace TimberWinR.Parser
         [JsonProperty(PropertyName = "max_queue_size")]
         public int MaxQueueSize { get; set; }
         [JsonProperty(PropertyName = "queue_overflow_discard_oldest")]
-        public bool QueueOverflowDiscardOldest { get; set; }
+        public bool QueueOverflowDiscardOldest { get; set; }        
         [JsonProperty(PropertyName = "enable_ping")]
-        public bool EnablePing { get; set; }
+        public bool EnablePing { get; set; }      
         [JsonProperty(PropertyName = "ping_timeout")]
         public int PingTimeout { get; set; }
 
@@ -661,6 +646,7 @@ namespace TimberWinR.Parser
             Host = new string[] { "localhost" };
             Timeout = 10000;
             BatchCount = 200;
+            MaxBatchCount = BatchCount*10;
             NumThreads = 1;
             Interval = 5000;
             QueueOverflowDiscardOldest = true;
@@ -679,43 +665,6 @@ namespace TimberWinR.Parser
         }
     }
 
-    public class FileOutputParameters
-    {
-        public enum FormatKind
-        {
-            none, indented
-        };
-
-        [JsonProperty(PropertyName = "interval")]
-        public int Interval { get; set; }
-
-        [JsonProperty(PropertyName = "file_name")]
-        public string FileName { get; set; }
-
-        [JsonProperty(PropertyName = "format")]
-        public FormatKind Format { get; set; }
-
-        public FileOutputParameters()
-        {
-            Format = FormatKind.none;            
-            Interval = 1000;
-            FileName = "timberwinr.out";
-        }
-
-        public Newtonsoft.Json.Formatting ToFormat()
-        {
-            switch (Format)
-            {
-                case FormatKind.indented:
-                    return Newtonsoft.Json.Formatting.Indented;
-
-                case FormatKind.none:
-                default:
-                    return Newtonsoft.Json.Formatting.None;
-            }
-        }
-    }
-
     public class OutputTargets
     {
         [JsonProperty("Redis")]
@@ -726,9 +675,6 @@ namespace TimberWinR.Parser
 
         [JsonProperty("Stdout")]
         public StdoutOutputParameters[] Stdout { get; set; }
-
-        [JsonProperty("File")]
-        public FileOutputParameters[] File { get; set; }
     }
 
     public class InputSources
@@ -805,7 +751,7 @@ namespace TimberWinR.Parser
 
         public override void Validate()
         {
-            if (Match == null || Match.Length % 2 != 0)
+            if (Match == null || Match.Length != 2)
                 throw new GrokFilterException();
 
             if (AddTag != null && AddTag.Length % 2 != 0)
@@ -946,7 +892,7 @@ namespace TimberWinR.Parser
         }
     }
 
-
+   
     public partial class Json : LogstashFilter
     {
         public class JsonMissingSourceException : Exception
@@ -1030,7 +976,7 @@ namespace TimberWinR.Parser
 
         [JsonProperty("grokFilters")]
         public Grok[] Groks { get; set; }
-
+      
         [JsonProperty("mutateFilters")]
         public Mutate[] Mutates { get; set; }
 
@@ -1041,7 +987,7 @@ namespace TimberWinR.Parser
         public Json[] Jsons { get; set; }
 
         [JsonProperty("geoipFilters")]
-        public GeoIP[] GeoIPs { get; set; }
+        public GeoIP[] GeoIPs { get; set; }    
     }
 
     public class TimberWinR

TimberWinR/ReleaseNotes.md

@@ -3,13 +3,6 @@
 A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
 
 Version / Date
-### 1.3.25.0 - 2015-04-30
-1. Fixed Issue [#49](https://github.com/Cimpress-MCP/TimberWinR/issues/49)
-2. Fixed potential non-thread safe when renaming properties
-3. Added add_field, rename support to Udp/Tcp Input Listeners
-4. Fixed issue with multiple renames (was previously only renaming the first one)
-5. Added File outputter for testing.
-
 ### 1.3.24.0 - 2015-04-29
 1. Fixed potential bug in TailFiles when tailing log files which are partially flushed
    to disk, it now will not process the line until the \r\n has been seen.

TimberWinR/TimberWinR.csproj

@@ -115,7 +115,6 @@
     <Compile Include="Outputs\Elasticsearch.cs" />
     <Compile Include="Outputs\OutputSender.cs" />
     <Compile Include="Outputs\Redis.cs" />
-    <Compile Include="Outputs\File.cs" />
     <Compile Include="Outputs\Stdout.cs" />
     <Compile Include="Parser.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
@@ -142,7 +141,6 @@
     <None Include="mdocs\Filters.md" />
     <None Include="mdocs\GeoIPFilter.md" />
     <None Include="mdocs\Generator.md" />
-    <None Include="mdocs\FileOutput.md" />
     <None Include="mdocs\TailFiles.md" />
     <None Include="mdocs\UdpInput.md" />
     <None Include="mdocs\W3CInput.md" />

TimberWinR/mdocs/FileOutput.md

@@ -1,27 +0,0 @@
-# Output: File
-
-The File output passes on data into a text file.
-
-## Parameters
-The following parameters are allowed when configuring the File output.
-
-| Parameter     |   Type   |  Description                                                | Details               |  Default |
-| :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
-| *interval*    | integer  | Interval in milliseconds to sleep before appending data       | Interval      | 1000 |
-| *file_name*   | string   | Name of the file to be created                              |               | timberwinr.out |
-
-Example Input:
-```json
-{
-    "TimberWinR": {
-        "Outputs": {
-            "File": [
-               { 
-                    "file_name":  "foo.out",   
-                    "interval": 1000                   
-                }
-            ]
-		}
-	}
-}
-```

TimberWinR/mdocs/GrokFilter.md

@@ -26,14 +26,14 @@ The following operations are allowed when mutating a field.
 
 | Operation       |     Type        | Description                                                            
 | :---------------|:----------------|:-----------------------------------------------------------------------|
-| *add_field*     | property:array  |If the filter is successful, add an arbitrary field to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.                                    
-| *add_tag*       | property:array  |If the filter is successful, add an arbitrary tag to this event.  Tag names can be dynamic and include parts of the event using the %{field} syntax.                                  
-| *condition*     | property:string |C# expression
-| *match*         | property:array  |Required field must match (any) before any subsequent grok operations are executed.
-| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
-| *remove_tag*    | property:array  |If the filter is successful, remove arbitrary tags from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                          
-| *rename*        | property:array  |Rename one or more fields                                       
 | *type*          | property:string |Type to which this filter applyes, if empty, applies to all types.
+| *condition*     | property:string |C# expression
+| *rename*        | property:array  |Rename one or more fields                                       
+| *match*         | property:string |Required field must match before any subsequent grok operations are executed.
+| *add_field*     | property:array  |If the filter is successful, add an arbitrary field to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.                                    
+| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
+| *add_tag*       | property:array  |If the filter is successful, add an arbitrary tag to this event.  Tag names can be dynamic and include parts of the event using the %{field} syntax.                                  
+| *remove_tag*    | property:array  |If the filter is successful, remove arbitrary tags from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                          
 
 ## Operation Details
 ### match 
@@ -67,28 +67,6 @@ Given this configuration
     }     
   ]
 ```
-
-Given this configuration
-```json
-  "Filters": [     
-    {
-	   "grok": {
-           "matches": [
-               "message",
-               "%{IP:client} %{WORD:method} %{URIPATHPARAM:request} %{NUMBER:bytes} %{NUMBER:duration}"
-           ],
-           "add_tag": [               
-               "http_log"
-           ],
-           "add_field": [
-               "verb", "%{method}"
-           ]
-        }           
-    }     
-  ]
-```
-
-
 And if the message matches, then 6 fields would be added to the event:
  1. client=55.3.244.1
  2. method=GET

TimberWinR/mdocs/JsonFilter.md

@@ -7,27 +7,25 @@ The following operations are allowed when mutating a field.
 
 | Operation       |     Type        | Description                                                            
 | :---------------|:----------------|:-----------------------------------------------------------------------|
-| *add_field*     | property:array  |If the filter is successful, add an arbitrary field to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.                                    
-| *add_tag*       | property:array  |If the filter is successful, add an arbitrary tag to this event.  Tag names can be dynamic and include parts of the event using the %{field} syntax.                                  
-| *condition*     | property:string |C# expression, if the expression is true, continue, otherwise, ignore
-| *promote*       | property:string |If supplied any properties named *promote* will be promoted to top-level
-| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
-| *remove_tag*    | property:array  |If the filter is successful, remove arbitrary tags from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                          
-| *remove_source* | property:bool   |If true, the source property is removed, default: true
-| *rename*        | property:array  |Rename one or more fields                                       
-| *source*        | property:string |Required field indicates which field contains the Json to be parsed
-| *target*        | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
 | *type*          | property:string |Type to which this filter applies, if empty, applies to all types.
+| *condition*     | property:string |C# expression, if the expression is true, continue, otherwise, ignore
+| *remove_source* | property:bool   |If true, the source property is removed, default: true
+| *source*        | property:string |Required field indicates which field contains the Json to be parsed
+| *promote*       | property:string |If supplied any properties named *promote* will be promoted to top-level
+| *target*        | property:string |If suppled, the parsed json will be contained underneath a propery named *target*
+| *add_field*     | property:array  |If the filter is successful, add an arbitrary field to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.                                    
+| *remove_field*  | property:array  |If the filter is successful, remove arbitrary fields from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                                
+| *add_tag*       | property:array  |If the filter is successful, add an arbitrary tag to this event.  Tag names can be dynamic and include parts of the event using the %{field} syntax.                                  
+| *remove_tag*    | property:array  |If the filter is successful, remove arbitrary tags from this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.                          
 
 ## Operation Details
 ### source 
-The source field is required, and indicates what Field contains the target Json, In the
-below example, the [Logs](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) input produces a Text field, 
-which contains a line to be parsed as Json.
+The match field is required, the first argument is the field to inspect, and compare to the expression specified by the second
+argument.  In the below example, the message is spected to be something like this from a fictional sample log:
 
 Given this input configuration:
 
-Lets assume that a new line such as the following is appended to foo.jlog:, this would end up being the Text field
+Lets assume that a newline such as the following is appended to foo.jlog:
 ```
    {"Email":"james@example.com","Active":true,"CreatedDate":"2013-01-20T00:00:00Z","Roles":["User","Admin"]}
 ```
@@ -56,7 +54,7 @@ Lets assume that a new line such as the following is appended to foo.jlog:, this
         }       
 ```
 
-In the above example, the file foo.jlog is being tailed, and when a new line is appended, it is assumed
+In the above example, the file foo.jlog is being tailed, and when a newline is appended, it is assumed
 to be Json and is parsed from the Text field, the parsed Json is then inserted underneath a property *stuff*
 
 The resulting output would be:
@@ -86,8 +84,8 @@ The fields must be in pairs with oldname first and newname second.
         "target": "stuff",
         "source": "Text",
         "rename": [
-            "level",
-            "Level"
+            "Text",
+            "Data"
         ]                   
      } 
    }     
@@ -98,9 +96,7 @@ The fields must be in pairs with fieldName first and value second.
 ```json
   "Filters": [     
     {
-		"json": {  
-          "type":  "Win32-FileLog",       
-          "source": "Text",   			
+		"json": {      			
 			"add_field": [
               "ComputerName", "Host",
               "Username", "%{SID}"				         
@@ -116,9 +112,7 @@ Remove the fields.  More than one field can be specified at a time.
   "Filters": [     
     {
 		"json": {      			
-          "type":  "Win32-FileLog",       
-          "source": "Text",   			
-			"remove_field": [             
+			"remove_tag": [             
              "static_tag1",
              "Computer_%{Host}"
 			]
@@ -134,8 +128,6 @@ Adds the tag(s) to the tag array.
   "Filters": [     
     {
 		"json": {      			
-          "type":  "Win32-FileLog",       
-          "source": "Text",   			
 			"add_tag": [
                "foo_%{Host}",
 			   "static_tag1"      
@@ -151,8 +143,6 @@ Remove the tag(s) to the tag array.  More than one tag can be specified at a tim
   "Filters": [     
     {
 		"json": {      			
-          "type":  "Win32-FileLog",       
-          "source": "Text",   			
 			"remove_tag": [             
              "static_tag1",
              "Username"

TimberWinR/mdocs/Logs.md

@@ -7,12 +7,11 @@ The following parameters are allowed when configuring WindowsEvents.
 
 | Parameter         |     Type       |  Description                                                             | Details               |  Default |
 | :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
-| *iCodepage*       | integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
 | *location*        | string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
 | *logSource*       | string  |Source name                                  | Used for conditions |     |
 | *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
 | *splitLongLines*  | boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false |
-| *type*            | string  |Typename for this Input                                         |  |  Win32-FileLog  |
+| *iCodepage*       | integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
 | [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
 
 Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern.  I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
@@ -40,4 +39,3 @@ After a successful parse of an event, the following fields are added:
 | LogFilename | STRING |Full path of the file containing this line | 
 | Index | INTEGER | Line number |
 | Text | STRING | Text line content  |
-| type | STRING | Win32-FileLog |

TimberWinR/mdocs/RedisOutput.md

@@ -7,15 +7,15 @@ The following parameters are allowed when configuring the Redis output.
 
 | Parameter     |   Type   |  Description                                                | Details               |  Default |
 | :-------------|:---------|:------------------------------------------------------------| :---------------------------  | :-- |
-| *batch_count* | integer  | Sent as a single message                         | Number of messages to aggregate | 200 |
-| *host*        | string | The hostname(s) of your Redis server(s) | IP or DNS name |  |
-| *index*       | string   | The name of the redis list                                  | logstash index name | logstash |
-| *interval*    | integer  | Interval in milliseconds to sleep during batch sends        | Interval      | 5000 |
-| *max_batch_count* | integer  | Dynamically adjusted count maximum                      | Increases over time | batch_count * 10 |
-| *max_queue_size* | integer  | Maximum redis queue depth       |  | 50000 |
-| *port*        | integer  | Redis port number                                           | This port must be open  | 6379  |
-| *queue_overflow_discard_oldest* | bool  | If true, discard oldest messages when max_queue_size reached otherwise discard newest |  | true |
 | *threads*     | string   | Location of log files(s) to monitor                         | Number of worker theads to send messages | 1 |
+| *batch_count* | integer  | Sent as a single message                         | Number of messages to aggregate | 200 |
+| *max_batch_count* | integer  | Dynamically adjusted count maximum                      | Increases over time | batch_count*10 |
+| *interval*    | integer  | Interval in milliseconds to sleep during batch sends        | Interval      | 5000 |
+| *index*       | string   | The name of the redis list                                  | logstash index name | logstash |
+| *host*        | [string] | The hostname(s) of your Redis server(s) | IP or DNS name |  |
+| *port*        | integer  | Redis port number                                           | This port must be open  | 6379  |
+| *max_queue_size* | integer  | Maximum redis queue depth       |  | 50000 |
+| *queue_overflow_discard_oldest* | bool  | If true, discard oldest messages when max_queue_size reached otherwise discard newest |  | true |
 
 Example Input:
 ```json

TimberWinR/mdocs/TailFiles.md

@@ -8,7 +8,6 @@ The following parameters are allowed when configuring WindowsEvents.
 
 | Parameter         |     Type       |  Description                                                             | Details               |  Default |
 | :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
-| *type*            | string  |Typename for this Input                                         |  |  Win32-TailLog  |
 | *location*        | string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
 | *logSource*       | string  |Source name                                  | Used for conditions |     |
 | *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
@@ -40,4 +39,3 @@ After a successful parse of an event, the following fields are added:
 | LogFilename | STRING |Full path of the file containing this line | 
 | Index | INTEGER | Line number |
 | Text | STRING | Text line content  |
-| type | STRING | Win32-TailLog |

TimberWinR/mdocs/TcpInput.md

@@ -5,12 +5,9 @@ The Tcp input will open a port and listen for properly formatted JSON and will f
 ## Parameters
 The following parameters are allowed when configuring the Tcp input.
 
-| Parameter         |     Type         |  Description                                                             | Details               |  Default |
-| :---------------- |:-----------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
-| *add_field*       | property:array   |Add field(s) to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.  |     |
-| *port*            | integer          |Port number to open        | Must be an available port |     |
-| *rename*          | property:array   |Rename one or more fields  |  |  |                             
-| *type*            | string           |Typename for this Input    |  |  Win32-Tcp  |
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *port*        | integer  |Port number to open        | Must be an available port |     |
 
 Example Input: Listen on Port 5140
 

TimberWinR/mdocs/UdpInput.md

@@ -7,10 +7,7 @@ The following parameters are allowed when configuring the Udp input.
 
 | Parameter         |     Type       |  Description                                                             | Details               |  Default |
 | :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
-| *add_field*       | property:array   |Add field(s) to this event.  Field names can be dynamic and include parts of the event using the %{field} syntax.  This property must be specified in pairs.  |     |
-| *port*            | integer  |Port number to open        | Must be an available port |     |
-| *rename*          | property:array   |Rename one or more fields  |  |  |                             
-| *type*            | string   |Typename for this Input                                         |  |  Win32-Udp  |
+| *port*        | integer  |Port number to open        | Must be an available port |     |
 
 Example Input: Listen on Port 5142