From dec51efccdeda48545753d8a6763338a9245ddeb Mon Sep 17 00:00:00 2001
From: Eric Fontana <efontana@vistaprint.com>
Date: Mon, 12 Jan 2015 12:35:20 -0500
Subject: [PATCH] Added Multiline codec closed issue #23

---
 .../Properties/AssemblyInfo.cs                |   4 +-
 TimberWinR.UnitTests/Multiline1.txt           |  13 +
 TimberWinR.UnitTests/Multiline2.txt           |  19 ++
 TimberWinR.UnitTests/MultilineTests.cs        | 119 +++++++++
 .../TimberWinR.UnitTests.csproj               |   9 +
 TimberWinR/Inputs/InputListener.cs            |   2 +-
 TimberWinR/Inputs/LogsListener.cs             | 249 +++++++++++++++++-
 TimberWinR/Inputs/StdinListener.cs            | 115 +++++++-
 TimberWinR/Manager.cs                         |   9 +-
 TimberWinR/Outputs/Redis.cs                   |   5 +-
 TimberWinR/Outputs/Stdout.cs                  |  14 +-
 TimberWinR/Parser.cs                          |  63 ++++-
 TimberWinR/ReleaseNotes.md                    |   4 +
 TimberWinR/TimberWinR.csproj                  |   1 +
 TimberWinR/mdocs/Codec.md                     |  31 +++
 TimberWinR/mdocs/Logs.md                      |   1 +
 TimberWinR/mdocs/StdinInput.md                |   1 +
 17 files changed, 623 insertions(+), 36 deletions(-)
 create mode 100644 TimberWinR.UnitTests/Multiline1.txt
 create mode 100644 TimberWinR.UnitTests/Multiline2.txt
 create mode 100644 TimberWinR.UnitTests/MultilineTests.cs
 create mode 100644 TimberWinR/mdocs/Codec.md

diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
index eec81fc..ee7828b 100644
--- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
+++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
 // You can specify all the values or you can default the Build and Revision Numbers 
 // by using the '*' as shown below:
 // [assembly: AssemblyVersion("1.0.*")]
-[assembly: AssemblyVersion("1.3.18.0")]
-[assembly: AssemblyFileVersion("1.3.18.0")]
+[assembly: AssemblyVersion("1.3.19.0")]
+[assembly: AssemblyFileVersion("1.3.19.0")]
diff --git a/TimberWinR.UnitTests/Multiline1.txt b/TimberWinR.UnitTests/Multiline1.txt
new file mode 100644
index 0000000..dd38336
--- /dev/null
+++ b/TimberWinR.UnitTests/Multiline1.txt
@@ -0,0 +1,13 @@
+multiline1 \
+ml1_1 \
+ml1_2 \
+ml1_2 
+singleline1
+singleline2
+multiline2 \
+ml2_1 \
+ml2_2
+multiline3 \
+ml3_1 \
+ml3_2
+singleline3
\ No newline at end of file
diff --git a/TimberWinR.UnitTests/Multiline2.txt b/TimberWinR.UnitTests/Multiline2.txt
new file mode 100644
index 0000000..449f387
--- /dev/null
+++ b/TimberWinR.UnitTests/Multiline2.txt
@@ -0,0 +1,19 @@
+2015-01-07 13:14:26,572 TEST DEBUG [THREAD : 25] - Sending message to TServer - tcp://10.1111.11.111:1111
+'RequestAttachUserData' ('30')
+message attributes:
+AttributeConnID [long] = 00890
+AttributeReferenceID [int] = 88
+AttributeThisDN [str] = "2214"
+AttributeUserData [bstr] = KVList: 
+		'ActivityID' [str] = "1-XXXXXX"
+2015-01-07 13:14:26,574 TEST DEBUG [THREAD : 25] - Writing message RequestAttachUserData in 'proxy1' via '.StatePrimary proxy: proxy1'
+2015-01-07 13:14:26,575 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection
+2015-01-07 13:20:31,665 TEST DEBUG [THREAD : SelectorThread] - Proxy got message 'EventOnHook' ('87')
+message attributes:
+AttributeEventSequenceNumber [long] = 4899493
+Time            = ComplexClass(TimeStamp):
+	AttributeTimeinuSecs [int] = 573000
+	AttributeTimeinSecs [int] = 1420644031
+AttributeThisDN [str] = "2214"
+. Processing with  state .StatePrimary proxy: proxy1
+2015-01-07 14:14:26,666 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection
diff --git a/TimberWinR.UnitTests/MultilineTests.cs b/TimberWinR.UnitTests/MultilineTests.cs
new file mode 100644
index 0000000..229d948
--- /dev/null
+++ b/TimberWinR.UnitTests/MultilineTests.cs
@@ -0,0 +1,119 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Text;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using Newtonsoft.Json;
+using NUnit.Framework;
+using TimberWinR.Inputs;
+using TimberWinR.Parser;
+using Newtonsoft.Json.Linq;
+using System.Diagnostics;
+
+namespace TimberWinR.UnitTests
+{
+    [TestFixture]
+    public class MultilineTests
+    {
+        [Test(Description = "Test using next")]
+        public void TestMultiline1()
+        {
+            using (StreamReader sr = new StreamReader("Multiline1.txt"))
+            {
+                List<JObject> events = new List<JObject>();
+
+                Console.SetIn(sr);
+
+                Stdin sin = new Stdin();
+
+                sin.Codec = new Codec();
+                sin.Codec.Pattern = "\\\\$";
+                sin.Codec.What = Codec.WhatType.next;
+                sin.Codec.Type = Codec.CodecType.multiline;
+
+                var cancelTokenSource = new CancellationTokenSource();
+
+                using (var syncHandle = new ManualResetEventSlim())
+                {
+                    try
+                    {
+                        StdinListener sl = new StdinListener(sin, cancelTokenSource.Token);
+
+                        sl.OnMessageRecieved += o =>
+                        {
+                            events.Add(o);
+                            if (events.Count >= 6)
+                                cancelTokenSource.Cancel();
+                        };
+
+                        if (!cancelTokenSource.Token.IsCancellationRequested)
+                            syncHandle.Wait(TimeSpan.FromSeconds(10000), cancelTokenSource.Token);
+                    }
+                    catch (OperationCanceledException oex)
+                    {
+                    }
+                }
+
+                Assert.AreEqual(events.Count, 6);
+                Assert.AreEqual(events[0]["message"].ToString(), "multiline1 \\\nml1_1 \\\nml1_2 \\\nml1_2 ");
+                Assert.AreEqual(events[1]["message"].ToString(), "singleline1");
+                Assert.AreEqual(events[2]["message"].ToString(), "singleline2");
+                Assert.AreEqual(events[3]["message"].ToString(), "multiline2 \\\nml2_1 \\\nml2_2");
+                Assert.AreEqual(events[4]["message"].ToString(), "multiline3 \\\nml3_1 \\\nml3_2");
+                Assert.AreEqual(events[5]["message"].ToString(), "singleline3");
+            }
+        }
+
+        [Test(Description = "Test using previous")]
+        public void TestMultiline2()
+        {
+            using (StreamReader sr = new StreamReader("Multiline2.txt"))
+            {
+                List<JObject> events = new List<JObject>();
+
+                Console.SetIn(sr);
+
+                Stdin sin = new Stdin();
+
+                sin.Codec = new Codec();
+                sin.Codec.Pattern = "^(\\d{4}-\\d{2}-\\d{2}\\s\\d{2}:\\d{2}:\\d{2},\\d{3})(.*)$";              
+                sin.Codec.What = Codec.WhatType.previous;
+                sin.Codec.Type = Codec.CodecType.multiline;
+                sin.Codec.Negate = true;
+
+                var cancelTokenSource = new CancellationTokenSource();
+
+                using (var syncHandle = new ManualResetEventSlim())
+                {
+                    try
+                    {
+                        StdinListener sl = new StdinListener(sin, cancelTokenSource.Token);
+
+                        sl.OnMessageRecieved += o =>
+                        {
+                            events.Add(o);
+                            if (events.Count >= 4)
+                                cancelTokenSource.Cancel();
+                        };
+
+                        if (!cancelTokenSource.Token.IsCancellationRequested)
+                            syncHandle.Wait(TimeSpan.FromSeconds(10000), cancelTokenSource.Token);
+                    }
+                    catch (OperationCanceledException oex)
+                    {
+                    }
+                }
+
+                Assert.AreEqual(events.Count, 4);
+                Assert.AreEqual(events[0]["message"].ToString(), "2015-01-07 13:14:26,572 TEST DEBUG [THREAD : 25] - Sending message to TServer - tcp://10.1111.11.111:1111\n'RequestAttachUserData' ('30')\nmessage attributes:\nAttributeConnID [long] = 00890\nAttributeReferenceID [int] = 88\nAttributeThisDN [str] = \"2214\"\nAttributeUserData [bstr] = KVList: \n\t\t'ActivityID' [str] = \"1-XXXXXX\"");
+                Assert.AreEqual(events[1]["message"].ToString(), "2015-01-07 13:14:26,574 TEST DEBUG [THREAD : 25] - Writing message RequestAttachUserData in 'proxy1' via '.StatePrimary proxy: proxy1'");
+                Assert.AreEqual(events[2]["message"].ToString(), "2015-01-07 13:14:26,575 TEST DEBUG [THREAD : 25] - sending RequestAttachUserData to Test.Platform.Commons.Connection.CommonConnection");
+                Assert.AreEqual(events[3]["message"].ToString(), "2015-01-07 13:20:31,665 TEST DEBUG [THREAD : SelectorThread] - Proxy got message 'EventOnHook' ('87')\nmessage attributes:\nAttributeEventSequenceNumber [long] = 4899493\nTime            = ComplexClass(TimeStamp):\n\tAttributeTimeinuSecs [int] = 573000\n\tAttributeTimeinSecs [int] = 1420644031\nAttributeThisDN [str] = \"2214\"\n. Processing with  state .StatePrimary proxy: proxy1");
+            }
+        }
+
+    }
+}
diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
index 95cd6ec..09026ad 100644
--- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
+++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
@@ -64,6 +64,7 @@
     <Compile Include="Inputs\IisW3CRowReaderTests.cs" />
     <Compile Include="JsonFilterTests.cs" />
     <Compile Include="GrokFilterTests.cs" />
+    <Compile Include="MultilineTests.cs" />
     <Compile Include="Parser\ElasticsearchOutputTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="TestBase.cs" />
@@ -82,6 +83,14 @@
       <SubType>Designer</SubType>
     </None>
   </ItemGroup>
+  <ItemGroup>
+    <Content Include="Multiline2.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+    <Content Include="Multiline1.txt">
+      <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+    </Content>
+  </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
   <!-- To modify your build process, add your task inside one of the targets below and uncomment it. 
        Other similar extension points exist, see Microsoft.Common.targets.
diff --git a/TimberWinR/Inputs/InputListener.cs b/TimberWinR/Inputs/InputListener.cs
index a6d415c..77bd0b3 100644
--- a/TimberWinR/Inputs/InputListener.cs
+++ b/TimberWinR/Inputs/InputListener.cs
@@ -18,7 +18,7 @@ namespace TimberWinR.Inputs
         private string _typeName;
         public AutoResetEvent FinishedEvent { get; set; }
         public string CheckpointFileName { get; set; }
-
+      
         public string InputType
         {
             get { return _typeName; }
diff --git a/TimberWinR/Inputs/LogsListener.cs b/TimberWinR/Inputs/LogsListener.cs
index 98555ce..65a07f2 100644
--- a/TimberWinR/Inputs/LogsListener.cs
+++ b/TimberWinR/Inputs/LogsListener.cs
@@ -2,7 +2,9 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net.Configuration;
+using System.Runtime.InteropServices;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using System.IO;
@@ -16,9 +18,150 @@ using NLog;
 using LogQuery = Interop.MSUtil.LogQueryClassClass;
 using TextLineInputFormat = Interop.MSUtil.COMTextLineInputContextClass;
 using LogRecordSet = Interop.MSUtil.ILogRecordset;
+using TimberWinR.Parser;
 
 namespace TimberWinR.Inputs
 {
+    public class LogsFileDatabase
+    {
+        private static readonly object _locker = new object();
+        private List<LogsFileDatabaseEntry> Entries { get; set; }
+        private string DatabaseDirectory { get; set; }
+        public string DatabaseFileName
+        {
+            get { return Path.Combine(DatabaseDirectory, ".timberwinrdb"); }
+        }
+
+        public static Manager Manager { get; set; }
+
+        private static LogsFileDatabase instance;
+
+        private bool ExistingFile(string logName)
+        {
+            lock (_locker)
+            {
+                return ExistingFileTest(logName);
+            }
+        }
+
+        private bool ExistingFileTest(string logName)
+        {
+            var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+            return existingEntry != null;
+        }
+
+        private void RemoveFileEntry(string logName)
+        {
+            lock (_locker)
+            {
+                var existingEntry = (from e in Entries where e.FileName == logName select e).FirstOrDefault();
+                if (existingEntry != null)
+                {
+                    Entries.Remove(existingEntry);
+                    WriteDatabaseFileNoLock();
+                }
+            }
+        }
+
+        private LogsFileDatabaseEntry AddFileEntry(string logName, TextLineInputFormat fmt)
+        {
+            LogsFileDatabaseEntry de = new LogsFileDatabaseEntry();
+            lock (_locker)
+            {
+                var lq = new LogQuery();
+                FileInfo fi = new FileInfo(logName);
+                de.FileName = logName;
+                de.Size = fi.Length;
+                de.SampleTime = DateTime.UtcNow;
+                de.CreationTime = fi.CreationTimeUtc;
+                if (fi.Exists)
+                {
+                    var qcount = string.Format("SELECT max(Index) as MaxRecordNumber FROM {0}", logName);
+                    var rcount = lq.Execute(qcount, fmt);
+                    var qr = rcount.getRecord();
+                    var lrn = (Int64)qr.getValueEx("MaxRecordNumber");
+                    de.MaxRecords = lrn;
+                }
+                Entries.Add(de);
+                WriteDatabaseFileNoLock();
+            }
+            return de;
+        }
+
+        public static LogsFileDatabaseEntry AddLogFile(string logName, TextLineInputFormat fmt)
+        {
+            Instance.RemoveFileEntry(logName); // Remove if already exists, otherwise ignores.
+            return Instance.AddFileEntry(logName, fmt);
+        }
+
+        public static LogsFileDatabase Instance
+        {
+            get
+            {
+                if (instance == null)
+                {
+                    instance = new LogsFileDatabase(Manager.LogfileDir);
+                    lock (_locker)
+                    {
+                        if (!Directory.Exists(instance.DatabaseDirectory))
+                        {
+                            Directory.CreateDirectory(instance.DatabaseDirectory);
+                        }
+                        if (File.Exists(instance.DatabaseFileName))
+                            instance.ReadDatabaseNoLock();
+                        else
+                            instance.WriteDatabaseFileNoLock();
+                    }
+                }
+                return instance;
+            }
+        }
+
+        private void ReadDatabaseNoLock()
+        {
+            JsonSerializer serializer = new JsonSerializer();
+            if (File.Exists(DatabaseFileName))
+                Entries = JsonConvert.DeserializeObject<List<LogsFileDatabaseEntry>>(File.ReadAllText(DatabaseFileName));
+        }
+        private void WriteDatabaseFileNoLock()
+        {
+            File.WriteAllText(DatabaseFileName, JsonConvert.SerializeObject(instance.Entries), Encoding.UTF8);
+        }
+
+
+        private void ReadDatabaseLock()
+        {
+            lock (_locker)
+            {
+                ReadDatabaseNoLock();
+            }
+
+        }
+        private void WriteDatabaseLock()
+        {
+            lock (_locker)
+            {
+                WriteDatabaseFileNoLock();
+            }
+        }
+
+        private LogsFileDatabase(string databaseDirectory)
+        {
+            DatabaseDirectory = databaseDirectory;
+            Entries = new List<LogsFileDatabaseEntry>();
+        }
+
+    }
+
+    public class LogsFileDatabaseEntry
+    {
+        public string FileName { get; set; }
+        public Int64 MaxRecords { get; set; }
+        public DateTime CreationTime { get; set; }
+        public DateTime SampleTime { get; set; }
+        public long Size { get; set; }
+    }
+
     /// <summary>
     /// Tail a file.
     /// </summary>
@@ -31,14 +174,17 @@ namespace TimberWinR.Inputs
         private Dictionary<string, DateTime> _logFileCreationTimes;
         private Dictionary<string, DateTime> _logFileSampleTimes;
         private Dictionary<string, long> _logFileSizes;
-      
+        private Codec _codec;
+        private List<string> _multiline { get; set; }
+
         public bool Stop { get; set; }
 
         public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken)
             : base(cancelToken, "Win32-FileLog")
         {
             Stop = false;
-          
+
+            _codec = arguments.Codec;
             _logFileMaxRecords = new Dictionary<string, Int64>();
             _logFileCreationTimes = new Dictionary<string, DateTime>();
             _logFileSampleTimes = new Dictionary<string, DateTime>();
@@ -64,15 +210,18 @@ namespace TimberWinR.Inputs
 
         public override JObject ToJson()
         {
+
             JObject json = new JObject(
                 new JProperty("log",
                     new JObject(
                         new JProperty("messages", _receivedMessages),
                         new JProperty("type", InputType),
                         new JProperty("location", _arguments.Location),
+                        new JProperty("logSource", _arguments.LogSource),
                         new JProperty("codepage", _arguments.CodePage),
                         new JProperty("splitLongLines", _arguments.SplitLongLines),
                         new JProperty("recurse", _arguments.Recurse),
+
                         new JProperty("files",
                             new JArray(from f in _logFileMaxRecords.Keys
                                        select new JValue(f))),
@@ -90,9 +239,96 @@ namespace TimberWinR.Inputs
                                        select new JValue(f)))
                         )));
 
+
+            if (_codec != null)
+            {
+                var cp = new JProperty("codec",
+                    new JArray(
+                        new JObject(
+                            new JProperty("type", _codec.Type.ToString()),
+                            new JProperty("what", _codec.What.ToString()),
+                            new JProperty("negate", _codec.Negate),
+                            new JProperty("multilineTag", _codec.MultilineTag),
+                            new JProperty("pattern", _codec.Pattern))));
+                json.Add(cp);
+            }
+
+
             return json;
         }
 
+        // return true to cancel codec
+        private void applyMultilineCodec(string msg)
+        {
+            if (_codec.Re == null)
+                _codec.Re = new Regex(_codec.Pattern);
+
+            Match match = _codec.Re.Match(msg);
+
+            bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
+
+            switch (_codec.What)
+            {
+                case Codec.WhatType.previous:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+
+                        _multiline.Add(msg);
+                    }
+                    else // No Match
+                    {
+                        if (_multiline != null)
+                        {
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codec.MultilineTag));
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);
+                            _receivedMessages++;
+                        }
+                        _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    break;
+                case Codec.WhatType.next:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    else // No match
+                    {
+                        if (_multiline != null)
+                        {
+                            _multiline.Add(msg);
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codec.MultilineTag));
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);
+                            _receivedMessages++;
+                        }
+                        else
+                        {
+                            JObject jo = new JObject();
+                            jo["message"] = msg;
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);
+                            _receivedMessages++;
+                        }
+                    }
+                    break;
+            }
+        }
+
+
         private void FileWatcher(string fileToWatch)
         {
             var iFmt = new TextLineInputFormat()
@@ -200,8 +436,13 @@ namespace TimberWinR.Inputs
                                     string msg = json["Text"].ToString();
                                     if (!string.IsNullOrEmpty(msg))
                                     {
-                                        ProcessJson(json);
-                                        _receivedMessages++;
+                                        if (_codec != null && _codec.Type == Codec.CodecType.multiline)
+                                            applyMultilineCodec(msg);
+                                        else
+                                        {
+                                            ProcessJson(json);
+                                            _receivedMessages++;
+                                        }
                                     }
 
                                     var lrn = (Int64)record.getValueEx("Index");
diff --git a/TimberWinR/Inputs/StdinListener.cs b/TimberWinR/Inputs/StdinListener.cs
index e71fa3d..1227fd2 100644
--- a/TimberWinR/Inputs/StdinListener.cs
+++ b/TimberWinR/Inputs/StdinListener.cs
@@ -1,21 +1,27 @@
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Text;
+using System.Text.RegularExpressions;
 using System.Threading;
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using NLog;
+using TimberWinR.Parser;
 
 namespace TimberWinR.Inputs
 {
     public class StdinListener : InputListener
     {
         private Thread _listenThread;
+        private Codec _codec;
+        private List<string> _multiline { get; set; }
 
-        public StdinListener(CancellationToken cancelToken)
+        public StdinListener(TimberWinR.Parser.Stdin arguments, CancellationToken cancelToken)
             : base(cancelToken, "Win32-Console")
         {
+            _codec = arguments.Codec;
             _listenThread = new Thread(new ThreadStart(ListenToStdin));
             _listenThread.Start();
         }
@@ -23,13 +29,28 @@ namespace TimberWinR.Inputs
         public override JObject ToJson()
         {
             JObject json = new JObject(
-                new JProperty("stdin", "enabled"));                 
+                new JProperty("stdin", "enabled"));
+
+            
+            if (_codec != null)
+            {
+                var cp = new JProperty("codec",
+                    new JArray(
+                        new JObject(
+                            new JProperty("type", _codec.Type.ToString()),
+                            new JProperty("what", _codec.What.ToString()),
+                            new JProperty("negate", _codec.Negate),
+                            new JProperty("multilineTag", _codec.MultilineTag),
+                            new JProperty("pattern", _codec.Pattern))));
+                json.Add(cp);              
+            }
+
             return json;
         }
 
         public override void Shutdown()
         {
-            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);         
+            LogManager.GetCurrentClassLogger().Info("Shutting Down {0}", InputType);
             base.Shutdown();
         }
 
@@ -38,20 +59,92 @@ namespace TimberWinR.Inputs
             LogManager.GetCurrentClassLogger().Info("StdIn Ready");
 
             while (!CancelToken.IsCancellationRequested)
-            {                       
+            {
                 string line = Console.ReadLine();
                 if (line != null)
                 {
                     string msg = ToPrintable(line);
-                    JObject jo = new JObject();
-                    jo["message"] = msg;
-                    AddDefaultFields(jo);                                    
-                    ProcessJson(jo);
-                }
-                else              
-                    break;              
+
+                    if (_codec != null && _codec.Type == Codec.CodecType.multiline)
+                        applyMultilineCodec(msg);
+                    else
+                    {
+                        JObject jo = new JObject();
+                        jo["message"] = msg;
+                        AddDefaultFields(jo);
+                        ProcessJson(jo);
+                    }
+                }              
             }
             Finished();
         }
+
+        // return true to cancel codec
+        private void applyMultilineCodec(string msg)
+        {
+            if (_codec.Re == null)
+                _codec.Re = new Regex(_codec.Pattern);
+
+            Match match = _codec.Re.Match(msg);
+
+            bool isMatch = (match.Success && !_codec.Negate) || (!match.Success && _codec.Negate);
+
+            switch (_codec.What)
+            {
+                case Codec.WhatType.previous:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+
+                        _multiline.Add(msg);
+                    }
+                    else // No Match
+                    {
+                        if (_multiline != null)
+                        {
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codec.MultilineTag));
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);                          
+                        }
+                        _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    break;
+                case Codec.WhatType.next:
+                    if (isMatch)
+                    {
+                        if (_multiline == null)
+                            _multiline = new List<string>();
+                        _multiline.Add(msg);
+                    }
+                    else // No match
+                    {
+                        if (_multiline != null)
+                        {
+                            _multiline.Add(msg);
+                            string single = string.Join("\n", _multiline.ToArray());
+                            _multiline = null;
+                            JObject jo = new JObject();
+                            jo["message"] = single;
+                            jo.Add("tags", new JArray(_codec.MultilineTag));
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);                          
+                        }
+                        else
+                        {
+                            JObject jo = new JObject();
+                            jo["message"] = msg;
+                            AddDefaultFields(jo);
+                            ProcessJson(jo);                          
+                        }
+                    }
+                    break;
+            }
+        }
     }
 }
diff --git a/TimberWinR/Manager.cs b/TimberWinR/Manager.cs
index 0465cb3..9ee6fdf 100644
--- a/TimberWinR/Manager.cs
+++ b/TimberWinR/Manager.cs
@@ -62,6 +62,8 @@ namespace TimberWinR
 
         public Manager(string jsonConfigFile, string logLevel, string logfileDir, CancellationToken cancelToken)
         {
+            LogsFileDatabase.Manager = this;           
+  
             StartedOn = DateTime.UtcNow;
 
             var vfi = new FileInfo(jsonConfigFile);
@@ -99,6 +101,11 @@ namespace TimberWinR
             LogManager.GetCurrentClassLogger()
                 .Info("TimberWinR Version {0}", GetAssemblyByName("TimberWinR.ServiceHost").GetName().Version.ToString());
 
+
+            LogManager.GetCurrentClassLogger()
+                .Info("Database Directory: {0}", LogsFileDatabase.Instance.DatabaseFileName);
+
+
             try
             {
                 // Is it a directory?
@@ -211,7 +218,7 @@ namespace TimberWinR
 
                 foreach (var stdin in Config.Stdins)
                 {
-                    var elistner = new StdinListener(cancelToken);
+                    var elistner = new StdinListener(stdin, cancelToken);
                     Listeners.Add(elistner);
                     foreach (var output in Outputs)
                         output.Connect(elistner);
diff --git a/TimberWinR/Outputs/Redis.cs b/TimberWinR/Outputs/Redis.cs
index aab918d..a501057 100644
--- a/TimberWinR/Outputs/Redis.cs
+++ b/TimberWinR/Outputs/Redis.cs
@@ -13,6 +13,7 @@ using System.Threading.Tasks;
 using RapidRegex.Core;
 using System.Text.RegularExpressions;
 using System.Globalization;
+using TimberWinR.Parser;
 
 namespace TimberWinR.Outputs
 {
@@ -174,10 +175,10 @@ namespace TimberWinR.Outputs
             foreach (var filter in _manager.Config.Filters)
             {
                 if (!filter.Apply(json))
-                {
+                {  
                     LogManager.GetCurrentClassLogger().Debug("Dropping: {0}", json.ToString());
                     drop = true;
-                }
+                }              
             }
             return drop;
         }
diff --git a/TimberWinR/Outputs/Stdout.cs b/TimberWinR/Outputs/Stdout.cs
index 041e605..41347b3 100644
--- a/TimberWinR/Outputs/Stdout.cs
+++ b/TimberWinR/Outputs/Stdout.cs
@@ -93,7 +93,10 @@ namespace TimberWinR.Outputs
         protected override void MessageReceivedHandler(Newtonsoft.Json.Linq.JObject jsonMessage)
         {
             if (_manager.Config.Filters != null)
-                ApplyFilters(jsonMessage);
+            {
+                if (ApplyFilters(jsonMessage))
+                    return;
+            }
 
             var message = jsonMessage.ToString();
             LogManager.GetCurrentClassLogger().Debug(message);
@@ -104,12 +107,17 @@ namespace TimberWinR.Outputs
             }
         }
 
-        private void ApplyFilters(JObject json)
+        private bool ApplyFilters(JObject json)
         {
+            bool drop = false;
+
             foreach (var filter in _manager.Config.Filters)
             {
-                filter.Apply(json);
+                if (!filter.Apply(json))
+                    drop = true;
             }
+
+            return drop;
         }
 
     }
diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs
index 3ebfd3d..c067623 100644
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -23,7 +23,7 @@ namespace TimberWinR.Parser
     {
         void Validate();
     }
-   
+
 
     public abstract class LogstashFilter : IValidateSchema
     {
@@ -51,22 +51,21 @@ namespace TimberWinR.Parser
         }
 
         public abstract JObject ToJson();
-      
+
         protected bool EvaluateCondition(JObject json, string condition)
         {
-           
             var cond = condition;
 
             IList<string> keys = json.Properties().Select(pn => pn.Name).ToList();
             foreach (string key in keys)
                 cond = cond.Replace(string.Format("[{0}]", key), string.Format("{0}", json[key].ToString()));
 
-            var p = Expression.Parameter(typeof (JObject), "");
-            var e = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] {p}, null, cond);
+            var p = Expression.Parameter(typeof(JObject), "");
+            var e = System.Linq.Dynamic.DynamicExpression.ParseLambda(new[] { p }, null, cond);
 
             var result = e.Compile().DynamicInvoke(json);
 
-            return (bool) result;
+            return (bool)result;
         }
         protected void RemoveProperties(JToken token, string[] fields)
         {
@@ -255,12 +254,49 @@ namespace TimberWinR.Parser
 
     public class Stdin : IValidateSchema
     {
+        [JsonProperty(PropertyName = "codec")]
+        public Codec Codec { get; set; }
+
         public void Validate()
         {
 
         }
     }
 
+    public class Codec
+    {
+        public enum CodecType
+        {
+            singleline,
+            multiline
+        };
+
+        public enum WhatType
+        {
+            previous,
+            next
+        };
+
+        [JsonProperty(PropertyName = "type")]
+        public CodecType Type { get; set; }
+        [JsonProperty(PropertyName = "pattern")]
+        public string Pattern { get; set; }
+        [JsonProperty(PropertyName = "what")]
+        public WhatType What { get; set; }
+        [JsonProperty(PropertyName = "negate")]
+        public bool Negate { get; set; }
+        [JsonProperty(PropertyName = "multiline_tag")]
+        public string MultilineTag { get; set; }
+
+        public Regex Re { get; set; }
+
+        public Codec()
+        {
+            Negate = false;
+            MultilineTag = "multiline";
+        }
+    }
+
     public class Log : IValidateSchema
     {
         [JsonProperty(PropertyName = "location")]
@@ -277,6 +313,8 @@ namespace TimberWinR.Parser
         public int Interval { get; set; }
         [JsonProperty(PropertyName = "logSource")]
         public string LogSource { get; set; }
+        [JsonProperty(PropertyName = "codec")]
+        public Codec Codec { get; set; }
 
         public Log()
         {
@@ -763,6 +801,7 @@ namespace TimberWinR.Parser
         }
     }
 
+   
     public partial class Json : LogstashFilter
     {
         public class JsonMissingSourceException : Exception
@@ -840,13 +879,13 @@ namespace TimberWinR.Parser
 
         [JsonProperty("json")]
         public Json Json { get; set; }
-    
+
         [JsonProperty("geoip")]
         public GeoIP GeoIP { get; set; }
 
         [JsonProperty("grokFilters")]
         public Grok[] Groks { get; set; }
-
+      
         [JsonProperty("mutateFilters")]
         public Mutate[] Mutates { get; set; }
 
@@ -855,9 +894,9 @@ namespace TimberWinR.Parser
 
         [JsonProperty("jsonFilters")]
         public Json[] Jsons { get; set; }
-       
+
         [JsonProperty("geoipFilters")]
-        public GeoIP[] GeoIPs { get; set; }
+        public GeoIP[] GeoIPs { get; set; }    
     }
 
     public class TimberWinR
@@ -874,9 +913,9 @@ namespace TimberWinR.Parser
             get
             {
                 var list = new List<LogstashFilter>();
-                
+
                 foreach (var filter in Filters)
-                {                  
+                {
                     foreach (var prop in filter.GetType().GetProperties())
                     {
                         object typedFilter = filter.GetType().GetProperty(prop.Name).GetValue(filter, null);
diff --git a/TimberWinR/ReleaseNotes.md b/TimberWinR/ReleaseNotes.md
index eb70280..c7b25a0 100644
--- a/TimberWinR/ReleaseNotes.md
+++ b/TimberWinR/ReleaseNotes.md
@@ -4,6 +4,10 @@ A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
 
 Version History
 
+### 1.3.19.0 - 01/12/2015
+
+1. Added support for Multiline codecs for Stdin and Logs listeners, addresses issue #23
+
 ### 1.3.18.0 - 12/22/2014
 
 1. Fixed bug introduced in 1.3.17.0 which changed the meaning of the delay for Elasticsearch, Redis and Stdout 
diff --git a/TimberWinR/TimberWinR.csproj b/TimberWinR/TimberWinR.csproj
index 703af30..323fb71 100644
--- a/TimberWinR/TimberWinR.csproj
+++ b/TimberWinR/TimberWinR.csproj
@@ -122,6 +122,7 @@
     <Content Include="GeoLite2City.mmdb">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
     </Content>
+    <None Include="mdocs\Codec.md" />
     <None Include="mdocs\DateFilter.md" />
     <None Include="mdocs\Filters.md" />
     <None Include="mdocs\GeoIPFilter.md" />
diff --git a/TimberWinR/mdocs/Codec.md b/TimberWinR/mdocs/Codec.md
new file mode 100644
index 0000000..bfe478c
--- /dev/null
+++ b/TimberWinR/mdocs/Codec.md
@@ -0,0 +1,31 @@
+# Codec
+
+## Parameters
+The following parameters are allowed when configuring the Codec.
+
+| Parameter         |     Type       |  Description                                                             | Details                       |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *type*            | enum           |Codec type 'multiline'                                                    | Must be 'multiline'           |     |
+| *pattern*         | regex          |Regular expression to be matched                                          | Must be legal .NET Regex      |     |
+| *what*            | enum           |Value can be previous or next                                             | If the pattern matched, does event belong to the next or previous event? |     |
+
+Example Input: Mutliline input log file
+
+```json
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Logs": [
+            {
+                "location": "C:\\Logs1\\multiline.log",
+                "recurse": -1,
+                "codec": {
+                "type": "multiline",
+                "pattern": "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+... \\d+ more)|(^\\s*Caused by:.+)",
+                "what": "previous"
+                }
+            }
+		 }
+	}
+}
+```
diff --git a/TimberWinR/mdocs/Logs.md b/TimberWinR/mdocs/Logs.md
index 55efdbe..4bd7490 100644
--- a/TimberWinR/mdocs/Logs.md
+++ b/TimberWinR/mdocs/Logs.md
@@ -11,6 +11,7 @@ The following parameters are allowed when configuring WindowsEvents.
 | *recurse*         | integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
 | *splitLongLines*  | boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false |
 | *iCodepage*       | integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |
 
 Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern.  I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
 
diff --git a/TimberWinR/mdocs/StdinInput.md b/TimberWinR/mdocs/StdinInput.md
index 2d4228c..580ef8e 100644
--- a/TimberWinR/mdocs/StdinInput.md
+++ b/TimberWinR/mdocs/StdinInput.md
@@ -26,4 +26,5 @@ A field: "type": "Win32-Stdin" is automatically appended, and the entire JSON is
 | ---- |:-----| :-----------------------------------------------------------------------|
 | type | STRING |Win32-Stdin |
 | message | STRING | The message typed in  |
+| [codec](https://github.com/Cimpress-MCP/TimberWinR/blob/master/TimberWinR/mdocs/Codec.md)  | object | Codec to use  |