From 5eb75ab143c31e52ebb64e531516bcd565d0c3d1 Mon Sep 17 00:00:00 2001 From: Eric Fontana Date: Tue, 14 Apr 2015 07:33:57 -0400 Subject: [PATCH] Be more pessimistic upon restarts --- .../Properties/AssemblyInfo.cs | 4 +- .../TimberWinR.TestGenerator.csproj | 3 ++ TimberWinR.TestGenerator/test1-twconfig.json | 13 +++-- TimberWinR.TestGenerator/test1.json | 2 +- TimberWinR.TestGenerator/test2.json | 2 +- TimberWinR.TestGenerator/test3-tw.json | 17 ++++--- TimberWinR.TestGenerator/test3-twconfig.json | 50 +++++++++++++++++++ TimberWinR.TestGenerator/test3.json | 5 +- TimberWinR/Inputs/TailFileListener.cs | 16 ++++-- TimberWinR/ReleaseNotes.md | 5 ++ 10 files changed, 95 insertions(+), 22 deletions(-) create mode 100644 TimberWinR.TestGenerator/test3-twconfig.json diff --git a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs index bccff3a..6162eaa 100644 --- a/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs +++ b/TimberWinR.ServiceHost/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("1.3.21.0")] -[assembly: AssemblyFileVersion("1.3.21.0")] +[assembly: AssemblyVersion("1.3.22.0")] +[assembly: AssemblyFileVersion("1.3.22.0")] diff --git a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj index e4d287b..5d08e3d 100644 --- a/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj +++ b/TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj @@ -108,6 +108,9 @@ PreserveNewest + + PreserveNewest + diff --git a/TimberWinR.TestGenerator/test1-twconfig.json b/TimberWinR.TestGenerator/test1-twconfig.json index 9740666..4628dc6 100644 --- a/TimberWinR.TestGenerator/test1-twconfig.json +++ b/TimberWinR.TestGenerator/test1-twconfig.json @@ -1,14 +1,14 @@ { "TimberWinR": { - "Inputs": { + "Inputs": { "Udp": [ { "_comment": "Output from NLog", "port": 5140 } ], - "TailFiles": [ - { + "TailFiles": [ + { "interval": 5, "logSource": "log files", "location": "*.jlog", @@ -25,6 +25,11 @@ "" ], "drop": "true" + }, + "json": { + "type": "Win32-TailFile", + "source": "Text", + "promote": "Text" } } ], @@ -34,7 +39,7 @@ "_comment": "Change the host to your Redis instance", "port": 6379, "batch_count": 500, - "threads": 2, + "threads": 2, "host": [ "tstlexiceapp006.vistaprint.svc" ] diff --git a/TimberWinR.TestGenerator/test1.json b/TimberWinR.TestGenerator/test1.json index 241a50c..aa3e4cf 100644 --- a/TimberWinR.TestGenerator/test1.json +++ b/TimberWinR.TestGenerator/test1.json @@ -6,7 +6,7 @@ "--timberWinRConfig": "test1-twconfig.json", "--numMessages": 1234, "--logLevel": "debug", - "--udp-host": "::1", + "--udp-host": "localhost", "--udp": "5140", "--jroll": ["r1.jlog", "r2.jlog"], "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], diff --git a/TimberWinR.TestGenerator/test2.json b/TimberWinR.TestGenerator/test2.json index 223da98..f47863b 100644 --- a/TimberWinR.TestGenerator/test2.json +++ b/TimberWinR.TestGenerator/test2.json @@ -5,7 +5,7 @@ "--testDir": "test2", "--timberWinRConfig": "test2-tw.json", "--numMessages": 1234, - "--logLevel": "debug", + "--logLevel": "trace", "--udp": "5140", "--jroll": ["r1.jlog", "r2.jlog"], "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], diff --git a/TimberWinR.TestGenerator/test3-tw.json b/TimberWinR.TestGenerator/test3-tw.json index 4599c56..8714923 100644 --- a/TimberWinR.TestGenerator/test3-tw.json +++ b/TimberWinR.TestGenerator/test3-tw.json @@ -1,17 +1,17 @@ { "TimberWinR": { - "Inputs": { - "Tcp": [ + "Inputs": { + "Udp": [ { "_comment": "Output from NLog", "port": 5140 } ], - "Logs": [ - { + "TailFiles": [ + { "interval": 5, "logSource": "log files", - "location": "*.jlog", + "location": "d:\\logs\\sta\\sta.log", "recurse": -1 } ] @@ -25,6 +25,11 @@ "" ], "drop": "true" + }, + "json": { + "type": "Win32-TailFile", + "source": "Text", + "promote": "Text" } } ], @@ -34,7 +39,7 @@ "_comment": "Change the host to your Redis instance", "port": 6379, "batch_count": 500, - "threads": 2, + "threads": 2, "host": [ "tstlexiceapp006.vistaprint.svc" ] diff --git a/TimberWinR.TestGenerator/test3-twconfig.json b/TimberWinR.TestGenerator/test3-twconfig.json new file mode 100644 index 0000000..8714923 --- /dev/null +++ b/TimberWinR.TestGenerator/test3-twconfig.json @@ -0,0 +1,50 @@ +{ + "TimberWinR": { + "Inputs": { + "Udp": [ + { + "_comment": "Output from NLog", + "port": 5140 + } + ], + "TailFiles": [ + { + "interval": 5, + "logSource": "log files", + "location": "d:\\logs\\sta\\sta.log", + "recurse": -1 + } + ] + }, + "Filters": [ + { + "grok": { + "condition": "\"[EventTypeName]\" == \"Information Event\"", + "match": [ + "Text", + "" + ], + "drop": "true" + }, + "json": { + "type": "Win32-TailFile", + "source": "Text", + "promote": "Text" + } + } + ], + "Outputs": { + "Redis": [ + { + "_comment": "Change the host to your Redis instance", + "port": 6379, + "batch_count": 500, + "threads": 2, + "host": [ + "tstlexiceapp006.vistaprint.svc" + ] + } + ] + } + } +} diff --git a/TimberWinR.TestGenerator/test3.json b/TimberWinR.TestGenerator/test3.json index dadb1d5..28d5faf 100644 --- a/TimberWinR.TestGenerator/test3.json +++ b/TimberWinR.TestGenerator/test3.json @@ -5,10 +5,7 @@ "--testDir": "test3", "--timberWinRConfig": "test3-tw.json", "--numMessages": 1234, - "--logLevel": "debug", - "--tcp": "5140", - "--jroll": ["r1.jlog", "r2.jlog"], - "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], + "--logLevel": "debug", "--resultsFile": "results3.json" } } diff --git a/TimberWinR/Inputs/TailFileListener.cs b/TimberWinR/Inputs/TailFileListener.cs index e4390a8..131d2fc 100644 --- a/TimberWinR/Inputs/TailFileListener.cs +++ b/TimberWinR/Inputs/TailFileListener.cs @@ -115,7 +115,14 @@ namespace TimberWinR.Inputs //seek to the last max offset LogManager.GetCurrentClassLogger().Trace("{0}: File: {1} Seek to: {2}", Thread.CurrentThread.ManagedThreadId, fileName, lastMaxOffset); - reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin); + var seekedTo = reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin); + + // We couldn't seek to the position, so remember what we have seeked to. + if (seekedTo != lastMaxOffset) + { + lastMaxOffset = seekedTo; + LogsFileDatabase.Update(dbe, true, lastMaxOffset); + } //read out of the file until the EOF string line = ""; @@ -196,18 +203,19 @@ namespace TimberWinR.Inputs foreach (string fileName in Directory.GetFiles(path, name, so)) { var dbe = LogsFileDatabase.LookupLogFile(fileName); - + // We only spin up 1 thread for a file we haven't yet seen. if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile) { LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName); LogsFileDatabase.Update(dbe, false, dbe.LastPosition); - SaveVisitedFileName(fileName); + Task.Factory.StartNew(() => TailFileWatcher(fileName)); } else if (!isWildcardPattern) { FileInfo fi = new FileInfo(dbe.FileName); + SaveVisitedFileName(fileName); //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile); long length = fi.Length; @@ -219,7 +227,7 @@ namespace TimberWinR.Inputs LogsFileDatabase.Roll(dbe); } // Log has rolled or this is a file we are seeing for the first time. - bool processWholeFile = logHasRolled || !dbe.ProcessedFile; + bool processWholeFile = logHasRolled || !dbe.ProcessedFile || dbe.NewFile; if (processWholeFile) { LogsFileDatabase.Update(dbe, true, 0); diff --git a/TimberWinR/ReleaseNotes.md b/TimberWinR/ReleaseNotes.md index 22485fc..a3bdc7f 100644 --- a/TimberWinR/ReleaseNotes.md +++ b/TimberWinR/ReleaseNotes.md @@ -3,6 +3,11 @@ A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service. Version / Date + +### 1.3.22.0 - 2015-04-14 +1. Fixed minor bug with TailFiles and service re-starts not picking up + rolled files right away. + ### 1.3.21.0 - 2015-04-13 1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets and hosts file. If we want to add this back, I will add a Udpv6 input.