TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Be more pessimistic upon restarts

Browse Source
This commit is contained in:
Eric Fontana
2015-04-14 07:33:57 -04:00
parent 3e9ef6ae88
commit 5eb75ab143
10 changed files with 95 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers // You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below: // by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")] // [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.21.0")] [assembly: AssemblyVersion("1.3.22.0")]
[assembly: AssemblyFileVersion("1.3.21.0")] [assembly: AssemblyFileVersion("1.3.22.0")]

3
TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
View File

@@ -108,6 +108,9 @@
<Content Include="results3.json"> <Content Include="results3.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory> <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content> </Content>
<Content Include="test3-twconfig.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj"> <ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

13
TimberWinR.TestGenerator/test1-twconfig.json
View File

@@ -1,14 +1,14 @@
{ {
"TimberWinR": { "TimberWinR": {
"Inputs": { "Inputs": {
"Udp": [ "Udp": [
{ {
"_comment": "Output from NLog", "_comment": "Output from NLog",
"port": 5140 "port": 5140
} }
], ],
"TailFiles": [ "TailFiles": [
{ {
"interval": 5, "interval": 5,
"logSource": "log files", "logSource": "log files",
"location": "*.jlog", "location": "*.jlog",
@@ -25,6 +25,11 @@
"" ""
], ],
"drop": "true" "drop": "true"
},
"json": {
"type": "Win32-TailFile",
"source": "Text",
"promote": "Text"
} }
} }
], ],
@@ -34,7 +39,7 @@
"_comment": "Change the host to your Redis instance", "_comment": "Change the host to your Redis instance",
"port": 6379, "port": 6379,
"batch_count": 500, "batch_count": 500,
"threads": 2, "threads": 2,
"host": [ "host": [
"tstlexiceapp006.vistaprint.svc" "tstlexiceapp006.vistaprint.svc"
] ]

2
TimberWinR.TestGenerator/test1.json
View File

@@ -6,7 +6,7 @@
"--timberWinRConfig": "test1-twconfig.json", "--timberWinRConfig": "test1-twconfig.json",
"--numMessages": 1234, "--numMessages": 1234,
"--logLevel": "debug", "--logLevel": "debug",
"--udp-host": "::1", "--udp-host": "localhost",
"--udp": "5140", "--udp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"], "--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],

2
TimberWinR.TestGenerator/test2.json
View File

@@ -5,7 +5,7 @@
"--testDir": "test2", "--testDir": "test2",
"--timberWinRConfig": "test2-tw.json", "--timberWinRConfig": "test2-tw.json",
"--numMessages": 1234, "--numMessages": 1234,
"--logLevel": "debug", "--logLevel": "trace",
"--udp": "5140", "--udp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"], "--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"], "--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],

17
TimberWinR.TestGenerator/test3-tw.json
View File

@@ -1,17 +1,17 @@
{ {
"TimberWinR": { "TimberWinR": {
"Inputs": { "Inputs": {
"Tcp": [ "Udp": [
{ {
"_comment": "Output from NLog", "_comment": "Output from NLog",
"port": 5140 "port": 5140
} }
], ],
"Logs": [ "TailFiles": [
{ {
"interval": 5, "interval": 5,
"logSource": "log files", "logSource": "log files",
"location": "*.jlog", "location": "d:\\logs\\sta\\sta.log",
"recurse": -1 "recurse": -1
} }
] ]
@@ -25,6 +25,11 @@
"" ""
], ],
"drop": "true" "drop": "true"
},
"json": {
"type": "Win32-TailFile",
"source": "Text",
"promote": "Text"
} }
} }
], ],
@@ -34,7 +39,7 @@
"_comment": "Change the host to your Redis instance", "_comment": "Change the host to your Redis instance",
"port": 6379, "port": 6379,
"batch_count": 500, "batch_count": 500,
"threads": 2, "threads": 2,
"host": [ "host": [
"tstlexiceapp006.vistaprint.svc" "tstlexiceapp006.vistaprint.svc"
] ]

50
TimberWinR.TestGenerator/test3-twconfig.json Normal file
View File

@@ -0,0 +1,50 @@
{
"TimberWinR": {
"Inputs": {
"Udp": [
{
"_comment": "Output from NLog",
"port": 5140
}
],
"TailFiles": [
{
"interval": 5,
"logSource": "log files",
"location": "d:\\logs\\sta\\sta.log",
"recurse": -1
}
]
},
"Filters": [
{
"grok": {
"condition": "\"[EventTypeName]\" == \"Information Event\"",
"match": [
"Text",
""
],
"drop": "true"
},
"json": {
"type": "Win32-TailFile",
"source": "Text",
"promote": "Text"
}
}
],
"Outputs": {
"Redis": [
{
"_comment": "Change the host to your Redis instance",
"port": 6379,
"batch_count": 500,
"threads": 2,
"host": [
"tstlexiceapp006.vistaprint.svc"
]
}
]
}
}
}

5
TimberWinR.TestGenerator/test3.json
View File

@@ -5,10 +5,7 @@
"--testDir": "test3", "--testDir": "test3",
"--timberWinRConfig": "test3-tw.json", "--timberWinRConfig": "test3-tw.json",
"--numMessages": 1234, "--numMessages": 1234,
"--logLevel": "debug", "--logLevel": "debug",
"--tcp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
"--resultsFile": "results3.json" "--resultsFile": "results3.json"
} }
} }

16
TimberWinR/Inputs/TailFileListener.cs
View File

@@ -115,7 +115,14 @@ namespace TimberWinR.Inputs
//seek to the last max offset //seek to the last max offset
LogManager.GetCurrentClassLogger().Trace("{0}: File: {1} Seek to: {2}", Thread.CurrentThread.ManagedThreadId, fileName, lastMaxOffset); LogManager.GetCurrentClassLogger().Trace("{0}: File: {1} Seek to: {2}", Thread.CurrentThread.ManagedThreadId, fileName, lastMaxOffset);
reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin); var seekedTo = reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin);
// We couldn't seek to the position, so remember what we have seeked to.
if (seekedTo != lastMaxOffset)
{
lastMaxOffset = seekedTo;
LogsFileDatabase.Update(dbe, true, lastMaxOffset);
}
//read out of the file until the EOF //read out of the file until the EOF
string line = ""; string line = "";
@@ -196,18 +203,19 @@ namespace TimberWinR.Inputs
foreach (string fileName in Directory.GetFiles(path, name, so)) foreach (string fileName in Directory.GetFiles(path, name, so))
{ {
var dbe = LogsFileDatabase.LookupLogFile(fileName); var dbe = LogsFileDatabase.LookupLogFile(fileName);
// We only spin up 1 thread for a file we haven't yet seen. // We only spin up 1 thread for a file we haven't yet seen.
if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile) if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile)
{ {
LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName); LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
LogsFileDatabase.Update(dbe, false, dbe.LastPosition); LogsFileDatabase.Update(dbe, false, dbe.LastPosition);
SaveVisitedFileName(fileName);
Task.Factory.StartNew(() => TailFileWatcher(fileName)); Task.Factory.StartNew(() => TailFileWatcher(fileName));
} }
else if (!isWildcardPattern) else if (!isWildcardPattern)
{ {
FileInfo fi = new FileInfo(dbe.FileName); FileInfo fi = new FileInfo(dbe.FileName);
SaveVisitedFileName(fileName);
//LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile); //LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);
long length = fi.Length; long length = fi.Length;
@@ -219,7 +227,7 @@ namespace TimberWinR.Inputs
LogsFileDatabase.Roll(dbe); LogsFileDatabase.Roll(dbe);
} }
// Log has rolled or this is a file we are seeing for the first time. // Log has rolled or this is a file we are seeing for the first time.
bool processWholeFile = logHasRolled || !dbe.ProcessedFile; bool processWholeFile = logHasRolled || !dbe.ProcessedFile || dbe.NewFile;
if (processWholeFile) if (processWholeFile)
{ {
LogsFileDatabase.Update(dbe, true, 0); LogsFileDatabase.Update(dbe, true, 0);

5
TimberWinR/ReleaseNotes.md
View File

@@ -3,6 +3,11 @@
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service. A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version / Date Version / Date
### 1.3.22.0 - 2015-04-14
1. Fixed minor bug with TailFiles and service re-starts not picking up
rolled files right away.
### 1.3.21.0 - 2015-04-13 ### 1.3.21.0 - 2015-04-13
1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets 1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets
and hosts file. If we want to add this back, I will add a Udpv6 input. and hosts file. If we want to add this back, I will add a Udpv6 input.