TerribleDev/TimberWinR
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Be more pessimistic upon restarts

Browse Source
This commit is contained in:
Eric Fontana
2015-04-14 07:33:57 -04:00
parent 3e9ef6ae88
commit 5eb75ab143
10 changed files with 95 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
TimberWinR.ServiceHost/Properties/AssemblyInfo.cs
View File

@@ -32,5 +32,5 @@ using System.Runtime.InteropServices;
// You can specify all the values or you can default the Build and Revision Numbers
// by using the '*' as shown below:
// [assembly: AssemblyVersion("1.0.*")]
[assembly: AssemblyVersion("1.3.21.0")]
[assembly: AssemblyFileVersion("1.3.21.0")]
[assembly: AssemblyVersion("1.3.22.0")]
[assembly: AssemblyFileVersion("1.3.22.0")]

3
TimberWinR.TestGenerator/TimberWinR.TestGenerator.csproj
View File

@@ -108,6 +108,9 @@
<Content Include="results3.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
<Content Include="test3-twconfig.json">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</Content>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\TimberWinR\TimberWinR.csproj">

13
TimberWinR.TestGenerator/test1-twconfig.json
View File

@@ -1,14 +1,14 @@
{
"TimberWinR": {
"Inputs": {
"Inputs": {
"Udp": [
{
"_comment": "Output from NLog",
"port": 5140
}
],
"TailFiles": [
{
"TailFiles": [
{
"interval": 5,
"logSource": "log files",
"location": "*.jlog",
@@ -25,6 +25,11 @@
""
],
"drop": "true"
},
"json": {
"type": "Win32-TailFile",
"source": "Text",
"promote": "Text"
}
}
],
@@ -34,7 +39,7 @@
"_comment": "Change the host to your Redis instance",
"port": 6379,
"batch_count": 500,
"threads": 2,
"threads": 2,
"host": [
"tstlexiceapp006.vistaprint.svc"
]

2
TimberWinR.TestGenerator/test1.json
View File

@@ -6,7 +6,7 @@
"--timberWinRConfig": "test1-twconfig.json",
"--numMessages": 1234,
"--logLevel": "debug",
"--udp-host": "::1",
"--udp-host": "localhost",
"--udp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],

2
TimberWinR.TestGenerator/test2.json
View File

@@ -5,7 +5,7 @@
"--testDir": "test2",
"--timberWinRConfig": "test2-tw.json",
"--numMessages": 1234,
"--logLevel": "debug",
"--logLevel": "trace",
"--udp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],

17
TimberWinR.TestGenerator/test3-tw.json
View File

@@ -1,17 +1,17 @@
{
"TimberWinR": {
"Inputs": {
"Tcp": [
"Inputs": {
"Udp": [
{
"_comment": "Output from NLog",
"port": 5140
}
],
"Logs": [
{
"TailFiles": [
{
"interval": 5,
"logSource": "log files",
"location": "*.jlog",
"location": "d:\\logs\\sta\\sta.log",
"recurse": -1
}
]
@@ -25,6 +25,11 @@
""
],
"drop": "true"
},
"json": {
"type": "Win32-TailFile",
"source": "Text",
"promote": "Text"
}
}
],
@@ -34,7 +39,7 @@
"_comment": "Change the host to your Redis instance",
"port": 6379,
"batch_count": 500,
"threads": 2,
"threads": 2,
"host": [
"tstlexiceapp006.vistaprint.svc"
]

50
TimberWinR.TestGenerator/test3-twconfig.json Normal file
View File

@@ -0,0 +1,50 @@
{
"TimberWinR": {
"Inputs": {
"Udp": [
{
"_comment": "Output from NLog",
"port": 5140
}
],
"TailFiles": [
{
"interval": 5,
"logSource": "log files",
"location": "d:\\logs\\sta\\sta.log",
"recurse": -1
}
]
},
"Filters": [
{
"grok": {
"condition": "\"[EventTypeName]\" == \"Information Event\"",
"match": [
"Text",
""
],
"drop": "true"
},
"json": {
"type": "Win32-TailFile",
"source": "Text",
"promote": "Text"
}
}
],
"Outputs": {
"Redis": [
{
"_comment": "Change the host to your Redis instance",
"port": 6379,
"batch_count": 500,
"threads": 2,
"host": [
"tstlexiceapp006.vistaprint.svc"
]
}
]
}
}
}

5
TimberWinR.TestGenerator/test3.json
View File

@@ -5,10 +5,7 @@
"--testDir": "test3",
"--timberWinRConfig": "test3-tw.json",
"--numMessages": 1234,
"--logLevel": "debug",
"--tcp": "5140",
"--jroll": ["r1.jlog", "r2.jlog"],
"--json": ["1.jlog", "2.jlog", "3.jlog", "4.jlog"],
"--logLevel": "debug",
"--resultsFile": "results3.json"
}
}

16
TimberWinR/Inputs/TailFileListener.cs
View File

@@ -115,7 +115,14 @@ namespace TimberWinR.Inputs
//seek to the last max offset
LogManager.GetCurrentClassLogger().Trace("{0}: File: {1} Seek to: {2}", Thread.CurrentThread.ManagedThreadId, fileName, lastMaxOffset);
reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin);
var seekedTo = reader.BaseStream.Seek(lastMaxOffset, SeekOrigin.Begin);
// We couldn't seek to the position, so remember what we have seeked to.
if (seekedTo != lastMaxOffset)
{
lastMaxOffset = seekedTo;
LogsFileDatabase.Update(dbe, true, lastMaxOffset);
}
//read out of the file until the EOF
string line = "";
@@ -196,18 +203,19 @@ namespace TimberWinR.Inputs
foreach (string fileName in Directory.GetFiles(path, name, so))
{
var dbe = LogsFileDatabase.LookupLogFile(fileName);
// We only spin up 1 thread for a file we haven't yet seen.
if (isWildcardPattern && !HaveSeenFile(fileName) && dbe.NewFile)
{
LogManager.GetCurrentClassLogger().Debug(":{0} Starting Thread Tailing File: {1}", Thread.CurrentThread.ManagedThreadId, dbe.FileName);
LogsFileDatabase.Update(dbe, false, dbe.LastPosition);
SaveVisitedFileName(fileName);
Task.Factory.StartNew(() => TailFileWatcher(fileName));
}
else if (!isWildcardPattern)
{
FileInfo fi = new FileInfo(dbe.FileName);
SaveVisitedFileName(fileName);
//LogManager.GetCurrentClassLogger().Info("Located File: {0}, New: {1}", dbe.FileName, dbe.NewFile);
long length = fi.Length;
@@ -219,7 +227,7 @@ namespace TimberWinR.Inputs
LogsFileDatabase.Roll(dbe);
}
// Log has rolled or this is a file we are seeing for the first time.
bool processWholeFile = logHasRolled || !dbe.ProcessedFile;
bool processWholeFile = logHasRolled || !dbe.ProcessedFile || dbe.NewFile;
if (processWholeFile)
{
LogsFileDatabase.Update(dbe, true, 0);

5
TimberWinR/ReleaseNotes.md
View File

@@ -3,6 +3,11 @@
A Native Windows to Redis/Elasticsearch Logstash Agent which runs as a service.
Version / Date
### 1.3.22.0 - 2015-04-14
1. Fixed minor bug with TailFiles and service re-starts not picking up
rolled files right away.
### 1.3.21.0 - 2015-04-13
1. Rolled Udp listener support to V4 only, too many issues with dual mode sockets
and hosts file. If we want to add this back, I will add a Udpv6 input.