Added logs doc

2014-07-30 08:25:10 -04:00
parent 65ea328544
commit e682f5298c
9 changed files with 74 additions and 37 deletions
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ These are broken down into:

 ## Input Formats
 The current supported Input format sources are:
- 1. Logs (Files, a.k.a Tailing a file)
+ 1. [Logs](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/Logs.md) (Files, a.k.a Tailing a file)
 2. Tcp (listens on a port for JSON messages)
 3. IISW3C (Internet Information Services W3C Format)
 4. [WindowsEvents](https://github.com/efontana/TimberWinR/blob/master/TimberWinR/mdocs/WindowsEvents.md)
--- a/TimberWinR.sln
+++ b/TimberWinR.sln
@@ -19,7 +19,6 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Solution Items", "Solution
 		LICENSE.txt = LICENSE.txt
 		Package.nuspec = Package.nuspec
 		README.md = README.md
-		WindowsEvents.md = WindowsEvents.md
 	EndProjectSection
 EndProject
 Global
--- a/TimberWinR/Inputs/TailFileInputListener.cs
+++ b/TimberWinR/Inputs/TailFileInputListener.cs
@@ -20,12 +20,12 @@ namespace TimberWinR.Inputs
    /// <summary>
    /// Tail a file.
    /// </summary>
-    public class TailFileInputListener : InputListener
+    public class LogsListener : InputListener
    {
-        private int _pollingIntervalInSeconds = 1;
+        private int _pollingIntervalInSeconds;
        private TimberWinR.Parser.Log _arguments;

-        public TailFileInputListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 1)
+        public LogsListener(TimberWinR.Parser.Log arguments, CancellationToken cancelToken, int pollingIntervalInSeconds = 3)
            : base(cancelToken, "Win32-FileLog")
        {
            _arguments = arguments;
@@ -50,8 +50,7 @@ namespace TimberWinR.Inputs
            };

              // Create the query
-            var query = string.Format("SELECT * FROM {0}", _arguments.Location);
-          
+            var query = string.Format("SELECT * FROM {0}", _arguments.Location);          

            var firstQuery = true;
            // Execute the query
--- a/TimberWinR/Manager.cs
+++ b/TimberWinR/Manager.cs
@@ -81,7 +81,7 @@ namespace TimberWinR

            foreach (var logConfig in Config.Logs)
            {
-                var elistner = new TailFileInputListener(logConfig, cancelToken);
+                var elistner = new LogsListener(logConfig, cancelToken);
                Listeners.Add(elistner);
                foreach (var output in Outputs)
                    output.Connect(elistner);
--- a/TimberWinR/Outputs/Redis.cs
+++ b/TimberWinR/Outputs/Redis.cs
@@ -23,7 +23,7 @@ namespace TimberWinR.Outputs
        private readonly int _timeout;
        private readonly object _locker = new object();
        private readonly List<string> _jsonQueue;
-       // readonly Task _consumerTask;
+        // readonly Task _consumerTask;
        private readonly string[] _redisHosts;
        private int _redisHostIndex;
        private TimberWinR.Manager _manager;
@@ -52,7 +52,7 @@ namespace TimberWinR.Outputs

                    return client;
                }
-                catch (Exception )
+                catch (Exception)
                {
                }
                numTries++;
@@ -61,7 +61,7 @@ namespace TimberWinR.Outputs
            return null;
        }

-        public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken) //string[] redisHosts, string logstashIndexName = "logstash", int port = 6379, int timeout = 10000, int batch_count = 10)
+        public RedisOutput(TimberWinR.Manager manager, Parser.RedisOutput ro, CancellationToken cancelToken)
            : base(cancelToken)
        {
            _batchCount = ro.BatchCount;
@@ -105,7 +105,7 @@ namespace TimberWinR.Outputs
            foreach (var filter in _manager.Config.Filters)
            {
                filter.Apply(json);
-            }            
+            }
        }

        // 
@@ -119,7 +119,7 @@ namespace TimberWinR.Outputs
                lock (_locker)
                {
                    messages = _jsonQueue.Take(_batchCount).ToArray();
-                    _jsonQueue.RemoveRange(0, messages.Length);                   
+                    _jsonQueue.RemoveRange(0, messages.Length);
                }

                if (messages.Length > 0)
@@ -141,7 +141,7 @@ namespace TimberWinR.Outputs
                                    foreach (string jsonMessage in messages)
                                    {
                                        try
-                                        {                                           
+                                        {
                                            client.RPush(_logstashIndexName, jsonMessage);
                                        }
                                        catch (SocketException ex)
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -275,7 +275,6 @@ namespace TimberWinR.Parser
        public int Recurse { get; set; }
        [JsonProperty(PropertyName = "splitLongLines")]
        public bool SplitLongLines { get; set; }
-
        [JsonProperty(PropertyName = "fields")]
        public List<Field> Fields { get; set; }

@@ -310,9 +309,7 @@ namespace TimberWinR.Parser
    }
    
    public class IISW3CLog : IValidateSchema
-    {
-        [JsonProperty(PropertyName = "name")]
-        public string Name { get; set; }
+    {       
        [JsonProperty(PropertyName = "location")]
        public string Location { get; set; }
        [JsonProperty(PropertyName = "iCodepage")]
--- a/TimberWinR/TimberWinR.csproj
+++ b/TimberWinR/TimberWinR.csproj
@@ -76,7 +76,7 @@
    <Compile Include="Inputs\InputListener.cs" />
    <Compile Include="Inputs\ParameterDefinitions.cs" />
    <Compile Include="Inputs\TcpInputListener.cs" />
-    <Compile Include="Inputs\TailFileInputListener.cs" />
+    <Compile Include="Inputs\LogsListener.cs" />
    <Compile Include="Inputs\WindowsEvtInputListener.cs" />
    <Compile Include="Manager.cs" />
    <Compile Include="Outputs\OutputSender.cs" />
@@ -102,6 +102,7 @@
    <None Include="mdocs\Filters.md" />
    <None Include="mdocs\GrokFilter.md" />
    <None Include="mdocs\MutateFilter.md" />
+    <None Include="mdocs\Logs.md" />
    <None Include="mdocs\WindowsEvents.md" />
    <None Include="packages.config" />
  </ItemGroup>
--- a/TimberWinR/mdocs/Logs.md
+++ b/TimberWinR/mdocs/Logs.md
@@ -0,0 +1,37 @@
+# Input: Logs
+
+The Logs input will monitor a log (text) file similar to how a Linux "tail -f" command works. 
+
+## Parameters
+The following parameters are allowed when configuring WindowsEvents.
+
+| Parameter         |     Type       |  Description                                                             | Details               |  Default |
+| :---------------- |:---------------| :----------------------------------------------------------------------- | :---------------------------  | :-- |
+| *location*        | property:string  |Location of file(s) to monitor                                           | Path to text file(s) including wildcards. |     |
+| *recurse*         | property:integer |Max subdirectory recursion level.                                       | 0 disables subdirectory recursion; -1 enables unlimited recursion. | 0 |
+| *splitLongLines*  | property:boolean |Behavior when event messages or event category names cannot be resolved. |When a text line is longer than 128K characters, the format truncates the line and either discards the remaining of the line (when this parameter is set to "false"), or processes the remainder of the line as a new line (when this parameter is set to "true").| false |
+| *iCodepage*       | property:integer |Codepage of the text file.                                              | 0 is the system codepage, -1 is UNICODE.                         | 0  |
+
+Example Input: Monitors all files (recursively) located at C:\Logs1\ matching *.log as a pattern.  I.e. C:\Logs1\foo.log, C:\Logs1\Subdir\Log2.log, etc.
+
+```json
+{
+    "TimberWinR": {
+        "Inputs": {
+            "Logs": [
+                {
+                    "location": "C:\\Logs1\\*.log",
+                    "recurse": -1
+                }
+            ]
+		}
+	}
+}
+```
+## Fields
+After a successful parse of an event, the following fields are added:
+| Name | Type | Description |
+| ---- |:-----| :-----------------------------------------------------------------------|
+| LogFilename | STRING |Full path of the file containing this line | 
+| Index | INTEGER | Line number |
+| Text | STRING | Text line content  |
--- a/TimberWinR/mdocs/WindowsEvents.md
+++ b/TimberWinR/mdocs/WindowsEvents.md
@@ -1,6 +1,9 @@
-
 # Input: WindowsEvents

+The WindowsEvents input will collect events from the Windows Event Viewer.   The source parameter indicates which event 
+logs to collect data from.  You can specify more than one log by using the comma, i.e.  "Application,System" will collect
+logs from the Application and System event logs.
+
 ## Parameters
 The following parameters are allowed when configuring WindowsEvents.

@@ -10,7 +13,7 @@ The following parameters are allowed when configuring WindowsEvents.
 | *binaryFormat*    | property:string |Format of the "Data" binary field.                                       | ASC,HEX,PRINT               | **ASC** |
 | *msgErrorMode*    | property:string |Behavior when event messages or event category names cannot be resolved. |NULL,ERROR,MSG               | **MSG** |
 | *direction*       | property:string |Format of the "Data" binary field.                                       | FW,BW                        | **FW**  |
-| *stringsSep*      | property:string |Separator between values of the "Strings" field.                         | any string                    | vertical bar |
+| *stringsSep*      | property:string |Separator between values of the "Strings" field.                         | any string                   | vertical bar |
 | *fullEventCode*   | property:bool   |Return the full event ID code instead of the friendly code.              | true,false                   | **false** |
 | *fullText*        | property:bool   |Retrieve the full text message                                           | true,false                   | **true** |
 | *resolveSIDS*     | property:bool   |Resolve SID values into full account names                               | true,false                   | **true** |
@@ -33,19 +36,20 @@ Example Input:
 ```
 ## Fields
 After a successful parse of an event, the following fields are added:
-|Name|Type Description|
-|EventLog|STRING Name of the Event Log or Event Log backup file containing this event 
-|RecordNumber|INTEGER|Index of this event in the Event Log or Event Log backup file containing this event 
-|TimeGenerated|TIMESTAMP|The date and time at which the event was generated (local time) 
-|TimeWritten|TIMESTAMP|The date and time at which the event was logged (local time) 
-|EventID|INTEGER|The ID of the event 
-|EventType|INTEGER|The numeric type of the event 
-|EventTypeName|STRING|The descriptive type of the event 
-|EventCategory|INTEGER|The numeric category of the event 
-|EventCategoryName|STRING|The descriptive category of the event 
-|SourceName|STRING|The source that generated the event 
-|Strings|STRING|The textual data associated with the event 
-|ComputerName|STRING|The name of the computer on which the event was generated 
-|SID|STRING|The Security Identifier associated with the event 
-|Message|STRING|The full event message 
-|Data|STRING|The binary data associated with the event 
+| Name | Type | Description |
+| ---- |:-----| :-----------------------------------------------------------------------|
+| EventLog | STRING |Name of the Event Log or Event Log backup file containing this event 
+| RecordNumber | INTEGER | Index of this event in the Event Log or Event Log backup file containing this event  |
+| TimeGenerated | TIMESTAMP | The date and time at which the event was generated (local time)  |
+| TimeWritten | TIMESTAMP | The date and time at which the event was logged (local time)  |
+| EventID | INTEGER | The ID of the event  |
+| EventType | INTEGER | The numeric type of the event  |
+| EventTypeName | STRING | The descriptive type of the event  |
+| EventCategory | INTEGER | The numeric category of the event  |
+| EventCategoryName | STRING | The descriptive category of the event  |
+| SourceName | STRING | The source that generated the event  |
+| Strings | STRING | The textual data associated with the event 
+| ComputerName | STRING | The name of the computer on which the event was generated  |
+| SID | STRING | The Security Identifier associated with the event  |
+| Message | STRING | The full event message  |
+| Data | STRING | The binary data associated with the event  |