Merge pull request #14 from gediminasgu/iislog_timestamp_col

Elasticsearch output index name custom format

TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs

@@ -0,0 +1,55 @@
+namespace TimberWinR.UnitTests.Parser
+{
+    using System;
+
+    using Newtonsoft.Json.Linq;
+
+    using NUnit.Framework;
+
+    using TimberWinR.Parser;
+
+    public class ElasticsearchOutputTests
+    {
+        private ElasticsearchOutput parser;
+
+        [SetUp]
+        public void Setup()
+        {
+            this.parser = new ElasticsearchOutput();
+        }
+
+        [Test]
+        public void Given_no_index_returns_default_index_name()
+        {
+            this.parser.Index = "someindex";
+            var json = new JObject();
+
+            var result = this.parser.GetIndexName(json);
+
+            Assert.AreEqual("someindex", result);
+        }
+
+        [Test]
+        public void Given_index_with_date_format_and_timestamp_returns_name_by_timestamp()
+        {
+            this.parser.Index = "someindex-%{yyyy.MM.dd}";
+            var json = new JObject();
+            json.Add(new JProperty("@timestamp", "2011-11-30T18:45:32.450Z"));
+
+            var result = this.parser.GetIndexName(json);
+
+            Assert.AreEqual("someindex-2011.11.30", result);
+        }
+
+        [Test]
+        public void Given_index_with_date_format_and_no_timestamp_returns_name_by_current_date()
+        {
+            this.parser.Index = "someindex-%{yyyy.MM.dd}";
+            var json = new JObject();
+
+            var result = this.parser.GetIndexName(json);
+
+            Assert.AreEqual("someindex-" + DateTime.UtcNow.ToString("yyyy.MM.dd"), result);
+        }
+    }
+}

TimberWinR.UnitTests/TimberWinR.UnitTests.csproj

@@ -64,6 +64,7 @@
     <Compile Include="Inputs\IisW3CRowReaderTests.cs" />
     <Compile Include="JsonFilterTests.cs" />
     <Compile Include="GrokFilterTests.cs" />
+    <Compile Include="Parser\ElasticsearchOutputTests.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
     <Compile Include="TestBase.cs" />
   </ItemGroup>

TimberWinR/Outputs/Elasticsearch.cs

@@ -2,7 +2,6 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Net;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using Newtonsoft.Json.Linq;
@@ -11,6 +10,8 @@ using RestSharp;
 
 namespace TimberWinR.Outputs
 {
+    using System.Text.RegularExpressions;
+
     public partial class ElasticsearchOutput : OutputSender
     {
         private TimberWinR.Manager _manager;
@@ -18,7 +19,6 @@ namespace TimberWinR.Outputs
         private readonly int _interval;
         private readonly string[] _host;
         private readonly string _protocol;
-        private readonly string _index;
         private int _hostIndex;
         private readonly int _timeout;
         private readonly object _locker = new object();
@@ -26,6 +26,7 @@ namespace TimberWinR.Outputs
         private readonly int _numThreads;
         private long _sentMessages;
         private long _errorCount;
+        private Parser.ElasticsearchOutput eo;
 
         public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken)
             : base(cancelToken, "Elasticsearch")
@@ -33,13 +34,13 @@ namespace TimberWinR.Outputs
             _sentMessages = 0;
             _errorCount = 0;
 
+            this.eo = eo;
             _protocol = eo.Protocol;
             _timeout = eo.Timeout;
             _manager = manager;
             _port = eo.Port;
             _interval = eo.Interval;
             _host = eo.Host;
-            _index = eo.Index;
             _hostIndex = 0;
             _jsonQueue = new List<JObject>();
             _numThreads = eo.NumThreads;
@@ -102,18 +103,8 @@ namespace TimberWinR.Outputs
 
                                 foreach (JObject json in messages)
                                 {
-                                    string typeName = "Win32-Elasticsearch";
-                                    if (json["type"] != null)
-                                        typeName = json["type"].ToString();
-
-                                    ////check if the submitted JSON object provides a custom index. If yes, use this one
-                                    var token = json["_index"];
-                                    string indexName = token == null ? _index : token.Value<string>();
-                         
-                                    if (string.IsNullOrEmpty(indexName))
-                                    {
-                                        indexName = string.Format("logstash-{0}", DateTime.UtcNow.ToString("yyyy.MM.dd"));
-                                    }
+                                    var typeName = this.eo.GetTypeName(json);
+                                    var indexName = this.eo.GetIndexName(json);
                                     var req = new RestRequest(string.Format("/{0}/{1}/", indexName, typeName), Method.POST);                                  
 
                                     req.AddParameter("text/json", json.ToString(), ParameterType.RequestBody);

TimberWinR/Parser.cs

@@ -17,6 +17,8 @@ using System.CodeDom.Compiler;
 
 namespace TimberWinR.Parser
 {
+    using System.Text.RegularExpressions;
+
     interface IValidateSchema
     {
         void Validate();
@@ -424,6 +426,8 @@ namespace TimberWinR.Parser
 
     public class ElasticsearchOutput
     {
+        const string IndexDatePattern = "(%\\{(?<format>[^\\}]+)\\})";
+
         [JsonProperty(PropertyName = "host")]
         public string[] Host { get; set; }
         [JsonProperty(PropertyName = "index")]
@@ -449,6 +453,45 @@ namespace TimberWinR.Parser
             NumThreads = 1;
             Interval = 1000;
         }
+
+        public string GetIndexName(JObject json)
+        {
+            ////check if the submitted JSON object provides a custom index. If yes, use this one
+            var token = json["_index"];
+            var indexName = token == null ? this.Index : token.Value<string>();
+
+            if (string.IsNullOrEmpty(indexName))
+            {
+                indexName = string.Format("logstash-{0}", DateTime.UtcNow.ToString("yyyy.MM.dd"));
+            }
+            else
+            {
+                var date = DateTime.UtcNow;
+                if (json["@timestamp"] != null)
+                {
+                    date = DateTime.Parse(json["@timestamp"].ToString());
+                }
+
+                var match = Regex.Match(indexName, IndexDatePattern);
+                if (match.Success)
+                {
+                    indexName = Regex.Replace(indexName, IndexDatePattern, date.ToString(match.Groups["format"].Value));
+                }
+            }
+
+            return indexName;
+        }
+
+        public string GetTypeName(JObject json)
+        {
+            string typeName = "Win32-Elasticsearch";
+            if (json["type"] != null)
+            {
+                typeName = json["type"].ToString();
+            }
+            return typeName;
+        }
+
     }
 
     public class RedisOutput

TimberWinR/mdocs/ElasticsearchOutput.md

@@ -13,6 +13,9 @@ The following parameters are allowed when configuring the Redis output.
 | *host*        | [string] | The hostname(s) of your Elasticsearch server(s) | IP or DNS name |  |
 | *port*        | integer  | Redis port number                                           | This port must be open  | 9200  |
 
+### Index parameter
+If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need.
+
 Example Input:
 ```json
 {

appveyor.yml

@@ -12,4 +12,3 @@ assembly_info:
 
 artifacts:
   - path: '**\*.msi'
-