diff --git a/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs b/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs
new file mode 100644
index 0000000..e59914a
--- /dev/null
+++ b/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs
@@ -0,0 +1,55 @@
+namespace TimberWinR.UnitTests.Parser
+{
+ using System;
+
+ using Newtonsoft.Json.Linq;
+
+ using NUnit.Framework;
+
+ using TimberWinR.Parser;
+
+ public class ElasticsearchOutputTests
+ {
+ private ElasticsearchOutput parser;
+
+ [SetUp]
+ public void Setup()
+ {
+ this.parser = new ElasticsearchOutput();
+ }
+
+ [Test]
+ public void Given_no_index_returns_default_index_name()
+ {
+ this.parser.Index = "someindex";
+ var json = new JObject();
+
+ var result = this.parser.GetIndexName(json);
+
+ Assert.AreEqual("someindex", result);
+ }
+
+ [Test]
+ public void Given_index_with_date_format_and_timestamp_returns_name_by_timestamp()
+ {
+ this.parser.Index = "someindex-%{yyyy.MM.dd}";
+ var json = new JObject();
+ json.Add(new JProperty("@timestamp", "2011-11-30T18:45:32.450Z"));
+
+ var result = this.parser.GetIndexName(json);
+
+ Assert.AreEqual("someindex-2011.11.30", result);
+ }
+
+ [Test]
+ public void Given_index_with_date_format_and_no_timestamp_returns_name_by_current_date()
+ {
+ this.parser.Index = "someindex-%{yyyy.MM.dd}";
+ var json = new JObject();
+
+ var result = this.parser.GetIndexName(json);
+
+ Assert.AreEqual("someindex-" + DateTime.UtcNow.ToString("yyyy.MM.dd"), result);
+ }
+ }
+}
diff --git a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
index 499d5bf..ddbc8cd 100644
--- a/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
+++ b/TimberWinR.UnitTests/TimberWinR.UnitTests.csproj
@@ -64,6 +64,7 @@
+
diff --git a/TimberWinR/Outputs/Elasticsearch.cs b/TimberWinR/Outputs/Elasticsearch.cs
index 13c9849..0b80e47 100644
--- a/TimberWinR/Outputs/Elasticsearch.cs
+++ b/TimberWinR/Outputs/Elasticsearch.cs
@@ -2,7 +2,6 @@
using System.Collections.Generic;
using System.Linq;
using System.Net;
-using System.Text;
using System.Threading;
using System.Threading.Tasks;
using Newtonsoft.Json.Linq;
@@ -11,6 +10,8 @@ using RestSharp;
namespace TimberWinR.Outputs
{
+ using System.Text.RegularExpressions;
+
public partial class ElasticsearchOutput : OutputSender
{
private TimberWinR.Manager _manager;
@@ -18,7 +19,6 @@ namespace TimberWinR.Outputs
private readonly int _interval;
private readonly string[] _host;
private readonly string _protocol;
- private readonly string _index;
private int _hostIndex;
private readonly int _timeout;
private readonly object _locker = new object();
@@ -26,6 +26,7 @@ namespace TimberWinR.Outputs
private readonly int _numThreads;
private long _sentMessages;
private long _errorCount;
+ private Parser.ElasticsearchOutput eo;
public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutput eo, CancellationToken cancelToken)
: base(cancelToken, "Elasticsearch")
@@ -33,13 +34,13 @@ namespace TimberWinR.Outputs
_sentMessages = 0;
_errorCount = 0;
+ this.eo = eo;
_protocol = eo.Protocol;
_timeout = eo.Timeout;
_manager = manager;
_port = eo.Port;
_interval = eo.Interval;
_host = eo.Host;
- _index = eo.Index;
_hostIndex = 0;
_jsonQueue = new List();
_numThreads = eo.NumThreads;
@@ -102,18 +103,8 @@ namespace TimberWinR.Outputs
foreach (JObject json in messages)
{
- string typeName = "Win32-Elasticsearch";
- if (json["type"] != null)
- typeName = json["type"].ToString();
-
- ////check if the submitted JSON object provides a custom index. If yes, use this one
- var token = json["_index"];
- string indexName = token == null ? _index : token.Value();
-
- if (string.IsNullOrEmpty(indexName))
- {
- indexName = string.Format("logstash-{0}", DateTime.UtcNow.ToString("yyyy.MM.dd"));
- }
+ var typeName = this.eo.GetTypeName(json);
+ var indexName = this.eo.GetIndexName(json);
var req = new RestRequest(string.Format("/{0}/{1}/", indexName, typeName), Method.POST);
req.AddParameter("text/json", json.ToString(), ParameterType.RequestBody);
diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs
index 6f77ebe..33e01df 100644
--- a/TimberWinR/Parser.cs
+++ b/TimberWinR/Parser.cs
@@ -17,6 +17,8 @@ using System.CodeDom.Compiler;
namespace TimberWinR.Parser
{
+ using System.Text.RegularExpressions;
+
interface IValidateSchema
{
void Validate();
@@ -424,6 +426,8 @@ namespace TimberWinR.Parser
public class ElasticsearchOutput
{
+ const string IndexDatePattern = "(%\\{(?[^\\}]+)\\})";
+
[JsonProperty(PropertyName = "host")]
public string[] Host { get; set; }
[JsonProperty(PropertyName = "index")]
@@ -449,6 +453,45 @@ namespace TimberWinR.Parser
NumThreads = 1;
Interval = 1000;
}
+
+ public string GetIndexName(JObject json)
+ {
+ ////check if the submitted JSON object provides a custom index. If yes, use this one
+ var token = json["_index"];
+ var indexName = token == null ? this.Index : token.Value();
+
+ if (string.IsNullOrEmpty(indexName))
+ {
+ indexName = string.Format("logstash-{0}", DateTime.UtcNow.ToString("yyyy.MM.dd"));
+ }
+ else
+ {
+ var date = DateTime.UtcNow;
+ if (json["@timestamp"] != null)
+ {
+ date = DateTime.Parse(json["@timestamp"].ToString());
+ }
+
+ var match = Regex.Match(indexName, IndexDatePattern);
+ if (match.Success)
+ {
+ indexName = Regex.Replace(indexName, IndexDatePattern, date.ToString(match.Groups["format"].Value));
+ }
+ }
+
+ return indexName;
+ }
+
+ public string GetTypeName(JObject json)
+ {
+ string typeName = "Win32-Elasticsearch";
+ if (json["type"] != null)
+ {
+ typeName = json["type"].ToString();
+ }
+ return typeName;
+ }
+
}
public class RedisOutput
diff --git a/TimberWinR/mdocs/ElasticsearchOutput.md b/TimberWinR/mdocs/ElasticsearchOutput.md
index 6a47948..3067cbb 100644
--- a/TimberWinR/mdocs/ElasticsearchOutput.md
+++ b/TimberWinR/mdocs/ElasticsearchOutput.md
@@ -13,6 +13,9 @@ The following parameters are allowed when configuring the Redis output.
| *host* | [string] | The hostname(s) of your Elasticsearch server(s) | IP or DNS name | |
| *port* | integer | Redis port number | This port must be open | 9200 |
+### Index parameter
+If you want to output your data everyday to a new index, use following index format: "index-%{yyyy.MM.dd}". Here date format could be any forwat which you need.
+
Example Input:
```json
{