From a99b04e1b14f7b28c53f0e5010db58fbd92cc270 Mon Sep 17 00:00:00 2001 From: Doug Schmidt Date: Mon, 31 Aug 2015 17:11:57 -0700 Subject: [PATCH] Added support for HTTPS authentication to Found clusters Elasticsearch's newly acquired found.io hosted clusters require HTTPS basic authentication in order to feed the index. This feature branch adds optional support for HTTPS according to https://www.elastic.co/guide/en/found/current/elk-and-found.html#_using_logstash When SSL is true, a non-empty username and password are required to authenticate against an Elasticsearch cluster. --- .../Parser/ElasticsearchOutputTests.cs | 50 ++++++++++++++++++- TimberWinR/Outputs/Elasticsearch.cs | 20 +++++++- TimberWinR/Parser.cs | 24 ++++++++- TimberWinR/mdocs/ElasticsearchOutput.md | 3 ++ 4 files changed, 93 insertions(+), 4 deletions(-) diff --git a/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs b/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs index 4bafebb..54111be 100644 --- a/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs +++ b/TimberWinR.UnitTests/Parser/ElasticsearchOutputTests.cs @@ -1,4 +1,6 @@ -namespace TimberWinR.UnitTests.Parser +using TimberWinR.Outputs; + +namespace TimberWinR.UnitTests.Parser { using System; @@ -51,5 +53,51 @@ Assert.AreEqual("someindex-" + DateTime.UtcNow.ToString("yyyy.MM.dd"), result); } + + [Test] + public void Given_no_ssl_then_validate_does_not_throw() + { + parser.Ssl = false; + Assert.That(() => parser.Validate(), Throws.Nothing); + } + + [Test] + public void Given_ssl_and_no_username_then_validate_throws() + { + parser.Ssl = true; + parser.Password = "pass"; + + Assert.That(() => parser.Validate(), Throws.Exception.InstanceOf()); + } + + [Test] + public void Given_ssl_and_no_password_then_validate_throws() + { + parser.Ssl = true; + parser.Username = "user"; + + Assert.That(() => parser.Validate(), Throws.Exception.InstanceOf()); + } + + [Test] + public void Given_ssl_and_username_and_password_then_validate_does_not_throw() + { + parser.Ssl = true; + parser.Username = "user"; + parser.Password = "pass"; + + Assert.That(() => parser.Validate(), Throws.Nothing); + } + + [Test] + [TestCase("host", 1234, false, null, null, "http://host:1234/")] + [TestCase("host", 1234, true, "user", "pass", "https://user:pass@host:1234/")] + [TestCase("host", 1234, true, "user:", "pass@", "https://user%3A:pass%40@host:1234/")] + public void ComposeUri_Matches_Expected(string host, int port, bool ssl, string username, string password, string expectedUri) + { + var uri = ElasticsearchOutput.ComposeUri(host, port, ssl, username, password); + + Assert.That(uri.ToString(), Is.EqualTo(expectedUri)); + } } } diff --git a/TimberWinR/Outputs/Elasticsearch.cs b/TimberWinR/Outputs/Elasticsearch.cs index 4f71cea..904a5a0 100644 --- a/TimberWinR/Outputs/Elasticsearch.cs +++ b/TimberWinR/Outputs/Elasticsearch.cs @@ -29,6 +29,9 @@ namespace TimberWinR.Outputs { private TimberWinR.Manager _manager; private readonly int _port; + private readonly bool _ssl; + private readonly string _username; + private readonly string _password; private readonly int _interval; private readonly int _flushSize; private readonly int _idleFlushTimeSeconds; @@ -57,8 +60,8 @@ namespace TimberWinR.Outputs var nodes = new List(); foreach (var host in _hosts) { - var url = string.Format("http://{0}:{1}", host, _port); - nodes.Add(new Uri(url)); + var uri = ComposeUri(host, _port, _ssl, _username, _password); + nodes.Add(uri); } var pool = new StaticConnectionPool(nodes.ToArray()); var settings = new ConnectionSettings(pool) @@ -73,6 +76,13 @@ namespace TimberWinR.Outputs return client; } + public static Uri ComposeUri(string host, int port, bool ssl, string username, string password) + { + return ssl + ? new Uri(string.Format("https://{0}:{1}@{2}:{3}", Uri.EscapeDataString(username), Uri.EscapeDataString(password), host, port)) + : new Uri(string.Format("http://{0}:{1}", host, port)); + } + public ElasticsearchOutput(TimberWinR.Manager manager, Parser.ElasticsearchOutputParameters parameters, CancellationToken cancelToken) : base(cancelToken, "Elasticsearch") { @@ -86,6 +96,9 @@ namespace TimberWinR.Outputs _timeout = parameters.Timeout; _manager = manager; _port = parameters.Port; + _ssl = parameters.Ssl; + _username = parameters.Username; + _password = parameters.Password; _interval = parameters.Interval; _hosts = parameters.Host; _jsonQueue = new List(); @@ -111,6 +124,9 @@ namespace TimberWinR.Outputs new JProperty("messages", _sentMessages), new JProperty("queuedMessageCount", _jsonQueue.Count), new JProperty("port", _port), + new JProperty("ssl", _ssl), + new JProperty("username", _username), + new JProperty("password", _password), new JProperty("flushSize", _flushSize), new JProperty("idleFlushTime", _idleFlushTimeSeconds), new JProperty("interval", _interval), diff --git a/TimberWinR/Parser.cs b/TimberWinR/Parser.cs index 765d0b9..6824472 100644 --- a/TimberWinR/Parser.cs +++ b/TimberWinR/Parser.cs @@ -625,8 +625,16 @@ namespace TimberWinR.Parser } - public class ElasticsearchOutputParameters + public class ElasticsearchOutputParameters : IValidateSchema { + public class ElasticsearchBasicAuthException : Exception + { + public ElasticsearchBasicAuthException() + : base("Elasticsearch 'username' and 'password' properties must be set when SSL is enabled.") + { + } + } + const string IndexDatePattern = "(%\\{(?[^\\}]+)\\})"; [JsonProperty(PropertyName = "host")] @@ -635,6 +643,12 @@ namespace TimberWinR.Parser public string Index { get; set; } [JsonProperty(PropertyName = "port")] public int Port { get; set; } + [JsonProperty(PropertyName = "ssl")] + public bool Ssl { get; set; } + [JsonProperty(PropertyName = "username")] + public string Username { get; set; } + [JsonProperty(PropertyName = "password")] + public string Password { get; set; } [JsonProperty(PropertyName = "timeout")] public int Timeout { get; set; } [JsonProperty(PropertyName = "threads")] @@ -662,6 +676,9 @@ namespace TimberWinR.Parser IdleFlushTimeInSeconds = 10; Protocol = "http"; Port = 9200; + Ssl = false; + Username = string.Empty; + Password = string.Empty; Index = ""; Host = new string[] { "localhost" }; Timeout = 10000; @@ -711,6 +728,11 @@ namespace TimberWinR.Parser return typeName; } + public void Validate() + { + if (Ssl && (string.IsNullOrWhiteSpace(Username) || string.IsNullOrWhiteSpace(Password))) + throw new ElasticsearchBasicAuthException(); + } } public class RedisOutputParameters diff --git a/TimberWinR/mdocs/ElasticsearchOutput.md b/TimberWinR/mdocs/ElasticsearchOutput.md index 298236c..0db103f 100644 --- a/TimberWinR/mdocs/ElasticsearchOutput.md +++ b/TimberWinR/mdocs/ElasticsearchOutput.md @@ -14,6 +14,9 @@ The following parameters are allowed when configuring the Elasticsearch output. | *interval* | integer | Interval in milliseconds to sleep during batch sends | Interval | 5000 | | *max_queue_size* | integer | Maximum Elasticsearch queue depth | | 50000 | | *port* | integer | Elasticsearch port number | This port must be open | 9200 | +| *ssl* | bool | If true, use an HTTPS connection to Elasticsearch. See [this page] (https://www.elastic.co/guide/en/found/current/elk-and-found.html#_using_logstash) for a configuration example. | *username* and *password* are also required for HTTPS connections. | false | +| *username* | string | Username for Elasticsearch credentials. | Required for HTTPS connection. | | +| *password* | string | Password for Elasticsearch credentials. | Required for HTTPS connection. | | | *queue_overflow_discard_oldest* | bool | If true, discard oldest messages when max_queue_size reached otherwise discard newest | | true | | *threads* | [string] | Number of Threads | Number of worker threads processing messages | 1 | | *enable_ping* | bool | If true, pings the server to test for keep alive | | false |