TimberWinR/TimberWinR/Filters/GrokFilter.cs

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading;
using Microsoft.CSharp;
using Newtonsoft.Json.Linq;
using NLog;
using RapidRegex.Core;

namespace TimberWinR.Parser
{
    public class Fields
    {
        private Dictionary<string, string> fields { get; set; }

        public string this[string i]
        {
            get { return fields[i]; }
            set { fields[i] = value; }
        }

        public Fields(JObject json)
        {
            fields = new Dictionary<string, string>();
            IList<string> keys = json.Properties().Select(p => p.Name).ToList();
            foreach (string key in keys)
                fields[key] = json[key].ToString();
        }
    }

    public partial class Grok : LogstashFilter
    {
        public override JObject ToJson()
        {
            string field = Match[0];
            string expr = Match[1];

            JObject json = new JObject(
                 new JProperty("grok",
                     new JObject(
                         new JProperty("match", field),
                         new JProperty("expr", expr),
                         new JProperty("condition", Condition),
                         new JProperty("addfields", AddField),
                         new JProperty("addtags", AddTag),
                         new JProperty("drop", DropIfMatch),
                         new JProperty("type", Type),
                         new JProperty("removefields", RemoveField),
                         new JProperty("removetag", RemoveTag)
                         )));
            return json;
        }

        // Returns: true - Filter does not apply or has been applied successfully
        // Returns: false - Drop this object
        public override bool Apply(JObject json)
        {
            if (!string.IsNullOrEmpty(Type))
            {
                JToken json_type = json["type"];
                if (json_type != null && json_type.ToString() != Type)
                    return true; // Filter does not apply.
            }

            if (Matches(json))
            {
                if (Condition != null)
                {
                    var expr = EvaluateCondition(json, Condition);
                    if (expr)
                    {
                        if (DropIfMatch)
                            return false; // drop this one
                    }
                    else
                        return true;
                }

                if (DropIfMatch)
                    return false;

                AddFields(json);
                AddTags(json);
                RemoveFields(json);
                RemoveTags(json);
            }

            return true;
        }

        // Test for any true matching condition(s)
        private bool Matches(Newtonsoft.Json.Linq.JObject json)
        {
            for (int i = 0; i < Match.Length; i += 2)
            {
                string field = Match[i];
                string expr = Match[i + 1];

                JToken token = null;
                if (json.TryGetValue(field, out token))
                {
                    string text = token.ToString();
                    if (!string.IsNullOrEmpty(text))
                    {
                        var resolver = new RegexGrokResolver();
                        var pattern = resolver.ResolveToRegex(expr);
                        var match = Regex.Match(text, pattern);
                        if (match.Success)
                        {
                            var regex = new Regex(pattern);
                            var namedCaptures = regex.MatchNamedCaptures(text);
                            foreach (string fieldName in namedCaptures.Keys)
                            {
                                AddOrModify(json, fieldName, namedCaptures[fieldName]);
                            }
                            return true; // Yes!
                        }
                    }
                    if (string.IsNullOrEmpty(expr))
                        return true; // Empty field is no match                 
                }
            }
            return false; // Not specified is failure
        }

        private void AddFields(Newtonsoft.Json.Linq.JObject json)
        {
            if (AddField != null && AddField.Length > 0)
            {
                for (int i = 0; i < AddField.Length; i += 2)
                {
                    string fieldName = ExpandField(AddField[i], json);
                    string fieldValue = ExpandField(AddField[i + 1], json);
                    AddOrModify(json, fieldName, fieldValue);
                }
            }
        }       

        private void RemoveFields(Newtonsoft.Json.Linq.JObject json)
        {
            if (RemoveField != null && RemoveField.Length > 0)
            {
                for (int i = 0; i < RemoveField.Length; i++)
                {
                    string fieldName = ExpandField(RemoveField[i], json);
                    RemoveProperties(json, new string[] { fieldName });
                }
            }
        }

        private void AddTags(Newtonsoft.Json.Linq.JObject json)
        {
            if (AddTag != null && AddTag.Length > 0)
            {
                for (int i = 0; i < AddTag.Length; i++)
                {
                    string value = ExpandField(AddTag[i], json);

                    JToken tags = json["tags"];
                    if (tags == null)
                        json.Add("tags", new JArray(value));
                    else
                    {
                        JArray a = tags as JArray;
                        a.Add(value);
                    }
                }
            }
        }

        private void RemoveTags(Newtonsoft.Json.Linq.JObject json)
        {
            if (RemoveTag != null && RemoveTag.Length > 0)
            {
                JToken tags = json["tags"];
                if (tags != null)
                {
                    List<JToken> children = tags.Children().ToList();
                    for (int i = 0; i < RemoveTag.Length; i++)
                    {
                        string tagName = ExpandField(RemoveTag[i], json);
                        foreach (JToken token in children)
                        {
                            if (token.ToString() == tagName)
                                token.Remove();
                        }
                    }
                }
            }
        }
    }
}