Merge "Support configuration of custom fluentd filters"

ansible/roles/common/handlers/main.yml

@@ -21,6 +21,7 @@
       or fluentd_output_custom.changed | bool
       or fluentd_format.changed | bool
       or fluentd_filter.changed | bool
+      or fluentd_filter_custom.changed | bool
       or fluentd_td_agent.changed | bool
       or fluentd_container.changed | bool
 

ansible/roles/common/tasks/config.yml

@@ -124,6 +124,23 @@
   notify:
     - Restart fluentd container
 
+- name: Find custom fluentd filter config files
+  local_action:
+    module: find
+    path: "{{ node_custom_config }}/fluentd/filter"
+    pattern: "*.conf"
+  run_once: True
+  register: find_custom_fluentd_filters
+
+- name: Copying over custom fluentd filter config files
+  template:
+    src: "{{ item.path }}"
+    dest: "{{ node_config_directory }}/fluentd/filter/{{ item.path | basename }}"
+  register: fluentd_filter_custom
+  with_items: "{{ find_custom_fluentd_filters.files }}"
+  notify:
+    - Restart fluentd container
+
 - name: Copying over td-agent.conf
   template:
     src: "td-agent.conf.j2"

ansible/roles/common/templates/fluentd.json.j2

@@ -47,15 +47,11 @@
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },
+        {# Copy all configuration files in filter/ directory to include #}
+        {# custom filter configs. #}
         {
-            "source": "{{ container_config_directory }}/filter/00-record_transformer.conf",
+            "source": "{{ container_config_directory }}/filter/*.conf",
-            "dest": "{{ fluentd_dir }}/filter/00-record_transformer.conf",
+            "dest": "{{ fluentd_dir }}/filter/",
-            "owner": "{{ fluentd_user }}",
-            "perm": "0600"
-        },
-        {
-            "source": "{{ container_config_directory }}/filter/01-rewrite.conf",
-            "dest": "{{ fluentd_dir }}/filter/01-rewrite.conf",
             "owner": "{{ fluentd_user }}",
             "perm": "0600"
         },

doc/source/reference/central-logging-guide.rst

@@ -155,6 +155,18 @@ can also be exported at once by choosing *export everything* option.
 In the same tab (Settings - Objects) one can also import saved items by
 choosing *import* option.
 
+Custom log filtering
+====================
+
+In some scenarios it may be useful to apply custom filters to logs before
+forwarding them.  This may be useful to add additional tags to the messages
+or to modify the tags to conform to a log format that differs from the one
+defined by kolla-ansible.
+
+Configuration of custom fluentd filters is possible by placing filter
+configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
+control host.
+
 Custom log forwarding
 =====================
 

releasenotes/notes/fluentd-custom-filter-config-26ee8801c7782b08.yaml

@@ -0,0 +1,13 @@
+---
+features:
+  - |
+    Adds ability to configure custom fluentd filters.
+
+    In some scenarios it may be useful to apply custom filters to logs before
+    forwarding them.  This may be useful to add additional tags to the messages
+    or to modify the tags to conform to a log format that differs from the one
+    defined by kolla-ansible.
+
+    Configuration of custom fluentd filters is possible by placing filter
+    configuration files in ``/etc/kolla/config/fluentd/filter/*.conf`` on the
+    control host.