From 6405ae32f940bcdef727bc5a355f118d5fa9bf5e Mon Sep 17 00:00:00 2001
From: Clark Boylan <clark.boylan@gmail.com>
Date: Thu, 30 May 2013 12:11:05 -0700
Subject: [PATCH] Add swift logs to Logstash.
* modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml:
Add the swift log files to the list of files to be processed by
logstash-worker1. This does not include the swift proxy log file as it
will require extra parsing.
* modules/openstack_project/templates/logstash/indexer.conf.erb: Add
Logstash filters for apache combined log format files.
Change-Id: I7545ace8f7601bdca453e0d2ac1b2233823878ce
Reviewed-on: https://review.openstack.org/31103
Reviewed-by: Jeremy Stanley <fungi@yuggoth.org>
Approved: James E. Blair <corvus@inaugust.com>
Reviewed-by: James E. Blair <corvus@inaugust.com>
Reviewed-by: Clark Boylan <clark.boylan@gmail.com>
Tested-by: Jenkins
---
.../logstash-worker1/jenkins-log-pusher.yaml | 14 +++++++++++++-
.../templates/logstash/indexer.conf.erb | 8 +++++++-
2 files changed, 20 insertions(+), 2 deletions(-)
diff --git a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
index 287bc3ff6d..7256e31de7 100644
--- a/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
+++ b/modules/openstack_project/files/logstash/logstash-worker1/jenkins-log-pusher.yaml
@@ -30,7 +30,19 @@ source-files:
tags:
- screen
- keystonefmt
+ - name: logs/screen-s-account.txt
+ tags:
+ - screen
+ - apachecombined
+ - name: logs/screen-s-container.txt
+ tags:
+ - screen
+ - apachecombined
+ - name: logs/screen-s-object.txt
+ tags:
+ - screen
+ - apachecombined
+# TODO(clarkb) Add swift proxy logs here.
- name: logs/syslog.txt
tags:
- syslog
-# TODO(clarkb) Add swift logs here.
diff --git a/modules/openstack_project/templates/logstash/indexer.conf.erb b/modules/openstack_project/templates/logstash/indexer.conf.erb
index bd651b97e6..7b7f874fda 100644
--- a/modules/openstack_project/templates/logstash/indexer.conf.erb
+++ b/modules/openstack_project/templates/logstash/indexer.conf.erb
@@ -84,6 +84,12 @@ filter {
pattern => [ "(?m)^\(\b%{NOTSPACE:module}\b\):%{SPACE}%{DATESTAMP:logdate}%{SPACE}(?<loglevel>AUDIT|CRITICAL|DEBUG|INFO|TRACE|WARNING|ERROR)%{SPACE}%{GREEDYDATA:logmessage}" ]
add_field => [ "received_at", "%{@timestamp}" ]
}
+ grok {
+ type => "jenkins"
+ tags => ["apachecombined"]
+ pattern => [ "%{COMBINEDAPACHELOG}" ]
+ add_field => [ "received_at", "%{@timestamp}", "logdate", "%{timestamp}", "logmessage", "%{verb} %{request} %{response}" ]
+ }
grok {
type => "jenkins"
tags => ["syslog"]
@@ -97,7 +103,7 @@ filter {
date {
type => "jenkins"
exclude_tags => "_grokparsefailure"
- match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss" ]
+ match => [ "logdate", "yyyy-MM-dd HH:mm:ss.SSS", "yyyy-MM-dd HH:mm:ss,SSS", "yyyy-MM-dd HH:mm:ss", "MMM d HH:mm:ss", "MMM dd HH:mm:ss", "dd/MMM/yyyy:HH:mm:ss Z" ]
}
mutate {
type => "jenkins"