diff --git a/controllers/mcrouter_controller.go b/controllers/mcrouter_controller.go index be7e58a5..2450ceb9 100755 --- a/controllers/mcrouter_controller.go +++ b/controllers/mcrouter_controller.go @@ -165,15 +165,18 @@ func (r *McrouterReconciler) ReconcilePrometheusRule(ctx context.Context, req ct Name("mcrouter-rule"). Rules( builders.Rule(). - Alert("McrouterBackendDown"). - Message("Backend Memcached servers are down."). - Priority(1). - Expr("mcrouter_servers{state='down'}!=0"), + Alert("McrouterDown"). + Priority(2). + Expr("mcrouter_up != 1"), + builders.Rule(). - Alert("McrouterBackendTimeout"). - Message("Backend Memcached servers are timeout."). - Priority(1). - Expr("mcrouter_server_memcached_timeout_count>0"), + Alert("McrouterAllBackendsDown"). + Priority(2). + Expr("mcrouter_servers{state='up'} / mcrouter_servers == 0"), + builders.Rule(). + Alert("McrouterBackendDown"). + Priority(3). + Expr("mcrouter_servers{state='down'} != 0"), ). Interval("1m")). Build() diff --git a/controllers/memcached_controller.go b/controllers/memcached_controller.go index c6c77aed..43390f5c 100755 --- a/controllers/memcached_controller.go +++ b/controllers/memcached_controller.go @@ -161,12 +161,19 @@ func (r *MemcachedReconciler) ReconcilePrometheusRule(ctx context.Context, req c RuleGroups(builders.RuleGroup(). Name("memcached-rule"). Rules( + builders.Rule(). + Alert("MemcachedDown"). + Priority(3). + Expr("memcached_up == 0"), builders.Rule(). - Alert("MemcachedConnectionLimit"). - Message("This memcached connection is over max."). - Priority(1). - Expr("memcached_current_connections/memcached_max_connections*100 >90"), + Alert("MemcachedMaxConnections"). + Priority(3). + Expr("memcached_current_connections/memcached_max_connections * 100 > 95"), + builders.Rule(). + Alert("MemcachedMaxConnections"). + Priority(4). + Expr("memcached_current_connections/memcached_max_connections * 100 > 90"), ). Interval("1m")). Build()