diff --git a/controllers/mcrouter_controller.go b/controllers/mcrouter_controller.go index 3d606cab..567b387f 100755 --- a/controllers/mcrouter_controller.go +++ b/controllers/mcrouter_controller.go @@ -179,15 +179,18 @@ func (r *McrouterReconciler) ReconcilePrometheusRule(ctx context.Context, req ct Name("mcrouter-rule"). Rules( builders.Rule(). - Alert("McrouterBackendDown"). - Message("Backend Memcached servers are down."). - Priority(1). - Expr("mcrouter_servers{state='down'}!=0"), + Alert("McrouterDown"). + Priority(2). + Expr("mcrouter_up != 1"), + builders.Rule(). - Alert("McrouterBackendTimeout"). - Message("Backend Memcached servers are timeout."). - Priority(1). - Expr("mcrouter_server_memcached_timeout_count>0"), + Alert("McrouterAllBackendsDown"). + Priority(2). + Expr("mcrouter_servers{state='up'} / mcrouter_servers == 0"), + builders.Rule(). + Alert("McrouterBackendDown"). + Priority(3). + Expr("mcrouter_servers{state='down'} != 0"), ). Interval("1m")). Build() diff --git a/controllers/memcached_controller.go b/controllers/memcached_controller.go index 2945375a..0d0f96ab 100755 --- a/controllers/memcached_controller.go +++ b/controllers/memcached_controller.go @@ -159,12 +159,19 @@ func (r *MemcachedReconciler) ReconcilePrometheusRule(ctx context.Context, req c RuleGroups(builders.RuleGroup(). Name("memcached-rule"). Rules( + builders.Rule(). + Alert("MemcachedDown"). + Priority(3). + Expr("memcached_up == 0"), builders.Rule(). - Alert("MemcachedConnectionLimit"). - Message("This memcached connection is over max."). - Priority(1). - Expr("memcached_current_connections/memcached_max_connections*100 >90"), + Alert("MemcachedMaxConnections"). + Priority(3). + Expr("memcached_current_connections/memcached_max_connections * 100 > 95"), + builders.Rule(). + Alert("MemcachedMaxConnections"). + Priority(4). + Expr("memcached_current_connections/memcached_max_connections * 100 > 90"), ). Interval("1m")). Build()