From 16af26d5589a5ca01f48df6556049d1124becbcf Mon Sep 17 00:00:00 2001 From: Clark Boylan Date: Tue, 12 Nov 2019 12:27:42 -0800 Subject: [PATCH] Timeout cleanup playbook tasks This adds timeouts to cleanup playbook tasks so that if they have problems running the entire job does not stall. We noticed that df in particular can hang on broken nfs mounts and if that happens the job's cleanup playbook does not end (even after a couple days). This change adds the timeouts to the testing playbook. If testing shows this works well we will add it to the production playbooks too. Change-Id: Ibc875bf99e6da29e2fffbddae8590ecde06b5c3b --- playbooks/base-test/cleanup.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/playbooks/base-test/cleanup.yaml b/playbooks/base-test/cleanup.yaml index 719e6d2..2aea1ce 100644 --- a/playbooks/base-test/cleanup.yaml +++ b/playbooks/base-test/cleanup.yaml @@ -4,13 +4,13 @@ when: not (zuul_success | bool) block: - name: get df disk usage - raw: df + raw: timeout -k 5 90 df - name: get df inode usage - raw: df -i + raw: timeout -k 5 90 df -i - name: get l2 networking - raw: ip link + raw: timeout -k 5 90 ip link - name: get l3 networking - raw: ip addr + raw: timeout -k 5 90 ip addr