Add support for Zuulv3-specific parameters in elastic-recheck

This commit ensures elastic-recheck is able to support zuul v2 and v3
simultaneously:

- Add message queries based on v3 completion messages
- Include job-output.txt where console.html was expected

Change-Id: If3d7990d892a9698a112d9ff1dd5160998a4efe6
Depends-On: I7e34206d7968bf128e140468b9a222ecbce3a8f1

elastic_recheck/cmd/check_success.py

@@ -24,6 +24,7 @@ import time
 
 from launchpadlib import launchpad
 
+import elastic_recheck.config as er_config
 import elastic_recheck.elasticRecheck as er
 import elastic_recheck.results as er_results
 
@@ -51,17 +52,14 @@ def all_fails(classifier):
     so we can figure out how good we are doing on total classification.
     """
     all_fails = {}
-    query = ('filename:"console.html" '
+    results = classifier.hits_by_query(er_config.ALL_FAILS_QUERY, size=30000)
-             'AND message:"Finished: FAILURE" '
-             'AND build_queue:"gate"')
-    results = classifier.hits_by_query(query, size=30000)
     facets = er_results.FacetSet()
     facets.detect_facets(results, ["build_uuid"])
     for build in facets:
         for result in facets[build]:
             # not perfect, but basically an attempt to show the integrated
             # gate. Would be nice if there was a zuul attr for this in es.
-            if re.search("(^openstack/|devstack|grenade)", result.project):
+            if re.search(er_config.INCLUDED_PROJECTS_REGEX, result.project):
                 all_fails["%s.%s" % (build, result.build_name)] = False
     return all_fails
 

elastic_recheck/cmd/uncategorized_fails.py

@@ -125,7 +125,10 @@ def all_fails(classifier, config=None):
             if result.project in integrated_gate_projects:
                 name = result.build_name
                 timestamp = dp.parse(result.timestamp)
-                log = result.log_url.split("console.html")[0]
+                if 'console.html' in result.log_url:
+                    log = result.log_url.split('console.html')[0]
+                elif 'job-output.txt' in result.log_url:
+                    log = result.log_url.split('job-output.txt')[0]
                 integrated_fails["%s.%s" % (build, name)] = {
                     'log': log,
                     'timestamp': timestamp,
@@ -137,7 +140,10 @@ def all_fails(classifier, config=None):
                 if re.search(config.included_projects_regex, result.project):
                     name = result.build_name
                     timestamp = dp.parse(result.timestamp)
-                    log = result.log_url.split("console.html")[0]
+                    if 'console.html' in result.log_url:
+                        log = result.log_url.split('console.html')[0]
+                    elif 'job-output.txt' in result.log_url:
+                        log = result.log_url.split('job-output.txt')[0]
                     other_fails["%s.%s" % (build, name)] = {
                         'log': log,
                         'timestamp': timestamp,

elastic_recheck/config.py

@@ -47,11 +47,16 @@ EXCLUDED_JOBS_REGEX = re.compile('(' + '|'.join(EXCLUDED_JOBS) + ')')
 
 INCLUDED_PROJECTS_REGEX = "(^openstack/|devstack|grenade)"
 
-ALL_FAILS_QUERY = ('filename:"console.html" '
+# TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported
-                   'AND (message:"Finished: FAILURE" '
+# Let's value legibility over pep8 line width here...
-                   'OR message:"[Zuul] Job complete, result: FAILURE") '
+ALL_FAILS_QUERY = (
-                   'AND build_queue:"gate" '
+    '((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh")'  # flake8: noqa
-                   'AND voting:"1"')
+    ' OR '
+    '(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))'  # flake8: noqa
+    ' AND build_status:"FAILURE"'
+    ' AND build_queue:"gate"'
+    ' AND voting:"1"'
+)
 
 UNCAT_MAX_SEARCH_SIZE = 30000
 

elastic_recheck/elasticRecheck.py

@@ -31,7 +31,7 @@ from elastic_recheck import results
 
 
 def required_files(job):
-    files = ['console.html']
+    files = []
     if re.match("tempest-dsvm", job):
         files.extend([
             'logs/screen-n-api.txt',
@@ -246,9 +246,11 @@ class Stream(object):
         query = qb.files_ready(change, patch, name, build_short_uuid)
         r = self.es.search(query, size='80', recent=True)
         files = [x['term'] for x in r.terms]
+        # TODO(dmsimard): Reliably differentiate zuul v2 and v3 jobs
         required = required_files(name)
         missing_files = [x for x in required if x not in files]
-        if len(missing_files) != 0:
+        if (len(missing_files) != 0 or
+           ('console.html' not in files and 'job-output.txt' not in files)):
             msg = ("%s missing for %s %s,%s,%s" % (
                 missing_files, name, change, patch, build_short_uuid))
             raise FilesNotReady(msg)

elastic_recheck/query_builder.py

@@ -69,22 +69,30 @@ def single_queue(query, queue, facet=None):
                    (query, queue), facet=facet)
 
 
-def result_ready(review, patch, name, build_short_uuid):
+def result_ready(change, patchset, name, short_uuid):
     """A query to determine if we have a failure for a particular patch.
 
     This is looking for a particular FAILURE line in the console log, which
     lets us know that we've got results waiting that we need to process.
     """
-    return generic('filename:"console.html" AND '
+    # TODO(dmsimard): Revisit this query once Zuul v2 is no longer supported
-                   '(message:"[SCP] Copying console log" '
+    # Let's value legibility over pep8 line width here...
-                   'OR message:"Grabbing consoleLog" '
+    query = (
-                   'OR message:"[Zuul] Job complete") '
+        '((filename:"job-output.txt" AND message:"POST-RUN END" AND message:"project-config/playbooks/base/post-ssh")'  # flake8: noqa
-                   'AND build_status:"FAILURE" '
+        ' OR '
-                   'AND build_change:"%s" '
+        '(filename:"console.html" AND (message:"[Zuul] Job complete" OR message:"[SCP] Copying console log" OR message:"Grabbing consoleLog"))'  # flake8: noqa
-                   'AND build_patchset:"%s" '
+        ' AND build_status:"FAILURE"'
-                   'AND build_name:"%s" '
+        ' AND build_change:"{change}"'
-                   'AND build_short_uuid:%s' %
+        ' AND build_patchset:"{patchset}"'
-                   (review, patch, name, build_short_uuid))
+        ' AND build_name:"{name}"'
+        ' AND build_short_uuid:"{short_uuid}"'
+    )
+    return generic(query.format(
+        change=change,
+        patchset=patchset,
+        name=name,
+        short_uuid=short_uuid
+    ))
 
 
 def files_ready(review, patch, name, build_short_uuid):
@@ -119,7 +127,7 @@ def single_patch(query, review, patch, build_short_uuid):
 
 def most_recent_event():
     return generic(
-        'filename:console.html '
+        '(filename:"console.html" OR filename:"job-output.txt") '
         'AND (build_queue:gate OR build_queue:check) '
         'AND NOT tags:_grokparsefailure '
         'AND NOT message:"%{logmessage}" ')

queries/1260311.yaml

@@ -1,4 +1,4 @@
 query: >
   message:"java.io.InterruptedIOException"
-  AND tags:"console.html"
+  AND (tags:"console.html" OR tags:"job-output.txt")
 suppress-graph: true

queries/1260654.yaml

@@ -2,5 +2,5 @@ query: >
   message:"java.io.IOException"
   AND message:"Remote call on"
   AND message:"failed"
-  AND tags:"console.html"
+  AND (tags:"console.html" OR tags:"job-output.txt")
 suppress-graph: true

queries/1269940.yaml

@@ -1,4 +1,4 @@
 query: >
   message:"[EnvInject] - [ERROR] - SEVERE ERROR occurs:"
-  AND tags:"console.html"
+  AND (tags:"console.html" OR tags:"job-output.txt")
 suppress-graph: true

queries/1270309.yaml

@@ -1,4 +1,4 @@
 query: >
     message:"java.lang.InterruptedException"
-    AND tags:"console.html"
+    AND (tags:"console.html" OR tags:"job-output.txt")
 suppress-graph: true

queries/1384373.yaml

@@ -1,3 +1,3 @@
 query: >
   message:"fatal: Could not read from remote repository."
-  AND tags:"console.html"
+  AND (tags:"console.html" OR tags:"job-output.txt")

queries/1695970.yaml

@@ -1,3 +1,3 @@
 query: >-
   message:"mount: mounting /dev/sr0 on /mnt failed: Device or resource busy"
-  AND tags:"console.html"
+  AND (tags:"console.html" OR tags:"job-output.txt")

queries/1718990.yaml

@@ -1,3 +1,3 @@
 query: >
   message:"HTTP 500 curl 22 The requested URL returned error: 500 Internal Server Error" AND
-  filename:"console.html"
+  (filename:"console.html" OR filename:"job-output.txt")