From 23d62acf55f30e472ecad5a3d324ad00de0371fb Mon Sep 17 00:00:00 2001 From: Gregory Haynes Date: Wed, 24 Jun 2015 04:01:00 +0000 Subject: [PATCH] Run geard for nodepool tests By running a geard as a fixture we can make tests which depend on gearman. Change-Id: Ia72e73cc78f291dcb514284496e174bc9ad7dc62 --- nodepool/tests/__init__.py | 67 ++++++++++++++++++- nodepool/tests/fixtures/leaked_node.yaml | 5 +- nodepool/tests/fixtures/node.yaml | 5 +- nodepool/tests/fixtures/node_az.yaml | 5 +- nodepool/tests/fixtures/node_cmd.yaml | 5 +- nodepool/tests/fixtures/node_dib.yaml | 5 +- .../tests/fixtures/node_dib_and_snap.yaml | 5 +- .../fixtures/node_dib_and_snap_fail.yaml | 5 +- .../node_dib_and_snap_upload_fail.yaml | 5 +- nodepool/tests/fixtures/node_dib_vhd.yaml | 5 +- .../fixtures/node_dib_vhd_and_qcow2.yaml | 5 +- nodepool/tests/fixtures/node_ipv6.yaml | 5 +- nodepool/tests/fixtures/node_osc.yaml | 5 +- nodepool/tests/fixtures/subnodes.yaml | 5 +- 14 files changed, 105 insertions(+), 27 deletions(-) diff --git a/nodepool/tests/__init__.py b/nodepool/tests/__init__.py index ae44b30fd..da04bcf5c 100644 --- a/nodepool/tests/__init__.py +++ b/nodepool/tests/__init__.py @@ -19,6 +19,7 @@ import logging import os import pymysql import random +import re import string import subprocess import threading @@ -26,6 +27,7 @@ import tempfile import time import fixtures +import gear import testresources import testtools @@ -38,6 +40,64 @@ class LoggingPopen(subprocess.Popen): pass +class FakeGearmanServer(gear.Server): + def __init__(self): + self.hold_jobs_in_queue = False + super(FakeGearmanServer, self).__init__(0) + + def getJobForConnection(self, connection, peek=False): + for queue in [self.high_queue, self.normal_queue, self.low_queue]: + for job in queue: + if not hasattr(job, 'waiting'): + if job.name.startswith('build:'): + job.waiting = self.hold_jobs_in_queue + else: + job.waiting = False + if job.waiting: + continue + if job.name in connection.functions: + if not peek: + queue.remove(job) + connection.related_jobs[job.handle] = job + job.worker_connection = connection + job.running = True + return job + return None + + def release(self, regex=None): + released = False + qlen = (len(self.high_queue) + len(self.normal_queue) + + len(self.low_queue)) + self.log.debug("releasing queued job %s (%s)" % (regex, qlen)) + for job in self.getQueue(): + cmd, name = job.name.split(':') + if cmd != 'build': + continue + if not regex or re.match(regex, name): + self.log.debug("releasing queued job %s" % + job.unique) + job.waiting = False + released = True + else: + self.log.debug("not releasing queued job %s" % + job.unique) + if released: + self.wakeConnections() + qlen = (len(self.high_queue) + len(self.normal_queue) + + len(self.low_queue)) + self.log.debug("done releasing queued jobs %s (%s)" % (regex, qlen)) + + +class GearmanServerFixture(fixtures.Fixture): + def setUp(self): + super(GearmanServerFixture, self).setUp() + self.gearman_server = FakeGearmanServer() + self.addCleanup(self.shutdownGearman) + + def shutdownGearman(self): + self.gearman_server.shutdown() + + class BaseTestCase(testtools.TestCase, testresources.ResourcedTestCase): def setUp(self): super(BaseTestCase, self).setUp() @@ -194,6 +254,10 @@ class DBTestCase(BaseTestCase): self.useFixture(f) self.dburi = f.dburi + gearman_fixture = GearmanServerFixture() + self.useFixture(gearman_fixture) + self.gearman_server = gearman_fixture.gearman_server + def setup_config(self, filename): images_dir = fixtures.TempDir() self.useFixture(images_dir) @@ -202,7 +266,8 @@ class DBTestCase(BaseTestCase): config = open(configfile).read() (fd, path) = tempfile.mkstemp() os.write(fd, config.format(dburi=self.dburi, - images_dir=images_dir.path)) + images_dir=images_dir.path, + gearman_port=self.gearman_server.port)) os.close(fd) return path diff --git a/nodepool/tests/fixtures/leaked_node.yaml b/nodepool/tests/fixtures/leaked_node.yaml index c091485cc..295788ddb 100644 --- a/nodepool/tests/fixtures/leaked_node.yaml +++ b/nodepool/tests/fixtures/leaked_node.yaml @@ -10,8 +10,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node.yaml b/nodepool/tests/fixtures/node.yaml index f2f0e3528..57ef24d14 100644 --- a/nodepool/tests/fixtures/node.yaml +++ b/nodepool/tests/fixtures/node.yaml @@ -10,8 +10,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node_az.yaml b/nodepool/tests/fixtures/node_az.yaml index 059559306..01fc4561d 100644 --- a/nodepool/tests/fixtures/node_az.yaml +++ b/nodepool/tests/fixtures/node_az.yaml @@ -10,8 +10,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node_cmd.yaml b/nodepool/tests/fixtures/node_cmd.yaml index f07d624fa..55978f022 100644 --- a/nodepool/tests/fixtures/node_cmd.yaml +++ b/nodepool/tests/fixtures/node_cmd.yaml @@ -10,8 +10,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label1 diff --git a/nodepool/tests/fixtures/node_dib.yaml b/nodepool/tests/fixtures/node_dib.yaml index c4aa8ccaa..121089369 100644 --- a/nodepool/tests/fixtures/node_dib.yaml +++ b/nodepool/tests/fixtures/node_dib.yaml @@ -12,8 +12,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-dib-label diff --git a/nodepool/tests/fixtures/node_dib_and_snap.yaml b/nodepool/tests/fixtures/node_dib_and_snap.yaml index 02b9dc4e9..e234206d3 100644 --- a/nodepool/tests/fixtures/node_dib_and_snap.yaml +++ b/nodepool/tests/fixtures/node_dib_and_snap.yaml @@ -12,8 +12,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node_dib_and_snap_fail.yaml b/nodepool/tests/fixtures/node_dib_and_snap_fail.yaml index dbb325acd..65486e85a 100644 --- a/nodepool/tests/fixtures/node_dib_and_snap_fail.yaml +++ b/nodepool/tests/fixtures/node_dib_and_snap_fail.yaml @@ -12,8 +12,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node_dib_and_snap_upload_fail.yaml b/nodepool/tests/fixtures/node_dib_and_snap_upload_fail.yaml index 9a361f985..0f415f7b8 100644 --- a/nodepool/tests/fixtures/node_dib_and_snap_upload_fail.yaml +++ b/nodepool/tests/fixtures/node_dib_and_snap_upload_fail.yaml @@ -12,8 +12,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node_dib_vhd.yaml b/nodepool/tests/fixtures/node_dib_vhd.yaml index 1819c616b..a6709dc34 100644 --- a/nodepool/tests/fixtures/node_dib_vhd.yaml +++ b/nodepool/tests/fixtures/node_dib_vhd.yaml @@ -12,8 +12,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-dib-label diff --git a/nodepool/tests/fixtures/node_dib_vhd_and_qcow2.yaml b/nodepool/tests/fixtures/node_dib_vhd_and_qcow2.yaml index 0b4e84322..767c4b505 100644 --- a/nodepool/tests/fixtures/node_dib_vhd_and_qcow2.yaml +++ b/nodepool/tests/fixtures/node_dib_vhd_and_qcow2.yaml @@ -12,8 +12,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/node_ipv6.yaml b/nodepool/tests/fixtures/node_ipv6.yaml index c7ac533a8..9d216d073 100644 --- a/nodepool/tests/fixtures/node_ipv6.yaml +++ b/nodepool/tests/fixtures/node_ipv6.yaml @@ -9,8 +9,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label1 diff --git a/nodepool/tests/fixtures/node_osc.yaml b/nodepool/tests/fixtures/node_osc.yaml index 225261a7f..569a5bed2 100644 --- a/nodepool/tests/fixtures/node_osc.yaml +++ b/nodepool/tests/fixtures/node_osc.yaml @@ -10,8 +10,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label diff --git a/nodepool/tests/fixtures/subnodes.yaml b/nodepool/tests/fixtures/subnodes.yaml index c93111906..5b562ee03 100644 --- a/nodepool/tests/fixtures/subnodes.yaml +++ b/nodepool/tests/fixtures/subnodes.yaml @@ -10,8 +10,9 @@ cron: zmq-publishers: - tcp://localhost:8881 -#gearman-servers: -# - host: localhost +gearman-servers: + - host: localhost + port: {gearman_port} labels: - name: fake-label