Consolidate node_list, add generic filter

node_list takes an argument "detail" which adds a rather arbitrary
list of results to the output.  This comes from the command-line,
where we're trying to keep width under a certain length; but doesn't
make as much sense here (especially for json).

For dashboard type applications, replace this with a simple "fields"
parameter which, if set, will only return those fields it sees in the
common text output function.

Note, this purposely doesn't apply to the JSON output, as it expected
client-side filtering is more appropriate there.  We could also add
generic field support to the command-line tools, if considered
worthwhile.

Add some documentation on all the end-points, and add info about these
parameters.

Change-Id: Ifbf1019b77368124961e7aa28dae403cabe50de1

doc/source/conf.py

@@ -25,7 +25,8 @@ sys.path.insert(0, os.path.abspath('../..'))
 
 # Add any Sphinx extension module names here, as strings. They can be extensions
 # coming with Sphinx (named 'sphinx.ext.*') or your custom ones.
-extensions = [ 'sphinxcontrib.programoutput' ]
+extensions = [ 'sphinxcontrib.programoutput',
+               'sphinxcontrib.httpdomain']
 #extensions = ['sphinx.ext.intersphinx']
 #intersphinx_mapping = {'python': ('http://docs.python.org/2.7', None)}
 

doc/source/operation.rst

@@ -218,3 +218,65 @@ same and makes it easy to turn the provider back on).
 If urgency is required you can delete the nodes directly instead of
 waiting for them to go through their normal lifecycle but the effect
 is the same.
+
+Web interface
+-------------
+
+If configured (see :ref:`webapp-conf`), a ``nodepool-launcher``
+instance can provide a range of end-points that can provide
+information in text and ``json`` format.  Note if there are multiple
+launchers, all will provide the same information.
+
+.. http:get:: /image-list
+
+   The status of uploaded images
+
+   :query fields: comma-separated list of fields to display
+   :resheader Content-Type: text/plain
+
+.. http:get:: /image-list.json
+
+   The status of uploaded images
+
+   :resheader Content-Type: application/json
+
+.. http:get:: /dib-image-list
+
+   The status of images built by ``diskimage-builder``
+
+   :query fields: comma-separated list of fields to display
+   :resheader Content-Type: text/plain
+
+.. http:get:: /dib-image-list.json
+
+   The status of images built by ``diskimage-builder``
+
+   :resheader Content-Type: application/json
+
+.. http:get:: /node-list
+
+   The status of currently active nodes
+
+   :query node_id: restrict to a specific node
+   :query fields: comma-separated list of fields to display
+   :resheader Content-Type: text/plain
+
+.. http:get:: /node-list.json
+
+   The status of currently active nodes
+
+   :query node_id: restrict to a specific node
+   :resheader Content-Type: application/json
+
+.. http:get:: /request-list
+
+   Outstanding requests
+
+   :query fields: comma-separated list of fields to display
+   :resheader Content-Type: text/plain
+
+.. http:get:: /request-list.json
+
+   Outstanding requests
+
+   :resheader Content-Type: application/json

nodepool/cmd/nodepoolcmd.py

@@ -156,8 +156,16 @@ class NodePoolCmd(NodepoolApp):
     def list(self, node_id=None, detail=False):
         if hasattr(self.args, 'detail'):
             detail = self.args.detail
-        results = status.node_list(self.zk, node_id, detail)
-        print(status.output(results, 'pretty'))
+
+        fields = ['id', 'provider', 'label', 'server_id',
+                  'public_ipv4', 'ipv6', 'state', 'age', 'locked']
+        if detail:
+            fields.extend(['hostname', 'private_ipv4', 'AZ',
+                           'connection_port', 'launcher',
+                           'allocated_to', 'hold_job',
+                           'comment'])
+        results = status.node_list(self.zk, node_id)
+        print(status.output(results, 'pretty', fields))
 
     def dib_image_list(self):
         results = status.dib_image_list(self.zk)

nodepool/status.py

@@ -53,13 +53,26 @@ def age(timestamp):
     return '%02d:%02d:%02d:%02d' % (d, h, m, s)
 
 
-def _to_pretty_table(objs, headers_table):
+def _to_pretty_table(objs, headers_table, fields):
+    '''Construct a pretty table output
+
+    :param objs: list of output objects
+    :param headers_table: list of (key, desr) header tuples
+    :param fields: list of fields to show; None means all
+
+    :return str: text output
+    '''
+    if fields:
+        headers_table = OrderedDict(
+            [h for h in headers_table.items() if h[0] in fields])
     headers = headers_table.values()
     t = PrettyTable(headers)
     t.align = 'l'
     for obj in objs:
         values = []
         for k in headers_table:
+            if fields and k not in fields:
+                continue
             if k == 'age':
                 try:
                     obj_age = age(int(obj[k]))
@@ -76,10 +89,18 @@ def _to_pretty_table(objs, headers_table):
     return t
 
 
-def output(results, fmt):
+def output(results, fmt, fields=None):
+    '''Generate output for webapp results
+
+    :param results: tuple (objs, headers) as returned by various _list
+                    functions
+    :param fmt: select from ascii pretty-table or json
+    :param fields: list of fields to show in pretty-table output
+    '''
     objs, headers_table = results
+
     if fmt == 'pretty':
-        t = _to_pretty_table(objs, headers_table)
+        t = _to_pretty_table(objs, headers_table, fields)
         return str(t)
     elif fmt == 'json':
         return json.dumps(objs)
@@ -87,7 +108,7 @@ def output(results, fmt):
         raise ValueError('Unknown format "%s"' % fmt)
 
 
-def node_list(zk, node_id=None, detail=False):
+def node_list(zk, node_id=None):
     headers_table = [
         ("id", "ID"),
         ("provider", "Provider"),
@@ -97,9 +118,7 @@ def node_list(zk, node_id=None, detail=False):
         ("ipv6", "IPv6"),
         ("state", "State"),
         ("age", "Age"),
-        ("locked", "Locked")
-    ]
-    detail_headers_table = [
+        ("locked", "Locked"),
         ("hostname", "Hostname"),
         ("private_ipv4", "Private IPv4"),
         ("AZ", "AZ"),
@@ -109,8 +128,6 @@ def node_list(zk, node_id=None, detail=False):
         ("hold_job", "Hold Job"),
         ("comment", "Comment")
     ]
-    if detail:
-        headers_table += detail_headers_table
     headers_table = OrderedDict(headers_table)
 
     def _get_node_values(node):
@@ -131,10 +148,7 @@ def node_list(zk, node_id=None, detail=False):
             node.public_ipv6,
             node.state,
             age(node.state_time),
-            locked
-        ]
-        if detail:
-            values += [
+            locked,
             node.hostname,
             node.private_ipv4,
             node.az,

nodepool/tests/test_webapp.py

@@ -45,6 +45,27 @@ class TestWebApp(tests.DBTestCase):
         data = f.read()
         self.assertTrue('fake-image' in data.decode('utf8'))
 
+    def test_image_list_filtered(self):
+        configfile = self.setup_config('node.yaml')
+        pool = self.useNodepool(configfile, watermark_sleep=1)
+        self.useBuilder(configfile)
+        pool.start()
+        webapp = self.useWebApp(pool, port=0)
+        webapp.start()
+        port = webapp.server.socket.getsockname()[1]
+
+        self.waitForImage('fake-provider', 'fake-image')
+        self.waitForNodes('fake-label')
+
+        req = request.Request(
+            "http://localhost:%s/image-list?fields=id,image,state" % port)
+        f = request.urlopen(req)
+        self.assertEqual(f.info().get('Content-Type'),
+                         'text/plain; charset=UTF-8')
+        data = f.read()
+        self.assertIn("| 0000000001 | fake-image | ready |",
+                      data.decode('utf8'))
+
     def test_image_list_json(self):
         configfile = self.setup_config('node.yaml')
         pool = self.useNodepool(configfile, watermark_sleep=1)

nodepool/webapp.py

@@ -105,8 +105,11 @@ class WebApp(threading.Thread):
         else:
             return None
 
-        output = status.output(results, out_fmt)
+        fields = None
+        if params.get('fields'):
+            fields = params.get('fields').split(',')
 
+        output = status.output(results, out_fmt, fields)
         return self.cache.put(index, output)
 
     def app(self, request):

test-requirements.txt

@@ -3,6 +3,7 @@ hacking>=0.10.2,<0.11
 coverage
 sphinx>=1.5.1,<1.6
 sphinxcontrib-programoutput
+sphinxcontrib-httpdomain
 fixtures>=0.3.12
 mock>=1.0
 python-subunit