Merge "Improvements to yaql CLI"

yaql/cli/cli_functions.py

@@ -12,11 +12,11 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import itertools
+from __future__ import print_function
+
 import json
 import locale
 import os
-import re
 import readline
 import sys
 
@@ -29,7 +29,6 @@ from yaql.language import utils
 
 
 PROMPT = "yaql> "
-LIMIT = 100
 
 
 def main(context, show_tokens, parser):
@@ -37,7 +36,7 @@ def main(context, show_tokens, parser):
     print("Version {0}".format(version))
     if context.get_data('legacy', False):
         print("Running in a legacy (0.2.x compatible) mode")
-    print("Copyright (c) 2013-2015 Mirantis, Inc")
+    print("Copyright (c) 2013-2017 Mirantis, Inc")
     print("")
     if not context['']:
         print("No data loaded into context ")
@@ -49,8 +48,10 @@ def main(context, show_tokens, parser):
     comm = True
     while comm != 'exit':
         try:
-            comm = six.moves.input(PROMPT).decode(
+            comm = six.moves.input(PROMPT)
-                sys.stdin.encoding or locale.getpreferredencoding(True))
+            if six.PY2:
+                comm = comm.decode(
+                    sys.stdin.encoding or locale.getpreferredencoding(True))
         except EOFError:
             return
         if not comm:
@@ -84,11 +85,9 @@ def main(context, show_tokens, parser):
             continue
         try:
             res = expr.evaluate(context=context)
-            if utils.is_iterator(res):
+            print_output(res, context)
-                res = list(itertools.islice(res, LIMIT))
-            print(json.dumps(res, indent=4, ensure_ascii=False))
         except Exception as ex:
-            print(u'Execution exception: {0}'.format(ex))
+            print(u'Execution exception: {0}'.format(ex), file=sys.stderr)
 
 
 def load_data(data_file, context):
@@ -107,19 +106,10 @@ def load_data(data_file, context):
     print('Data from file {0} loaded into context'.format(data_file))
 
 
-def regexp(self, pattern):
-    match = re.match(pattern, self)
-    if match:
-        return match.groups()
-    else:
-        return None
-
-
 def register_in_context(context, parser):
     context.register_function(
         lambda context, show_tokens: main(context, show_tokens, parser),
         name='__main')
-    context.register_function(regexp)
 
 
 def parse_service_command(comm):
@@ -131,6 +121,22 @@ def parse_service_command(comm):
     return func_name, args
 
 
+def evaluate(expr, parser, data, context):
+    try:
+        res = parser(expr).evaluate(data, context)
+        print_output(res, context)
+    except Exception as ex:
+        print(u'Execution exception: {0}'.format(ex), file=sys.stderr)
+        exit(1)
+
+
+def print_output(v, context):
+    if context['#nativeOutput']:
+        print(v)
+    else:
+        print(json.dumps(v, indent=4, ensure_ascii=False))
+
+
 SERVICE_FUNCTIONS = {
     # '@help':print_help,
     '@load': load_data,

yaql/cli/run.py

@@ -14,18 +14,39 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-import json
+from __future__ import print_function
 
+import json
 import optparse
+import sys
 
 import yaql
 from yaql.cli import cli_functions
 import yaql.legacy
 
 
+def read_data(f, options):
+    if options.string:
+        if options.array:
+            return [line.rstrip('\n') for line in f]
+        else:
+            return f.read()
+    else:
+        if options.array:
+            return [json.loads(s) for s in f.readlines()]
+        else:
+            return json.load(f)
+
+
 def main():
     p = optparse.OptionParser()
-    p.add_option('--data', '-d', help="input JSON file")
+    p.add_option('--data', '-d', help="input file")
+    p.add_option('--string', '-s', action='store_true',
+                 help="input is a string")
+    p.add_option('--native', '-n', action='store_true',
+                 help="output data in Python native format")
+    p.add_option('--array', '-a', action='store_true',
+                 help="read input line by line")
     p.add_option('--tokens', '-t', action='store_true', dest='tokens',
                  help="print lexical tokens info")
     p.add_option('--legacy', action='store_true', dest='legacy',
@@ -34,18 +55,21 @@ def main():
     options, arguments = p.parse_args()
     if options.data:
         try:
-            with open(options.data) as f:
+            if options.data == '-':
-                data = json.load(f)
+                data = read_data(sys.stdin, options)
+            else:
+                with open(options.data) as f:
+                    data = read_data(f, options)
         except Exception:
-            print('Unable to load data from ' + options.data)
+            print('Unable to load data from ' + options.data, file=sys.stderr)
-            return
+            exit(1)
     else:
         data = None
 
     engine_options = {
-        'yaql.limitIterators': 100,
+        'yaql.limitIterators': 1000,
         'yaql.convertSetsToLists': True,
-        'yaql.memoryQuota': 10000
+        'yaql.memoryQuota': 100000
     }
 
     if options.legacy:
@@ -56,9 +80,16 @@ def main():
         factory = yaql.YaqlFactory()
         context = yaql.create_context()
 
+    if options.native:
+        context['#nativeOutput'] = True
+
     parser = factory.create(options=engine_options)
     cli_functions.register_in_context(context, parser)
-    if options.tokens:
+
+    if len(arguments) > 0:
+        for arg in arguments:
+            cli_functions.evaluate(arg, parser, data, context)
+    elif options.tokens:
         parser('__main(true)').evaluate(data, context)
     else:
         parser('__main(false)').evaluate(data, context)