Start refactoring YAML tests into unittest framework

tests.py

@@ -20,6 +20,7 @@ import logging
 from validator import Validator
 import glob
 import re
+import unittest
 
 try:
     import argparse
@@ -28,10 +29,7 @@ except ImportError:
     sys.exit(-2)
 
 
-def load_unittest(fn):
+def yamltest_one(fn, schema):
-    """ Read a two-document YAML file from 'fn', and expect the first document
-    to be a unittest specification, and the second file to be a config file to
-    be tested. Return them as a tuple, or [None, None] on error. """
     unittest = None
     cfg = None
     try:
@@ -47,14 +45,64 @@ def load_unittest(fn):
                     n = n + 1
                 else:
                     logging.error("Too many documents in %s" % fn)
-                    return None, None
+                    return False
     except:
         logging.error("Couldn't read config from %s" % fn)
-        return None, None
+        return False
-    return unittest, cfg
+
+    logging.info("YAML %s" % fn)
+    logging.debug("yamltest: %s" % unittest)
+    logging.debug("config: %s" % cfg)
+    this_failed =False
+    v = Validator(schema=schema)
+    rv, msgs = v.validate(cfg)
+    try:
+        if len(msgs) != unittest['test']['errors']['count']:
+            logging.error("Unittest %s failed: expected %d error messages, got %d" % (fn, unittest['test']['errors']['count'], len(msgs)))
+            this_failed = True
+    except:
+        pass
+
+    msgs_unexpected = 0
+    msgs_expected = []
+    if 'test' in unittest and 'errors' in unittest['test'] and 'expected' in unittest['test']['errors']:
+        msgs_expected = unittest['test']['errors']['expected']
+
+    for m in msgs:
+        this_msg_expected = False
+        for expected in msgs_expected:
+            logging.debug("Checking expected '%s'" % expected)
+            if re.search(expected, m):
+                logging.debug("Expected msg '%s' based on regexp '%s'" % (m, expected))
+                this_msg_expected = True
+                break
+        if not this_msg_expected:
+            logging.error("Unexpected message: %s" % (m))
+            this_failed = True
+    if this_failed:
+        if 'test' in unittest and 'description' in unittest['test']:
+            logging.error("YAML %s failed: %s" % (fn, unittest['test']['description']))
+        else:
+            logging.error("YAML %s failed" % (fn))
+        return False
+    else:
+        logging.info("YAML %s passed" % (fn))
+    return True
 
 
-def main():
+class YAMLTest(unittest.TestCase):
+    def __init__(self, testName, yaml_filename, yaml_schema):
+        # calling the super class init varies for different python versions.  This works for 2.7
+        super(YAMLTest, self).__init__(testName)
+        self.yaml_filename = yaml_filename
+        self.yaml_schema = yaml_schema
+
+    def test_yaml(self):
+        print()
+        assert yamltest_one(self.yaml_filename, self.yaml_schema)
+
+
+if __name__ == "__main__":
     parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
     parser.add_argument('-t', '--test', dest='test', type=str, nargs='+', default=['unittest/yaml/*.yaml'], help="""YAML test file(s)""")
     parser.add_argument('-s', '--schema', dest='schema', type=str, default='./schema.yaml', help="""YAML schema validation file""")
@@ -67,56 +115,15 @@ def main():
     logging.basicConfig(format='[%(levelname)-8s] %(name)s.%(funcName)s: %(message)s', level=level)
     logging.debug("Arguments: %s" % args)
 
-    errors = 0
+    yaml_suite = unittest.TestSuite()
-    tests = 0
     for pattern in args.test:
         for fn in glob.glob(pattern):
-            tests = tests + 1
+            yaml_suite.addTest(YAMLTest('test_yaml', yaml_filename=fn, yaml_schema=args.schema))
-            unittest, cfg = load_unittest(fn)
+    yaml_ok = unittest.TextTestRunner(verbosity=2).run(yaml_suite)
-            if not unittest:
-                errors = errors + 1
-                continue
 
-            logging.info("Unittest %s" % fn)
+    tests = unittest.TestLoader().discover(start_dir=".", pattern='test_*.py')
-            logging.debug("Unittest: %s" % unittest)
+    unit_ok = unittest.TextTestRunner(verbosity=2).run(tests).wasSuccessful()
-            logging.debug("Config: %s" % cfg)
-            this_failed =False
-            v = Validator(schema=args.schema)
-            rv, msgs = v.validate(cfg)
-            try:
-                if len(msgs) != unittest['test']['errors']['count']:
-                    logging.error("Unittest %s failed: expected %d error messages, got %d" % (fn, unittest['test']['errors']['count'], len(msgs)))
-                    this_failed = True
-            except:
-                pass
 
-            msgs_unexpected = 0
+    if not yaml_ok or not unit_ok:
-            msgs_expected = []
+        sys.exit(-1)
-            if 'test' in unittest and 'errors' in unittest['test'] and 'expected' in unittest['test']['errors']:
+    sys.exit(0)
-                msgs_expected = unittest['test']['errors']['expected']
-
-            for m in msgs:
-                this_msg_expected = False
-                for expected in msgs_expected:
-                    logging.debug("Checking expected '%s'" % expected)
-                    if re.search(expected, m):
-                        logging.debug("Expected msg '%s' based on regexp '%s'" % (m, expected))
-                        this_msg_expected = True
-                        break
-                if not this_msg_expected:
-                    logging.error("Unexpected message: %s" % (m))
-                    this_failed = True
-            if this_failed:
-                if 'test' in unittest and 'description' in unittest['test']:
-                    logging.error("Unittest %s failed: %s" % (fn, unittest['test']['description']))
-                else:
-                    logging.error("Unittest %s failed" % (fn))
-                errors = errors + 1
-            else:
-                logging.info("Unittest %s passed" % (fn))
-
-    logging.info("Tests: %d run, %d failed" % (tests, errors))
-
-
-if __name__ == "__main__":
-    main()