From 3291c60d9ab52f9bc61be41004581566382323a3 Mon Sep 17 00:00:00 2001 From: Pim van Pelt Date: Sun, 20 Mar 2022 18:59:40 +0000 Subject: [PATCH] Start refactoring YAML tests into unittest framework --- tests.py | 121 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 64 insertions(+), 57 deletions(-) diff --git a/tests.py b/tests.py index 608672c..c34d466 100755 --- a/tests.py +++ b/tests.py @@ -20,6 +20,7 @@ import logging from validator import Validator import glob import re +import unittest try: import argparse @@ -28,10 +29,7 @@ except ImportError: sys.exit(-2) -def load_unittest(fn): - """ Read a two-document YAML file from 'fn', and expect the first document - to be a unittest specification, and the second file to be a config file to - be tested. Return them as a tuple, or [None, None] on error. """ +def yamltest_one(fn, schema): unittest = None cfg = None try: @@ -47,14 +45,64 @@ def load_unittest(fn): n = n + 1 else: logging.error("Too many documents in %s" % fn) - return None, None + return False except: logging.error("Couldn't read config from %s" % fn) - return None, None - return unittest, cfg + return False + + logging.info("YAML %s" % fn) + logging.debug("yamltest: %s" % unittest) + logging.debug("config: %s" % cfg) + this_failed =False + v = Validator(schema=schema) + rv, msgs = v.validate(cfg) + try: + if len(msgs) != unittest['test']['errors']['count']: + logging.error("Unittest %s failed: expected %d error messages, got %d" % (fn, unittest['test']['errors']['count'], len(msgs))) + this_failed = True + except: + pass + + msgs_unexpected = 0 + msgs_expected = [] + if 'test' in unittest and 'errors' in unittest['test'] and 'expected' in unittest['test']['errors']: + msgs_expected = unittest['test']['errors']['expected'] + + for m in msgs: + this_msg_expected = False + for expected in msgs_expected: + logging.debug("Checking expected '%s'" % expected) + if re.search(expected, m): + logging.debug("Expected msg '%s' based on regexp '%s'" % (m, expected)) + this_msg_expected = True + break + if not this_msg_expected: + logging.error("Unexpected message: %s" % (m)) + this_failed = True + if this_failed: + if 'test' in unittest and 'description' in unittest['test']: + logging.error("YAML %s failed: %s" % (fn, unittest['test']['description'])) + else: + logging.error("YAML %s failed" % (fn)) + return False + else: + logging.info("YAML %s passed" % (fn)) + return True -def main(): +class YAMLTest(unittest.TestCase): + def __init__(self, testName, yaml_filename, yaml_schema): + # calling the super class init varies for different python versions. This works for 2.7 + super(YAMLTest, self).__init__(testName) + self.yaml_filename = yaml_filename + self.yaml_schema = yaml_schema + + def test_yaml(self): + print() + assert yamltest_one(self.yaml_filename, self.yaml_schema) + + +if __name__ == "__main__": parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter) parser.add_argument('-t', '--test', dest='test', type=str, nargs='+', default=['unittest/yaml/*.yaml'], help="""YAML test file(s)""") parser.add_argument('-s', '--schema', dest='schema', type=str, default='./schema.yaml', help="""YAML schema validation file""") @@ -67,56 +115,15 @@ def main(): logging.basicConfig(format='[%(levelname)-8s] %(name)s.%(funcName)s: %(message)s', level=level) logging.debug("Arguments: %s" % args) - errors = 0 - tests = 0 + yaml_suite = unittest.TestSuite() for pattern in args.test: for fn in glob.glob(pattern): - tests = tests + 1 - unittest, cfg = load_unittest(fn) - if not unittest: - errors = errors + 1 - continue + yaml_suite.addTest(YAMLTest('test_yaml', yaml_filename=fn, yaml_schema=args.schema)) + yaml_ok = unittest.TextTestRunner(verbosity=2).run(yaml_suite) - logging.info("Unittest %s" % fn) - logging.debug("Unittest: %s" % unittest) - logging.debug("Config: %s" % cfg) - this_failed =False - v = Validator(schema=args.schema) - rv, msgs = v.validate(cfg) - try: - if len(msgs) != unittest['test']['errors']['count']: - logging.error("Unittest %s failed: expected %d error messages, got %d" % (fn, unittest['test']['errors']['count'], len(msgs))) - this_failed = True - except: - pass + tests = unittest.TestLoader().discover(start_dir=".", pattern='test_*.py') + unit_ok = unittest.TextTestRunner(verbosity=2).run(tests).wasSuccessful() - msgs_unexpected = 0 - msgs_expected = [] - if 'test' in unittest and 'errors' in unittest['test'] and 'expected' in unittest['test']['errors']: - msgs_expected = unittest['test']['errors']['expected'] - - for m in msgs: - this_msg_expected = False - for expected in msgs_expected: - logging.debug("Checking expected '%s'" % expected) - if re.search(expected, m): - logging.debug("Expected msg '%s' based on regexp '%s'" % (m, expected)) - this_msg_expected = True - break - if not this_msg_expected: - logging.error("Unexpected message: %s" % (m)) - this_failed = True - if this_failed: - if 'test' in unittest and 'description' in unittest['test']: - logging.error("Unittest %s failed: %s" % (fn, unittest['test']['description'])) - else: - logging.error("Unittest %s failed" % (fn)) - errors = errors + 1 - else: - logging.info("Unittest %s passed" % (fn)) - - logging.info("Tests: %d run, %d failed" % (tests, errors)) - - -if __name__ == "__main__": - main() + if not yaml_ok or not unit_ok: + sys.exit(-1) + sys.exit(0)