Start refactoring YAML tests into unittest framework
This commit is contained in:
Pim van Pelt
121
tests.py
121
tests.py
@ -20,6 +20,7 @@ import logging
|
||||
from validator import Validator
|
||||
import glob
|
||||
import re
|
||||
import unittest
|
||||
|
||||
try:
|
||||
import argparse
|
||||
@ -28,10 +29,7 @@ except ImportError:
|
||||
sys.exit(-2)
|
||||
|
||||
|
||||
def load_unittest(fn):
|
||||
""" Read a two-document YAML file from 'fn', and expect the first document
|
||||
to be a unittest specification, and the second file to be a config file to
|
||||
be tested. Return them as a tuple, or [None, None] on error. """
|
||||
def yamltest_one(fn, schema):
|
||||
unittest = None
|
||||
cfg = None
|
||||
try:
|
||||
@ -47,14 +45,64 @@ def load_unittest(fn):
|
||||
n = n + 1
|
||||
else:
|
||||
logging.error("Too many documents in %s" % fn)
|
||||
return None, None
|
||||
return False
|
||||
except:
|
||||
logging.error("Couldn't read config from %s" % fn)
|
||||
return None, None
|
||||
return unittest, cfg
|
||||
return False
|
||||
|
||||
logging.info("YAML %s" % fn)
|
||||
logging.debug("yamltest: %s" % unittest)
|
||||
logging.debug("config: %s" % cfg)
|
||||
this_failed =False
|
||||
v = Validator(schema=schema)
|
||||
rv, msgs = v.validate(cfg)
|
||||
try:
|
||||
if len(msgs) != unittest['test']['errors']['count']:
|
||||
logging.error("Unittest %s failed: expected %d error messages, got %d" % (fn, unittest['test']['errors']['count'], len(msgs)))
|
||||
this_failed = True
|
||||
except:
|
||||
pass
|
||||
|
||||
msgs_unexpected = 0
|
||||
msgs_expected = []
|
||||
if 'test' in unittest and 'errors' in unittest['test'] and 'expected' in unittest['test']['errors']:
|
||||
msgs_expected = unittest['test']['errors']['expected']
|
||||
|
||||
for m in msgs:
|
||||
this_msg_expected = False
|
||||
for expected in msgs_expected:
|
||||
logging.debug("Checking expected '%s'" % expected)
|
||||
if re.search(expected, m):
|
||||
logging.debug("Expected msg '%s' based on regexp '%s'" % (m, expected))
|
||||
this_msg_expected = True
|
||||
break
|
||||
if not this_msg_expected:
|
||||
logging.error("Unexpected message: %s" % (m))
|
||||
this_failed = True
|
||||
if this_failed:
|
||||
if 'test' in unittest and 'description' in unittest['test']:
|
||||
logging.error("YAML %s failed: %s" % (fn, unittest['test']['description']))
|
||||
else:
|
||||
logging.error("YAML %s failed" % (fn))
|
||||
return False
|
||||
else:
|
||||
logging.info("YAML %s passed" % (fn))
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
class YAMLTest(unittest.TestCase):
|
||||
def __init__(self, testName, yaml_filename, yaml_schema):
|
||||
# calling the super class init varies for different python versions. This works for 2.7
|
||||
super(YAMLTest, self).__init__(testName)
|
||||
self.yaml_filename = yaml_filename
|
||||
self.yaml_schema = yaml_schema
|
||||
|
||||
def test_yaml(self):
|
||||
print()
|
||||
assert yamltest_one(self.yaml_filename, self.yaml_schema)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
parser = argparse.ArgumentParser(formatter_class=argparse.RawTextHelpFormatter)
|
||||
parser.add_argument('-t', '--test', dest='test', type=str, nargs='+', default=['unittest/yaml/*.yaml'], help="""YAML test file(s)""")
|
||||
parser.add_argument('-s', '--schema', dest='schema', type=str, default='./schema.yaml', help="""YAML schema validation file""")
|
||||
@ -67,56 +115,15 @@ def main():
|
||||
logging.basicConfig(format='[%(levelname)-8s] %(name)s.%(funcName)s: %(message)s', level=level)
|
||||
logging.debug("Arguments: %s" % args)
|
||||
|
||||
errors = 0
|
||||
tests = 0
|
||||
yaml_suite = unittest.TestSuite()
|
||||
for pattern in args.test:
|
||||
for fn in glob.glob(pattern):
|
||||
tests = tests + 1
|
||||
unittest, cfg = load_unittest(fn)
|
||||
if not unittest:
|
||||
errors = errors + 1
|
||||
continue
|
||||
yaml_suite.addTest(YAMLTest('test_yaml', yaml_filename=fn, yaml_schema=args.schema))
|
||||
yaml_ok = unittest.TextTestRunner(verbosity=2).run(yaml_suite)
|
||||
|
||||
logging.info("Unittest %s" % fn)
|
||||
logging.debug("Unittest: %s" % unittest)
|
||||
logging.debug("Config: %s" % cfg)
|
||||
this_failed =False
|
||||
v = Validator(schema=args.schema)
|
||||
rv, msgs = v.validate(cfg)
|
||||
try:
|
||||
if len(msgs) != unittest['test']['errors']['count']:
|
||||
logging.error("Unittest %s failed: expected %d error messages, got %d" % (fn, unittest['test']['errors']['count'], len(msgs)))
|
||||
this_failed = True
|
||||
except:
|
||||
pass
|
||||
tests = unittest.TestLoader().discover(start_dir=".", pattern='test_*.py')
|
||||
unit_ok = unittest.TextTestRunner(verbosity=2).run(tests).wasSuccessful()
|
||||
|
||||
msgs_unexpected = 0
|
||||
msgs_expected = []
|
||||
if 'test' in unittest and 'errors' in unittest['test'] and 'expected' in unittest['test']['errors']:
|
||||
msgs_expected = unittest['test']['errors']['expected']
|
||||
|
||||
for m in msgs:
|
||||
this_msg_expected = False
|
||||
for expected in msgs_expected:
|
||||
logging.debug("Checking expected '%s'" % expected)
|
||||
if re.search(expected, m):
|
||||
logging.debug("Expected msg '%s' based on regexp '%s'" % (m, expected))
|
||||
this_msg_expected = True
|
||||
break
|
||||
if not this_msg_expected:
|
||||
logging.error("Unexpected message: %s" % (m))
|
||||
this_failed = True
|
||||
if this_failed:
|
||||
if 'test' in unittest and 'description' in unittest['test']:
|
||||
logging.error("Unittest %s failed: %s" % (fn, unittest['test']['description']))
|
||||
else:
|
||||
logging.error("Unittest %s failed" % (fn))
|
||||
errors = errors + 1
|
||||
else:
|
||||
logging.info("Unittest %s passed" % (fn))
|
||||
|
||||
logging.info("Tests: %d run, %d failed" % (tests, errors))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
if not yaml_ok or not unit_ok:
|
||||
sys.exit(-1)
|
||||
sys.exit(0)
|
||||
|
||||
Reference in New Issue
Block a user