build: fix python load paths

Fixed python load paths so that vppcfg will work installed as python
library and standalone from the source directory, fixing load
pathes for resources such as yaml files along the way.

Added a make target for pylint called 'make check-style', fixed a
number of minor pylint issues along the way.

Signed-off-by: Ray Kinsella <mdr@ashroe.eu>

Makefile

@@ -25,6 +25,10 @@ install:
 pkg-deb:
 	dpkg-buildpackage -uc -us -b
 
+.PHONY: check-style
+check-style:
+	PYTHONPATH=./$(VPPCFG) pylint ./$(VPPCFG)
+
 .PHONY: uninstall
 uninstall:
 	sudo $(PIP) uninstall $(VPPCFG)

debian/rules

@@ -3,4 +3,5 @@
 %:
 	dh $@ --with python3 --buildsystem=pybuild --with systemd
 
+# TODO: fix test.py to that unit tests can be automagically called.
 override_dh_auto_test:

docs/config-guide.md

@@ -13,7 +13,7 @@ types of validation:
     to a running VPP. *Note*: Some semantic checks are stricter than VPP, because applying
     them may leave the dataplane in a non-recoverable state.
 
-For the curious, the Yamale syntax validation lives in [this schema](../schema.yaml).
+For the curious, the Yamale syntax validation lives in [this schema](../vppcfg/schema.yaml).
 If you want to get started quickly and don't mind cargo-culting, take a look at [this example](../example.yaml).
 
 ### Basic structure

setup.py

@@ -1,4 +1,5 @@
-from setuptools import setup, find_packages
+"""vppcfg setuptools setup.py for pip and deb pkg installations"""
+from setuptools import setup
 
 setup(
     name="vppcfg",
@@ -17,5 +18,5 @@ setup(
             "vppcfg = vppcfg.vppcfg:main",
         ]
     },
-    include_package_data=True,
+    package_data={"vppcfg": ["*.yaml"]},
 )

vppcfg/__init__.py

@@ -1,9 +1,16 @@
-import os, sys
+#!/usr/bin/env python
-
+#
-ROOT_DIR = os.path.dirname(__file__)
+# Copyright (c) 2022 Ray Kinsella
-
+#
-# fix the module load path
+# Licensed under the Apache License, Version 2.0 (the "License");
-sys.path.insert(0, ROOT_DIR)
+# you may not use this file except in compliance with the License.
-
+# You may obtain a copy of the License at:
-# fix the yaml search path
+#     http://www.apache.org/licenses/LICENSE-2.0
-os.chdir(ROOT_DIR)
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- coding: utf-8 -*-
+""" __init__.py added to suppress pylint error """

vppcfg/config/__init__.py

@@ -32,12 +32,12 @@ except ImportError:
     sys.exit(-2)
 from yamale.validators import DefaultValidators, Validator
 
-from config.loopback import validate_loopbacks
+from .loopback import validate_loopbacks
-from config.bondethernet import validate_bondethernets
+from .bondethernet import validate_bondethernets
-from config.interface import validate_interfaces
+from .interface import validate_interfaces
-from config.bridgedomain import validate_bridgedomains
+from .bridgedomain import validate_bridgedomains
-from config.vxlan_tunnel import validate_vxlan_tunnels
+from .vxlan_tunnel import validate_vxlan_tunnels
-from config.tap import validate_taps
+from .tap import validate_taps
 
 
 class IPInterfaceWithPrefixLength(Validator):
@@ -104,13 +104,12 @@ class Validator:
         if self.schema:
             fname = self.schema
             self.logger.debug(f"Validating against --schema {fname}")
-        elif hasattr(sys, "_MEIPASS"):
-            ## See vppcfg.spec data_files that includes schema.yaml into the bundle
-            self.logger.debug("Validating against built-in schema")
-            fname = os.path.join(sys._MEIPASS, "schema.yaml")
         else:
-            fname = "./schema.yaml"
+            ## See setup.py data files that includes schema.yaml into the bundle
-            self.logger.debug(f"Validating against fallthrough default schema {fname}")
+            self.logger.debug("Validating against built-in schema")
+            fname = os.path.abspath(
+                os.path.join(os.path.dirname(__file__), "..", "schema.yaml")
+            )
 
         if not os.path.isfile(fname):
             self.logger.error(f"Cannot file schema file: {fname}")

vppcfg/config/bondethernet.py

@@ -13,8 +13,8 @@
 #
 """ A vppcfg configuration module that handles bondethernets """
 import logging
-from config import interface
+from . import interface
-from config import mac
+from . import mac
 
 
 def get_bondethernets(yaml):

vppcfg/config/bridgedomain.py

@@ -13,8 +13,8 @@
 #
 """ A vppcfg configuration module that handles bridgedomains """
 import logging
-from config import interface
+from . import interface
-from config import loopback
+from . import loopback
 
 
 def get_bridgedomains(yaml):

vppcfg/config/interface.py

@@ -13,14 +13,14 @@
 #
 """ A vppcfg configuration module that validates interfaces """
 import logging
-from config import bondethernet
+from . import bondethernet
-from config import bridgedomain
+from . import bridgedomain
-from config import loopback
+from . import loopback
-from config import vxlan_tunnel
+from . import vxlan_tunnel
-from config import lcp
+from . import lcp
-from config import address
+from . import address
-from config import mac
+from . import mac
-from config import tap
+from . import tap
 
 
 def get_qinx_parent_by_name(yaml, ifname):

vppcfg/config/loopback.py

@@ -13,9 +13,9 @@
 #
 """ A vppcfg configuration module that validates loopbacks """
 import logging
-from config import lcp
+from . import lcp
-from config import address
+from . import address
-from config import mac
+from . import mac
 
 
 def get_loopbacks(yaml):

vppcfg/config/tap.py

@@ -13,7 +13,7 @@
 #
 """ A vppcfg configuration module that validates taps """
 import logging
-from config import mac
+from . import mac
 
 
 def get_taps(yaml):

vppcfg/config/test_bondethernet.py

@@ -15,12 +15,13 @@
 """ Unit tests for bondethernet """
 import unittest
 import yaml
-import config.bondethernet as bondethernet
+from . import bondethernet
+from .unittestyaml import UnitTestYaml
 
 
 class TestBondEthernetMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_bondethernet.yaml", "r") as f:
+        with UnitTestYaml("test_bondethernet.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_get_by_name(self):

vppcfg/config/test_bridgedomain.py

@@ -15,12 +15,13 @@
 """ Unit tests for bridgedomains """
 import unittest
 import yaml
-import config.bridgedomain as bridgedomain
+from . import bridgedomain
+from .unittestyaml import UnitTestYaml
 
 
 class TestBridgeDomainMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_bridgedomain.yaml", "r") as f:
+        with UnitTestYaml("test_bridgedomain.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_get_by_name(self):

vppcfg/config/test_interface.py

@@ -15,12 +15,13 @@
 """ Unit tests for interfaces """
 import unittest
 import yaml
-import config.interface as interface
+from . import interface
+from .unittestyaml import UnitTestYaml
 
 
 class TestInterfaceMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_interface.yaml", "r") as f:
+        with UnitTestYaml("test_interface.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_enumerators(self):

vppcfg/config/test_lcp.py

@@ -15,13 +15,14 @@
 """ Unit tests for LCPs """
 import unittest
 import yaml
-import config.lcp as lcp
+from . import lcp
-import config.interface as interface
+from . import interface
+from .unittestyaml import UnitTestYaml
 
 
 class TestLCPMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_lcp.yaml", "r") as f:
+        with UnitTestYaml("test_lcp.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_enumerators(self):

vppcfg/config/test_loopback.py

@@ -15,12 +15,13 @@
 """ Unit tests for loopbacks """
 import unittest
 import yaml
-import config.loopback as loopback
+from . import loopback
+from .unittestyaml import UnitTestYaml
 
 
 class TestLoopbackMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_loopback.yaml", "r") as f:
+        with UnitTestYaml("test_loopback.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_get_by_lcp_name(self):

vppcfg/config/test_mac.py

@@ -14,7 +14,7 @@
 # -*- coding: utf-8 -*-
 """ Unit tests for MAC addresses """
 import unittest
-import config.mac as mac
+from . import mac
 
 
 class TestMACMethods(unittest.TestCase):

vppcfg/config/test_tap.py

@@ -15,12 +15,13 @@
 """ Unit tests for taps """
 import unittest
 import yaml
-import config.tap as tap
+from . import tap
+from .unittestyaml import UnitTestYaml
 
 
 class TestTAPMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_tap.yaml", "r") as f:
+        with UnitTestYaml("test_tap.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_get_by_name(self):

vppcfg/config/test_vxlan_tunnel.py

@@ -15,12 +15,13 @@
 """ Unit tests for vxlan_tunnels """
 import unittest
 import yaml
-import config.vxlan_tunnel as vxlan_tunnel
+from . import vxlan_tunnel
+from .unittestyaml import UnitTestYaml
 
 
 class TestVXLANMethods(unittest.TestCase):
     def setUp(self):
-        with open("unittest/test_vxlan_tunnel.yaml", "r") as f:
+        with UnitTestYaml("test_vxlan_tunnel.yaml") as f:
             self.cfg = yaml.load(f, Loader=yaml.FullLoader)
 
     def test_get_by_name(self):

vppcfg/config/unittestyaml.py

@@ -0,0 +1,41 @@
+""" module to help locate unittest resources """
+#!/usr/bin/env python
+#
+# Copyright (c) 2022 Ray Kinsella
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at:
+#     http://www.apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# -*- coding: utf-8 -*-
+
+import os
+
+
+class UnitTestYaml:
+    """Helper classes to find and load unit test yaml resources"""
+
+    def __init__(self, fpath):
+        self.file = None
+        self.filename = fpath
+        self.resdir = os.path.abspath(
+            os.path.join(os.path.dirname(__file__), "../unittest")
+        )
+
+    def __enter__(self):
+        self.file = open(
+            os.path.join(self.resdir, self.filename), "r", encoding="utf-8"
+        )
+        return self.file
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.file.close()
+
+    def __str__(self):
+        return self.file

vppcfg/tests.py

@@ -14,13 +14,19 @@
 #
 # -*- coding: utf-8 -*-
 """ This is a unit test suite for vppcfg """
-
+# pylint: disable=duplicate-code
+import os
 import sys
 import glob
 import re
 import unittest
 import yaml
-from config import Validator
+
+try:
+    from vppcfg.config import Validator
+except ModuleNotFoundError:
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+    from vppcfg.config import Validator
 
 try:
     import argparse

vppcfg/vpp/applier.py

@@ -17,7 +17,7 @@ The functions in this file interact with the VPP API to modify certain
 interface metadata.
 """
 
-from vpp.vppapi import VPPApi
+from .vppapi import VPPApi
 
 
 class Applier(VPPApi):

vppcfg/vpp/dumper.py

@@ -19,8 +19,8 @@ interface metadata and write it to a YAML file.
 
 import sys
 import yaml
-from vpp.vppapi import VPPApi
+from vppcfg.config import bondethernet
-from config import bondethernet
+from .vppapi import VPPApi
 
 
 class Dumper(VPPApi):

vppcfg/vpp/reconciler.py

@@ -19,14 +19,14 @@ metadata, and plan configuration changes towards a given YAML target configurati
 """
 import sys
 import logging
-from config import loopback
+from vppcfg.config import loopback
-from config import interface
+from vppcfg.config import interface
-from config import bondethernet
+from vppcfg.config import bondethernet
-from config import bridgedomain
+from vppcfg.config import bridgedomain
-from config import vxlan_tunnel
+from vppcfg.config import vxlan_tunnel
-from config import lcp
+from vppcfg.config import lcp
-from config import tap
+from vppcfg.config import tap
-from vpp.vppapi import VPPApi
+from .vppapi import VPPApi
 
 
 class Reconciler:

vppcfg/vppcfg.py

@@ -15,12 +15,20 @@
 # -*- coding: utf-8 -*-
 """vppcfg is a utility to configure a running VPP Dataplane using YAML
 config files. See http://github.com/pimvanpelt/vppcfg/README.md for details. """
+# pylint: disable=duplicate-code
+import os
 import sys
 import logging
 import yaml
-from config import Validator
+
-from vpp.reconciler import Reconciler
+# Ensure the paths are correct when we execute from the source tree
-from vpp.dumper import Dumper
+try:
+    from vppcfg.config import Validator
+except ModuleNotFoundError:
+    sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+    from vppcfg.config import Validator
+from vppcfg.vpp.reconciler import Reconciler
+from vppcfg.vpp.dumper import Dumper
 
 try:
     import argparse