Enable compliance tests to use plugins for cluster provisioning

Tests/kaas/plugin/README.md

@@ -0,0 +1,39 @@
+# Plugin for provisioning k8s clusters and performing conformance tests on these clusters
+
+## Development environment
+
+### requirements
+
+* [docker](https://docs.docker.com/engine/install/)
+* [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation)
+* [sonobuoy](https://sonobuoy.io/docs/v0.57.1/#installation)
+
+### setup for development
+
+1. Generate python 3.10 env
+
+   ```bash
+   sudo apt-get install python3.10-dev
+   virtualenv -p /usr/bin/python3.10 venv
+   echo "*" >> venv/.gitignore
+   source venv/bin/activate
+   (venv) curl -sS https://bootstrap.pypa.io/get-pip.py | python3.10
+   (venv) python3.10 -m pip install --upgrade pip
+   (venv) python3.10 -m pip --version
+
+   ```
+
+2. Install dependencies:
+
+   ```bash
+   (venv) pip install pip-tools
+   (venv) pip-compile requirements.in
+   (venv) pip-sync requirements.txt
+   ```
+
+3. Set environment variables and launch the process:
+
+   ```bash
+   (venv) export CLUSTER_PROVIDER="kind"
+   (venv) python run.py
+   ```

Tests/kaas/plugin/interface.py

@@ -0,0 +1,184 @@
+from abc import ABC, abstractmethod
+from typing import final
+from kubernetes import client, config
+import os
+import logging
+from junitparser import JUnitXml
+
+logging.basicConfig(level=logging.DEBUG)
+logger = logging.getLogger("interface")
+
+
+def setup_k8s_client(kubeconfigfile=None):
+
+    if kubeconfigfile:
+        logger.debug(f"loading kubeconfig file '{kubeconfigfile}'")
+        config.load_kube_config(kubeconfigfile)
+        logger.info("kubeconfigfile loaded successfully")
+    else:
+        logger.error("no kubeconfig file provided")
+        return None
+
+    k8s_api_client = client.CoreV1Api()
+
+    return k8s_api_client
+
+
+class KubernetesClusterPlugin(ABC):
+    """
+    An abstract base class for custom Kubernetes cluster provider plugins.
+    It represents an interface class from which the api provider-specific
+    plugins must be derived as child classes
+    """
+
+    kubeconfig_path = None
+    cluster_name = None
+    k8s_api_client = None
+    working_directory = None
+
+    @final
+    def __init__(self, config=None):
+        logger.debug(config)
+        self.working_directory = os.getcwd()
+        logger.debug(f"Working from {self.working_directory}")
+
+    @final
+    def _preflight_check(self):
+        """
+        Prefligth test to ensure that everything is set up correctly for execution
+        :param: None
+        :return: None
+        """
+        logger.info("check kubeconfig")
+        self.k8s_api_client = setup_k8s_client(self.kubeconfig_path)
+
+        for api in client.ApisApi().get_api_versions().groups:
+            versions = []
+            for v in api.versions:
+                name = ""
+                if v.version == api.preferred_version.version and len(api.versions) > 1:
+                    name += "*"
+                name += v.version
+                versions.append(name)
+            logger.debug(f"[supported api]: {api.name:<40} {','.join(versions)}")
+
+        logger.debug("checks if sonobuoy is availabe")
+        return_value = os.system(f"sonobuoy version --kubeconfig='{self.kubeconfig_path}'")
+        if return_value != 0:
+            raise Exception("sonobuoy is not installed")
+
+    @final
+    def _test_k8s_cncf_conformance(self):
+        """
+        This method invokes the conformance tests with sononbuoy
+        :param: None
+        :return: None
+        """
+        logger.info(" invoke cncf conformance test")
+        # ~ os.system(f"sonobuoy run --wait --mode=certified-conformance --kubeconfig='{self.kubeconfig_path}'")
+        # TODO:!!! switch to the real test on the final merge !!!
+        # Only one test is currently being carried out for development purposes
+        os.system(
+            f"sonobuoy run --wait --plugin-env e2e.E2E_FOCUS=pods --plugin-env e2e.E2E_DRYRUN=true --kubeconfig='{self.kubeconfig_path}'"
+        )
+
+    @final
+    def _test_scs_kaas_conformance(self):
+        """
+        This method invokes SCS's very own conformance tests by using sononbuoy
+        :param: None
+        :return: None
+        """
+        raise NotImplementedError
+
+    @final
+    def _cleanup_sonobuoy_resources(self):
+        """
+        This method deletes all resources that sonobuoy has created in a k8s cluster for a test
+        :param: None
+        :return: None
+        """
+        logger.info("removing sonobuoy tests from cluster")
+        os.system(f"sonobuoy delete --wait --kubeconfig='{self.kubeconfig_path}'")
+
+    @final
+    def _retrieve_result(self, result_dir_name):
+        """
+        This method invokes sonobouy to store the results in a subdirectory of
+        the working directory. The Junit results file contained in it is then
+        analyzed in order to interpret the relevant information it containes
+        :param: result_file_name:
+        :return: None
+        """
+        logger.debug(f"retrieving results to {result_dir_name}")
+        result_dir = self.working_directory + "/" + result_dir_name
+        if os.path.exists(result_dir):
+            os.system(f"rm -rf {result_dir}/*")
+        else:
+            os.mkdir(result_dir)
+        os.system(
+            f"sonobuoy retrieve {result_dir} -x --filename='{result_dir_name}' --kubeconfig='{self.kubeconfig_path}'"
+        )
+        logger.debug(
+            f"parsing JUnit result from {result_dir + '/plugins/e2e/results/global/junit_01.xml' } "
+        )
+        xml = JUnitXml.fromfile(
+            result_dir + "/plugins/e2e/results/global/junit_01.xml"
+        )
+        failed_test_cases = 0
+        passed_test_cases = 0
+        skipped_test_cases = 0
+        for suite in xml:
+            for case in suite:
+                if case.is_passed:
+                    passed_test_cases += 1
+                else:
+                    failed_test_cases += 1
+                if case.is_skipped:
+                    skipped_test_cases += 1
+
+        logger.info(
+            f" {passed_test_cases} passed, {failed_test_cases} failed of which {skipped_test_cases} were skipped"
+        )
+
+    @abstractmethod
+    def _create_cluster(self, cluster_name) -> (str, int):
+        """
+        Create a Kubernetes cluster to test aggainst.
+        :param: cluster_name:
+        :return: kubeconfig: kubeconfig of the cluster used for testing
+        """
+        pass
+
+    @abstractmethod
+    def _delete_cluster(self, cluster_name) -> (str, int):
+        """
+        Delete the Kubernetes cluster.
+        :param: cluster_name:
+        :return: None
+        """
+        pass
+
+    @final
+    def run(self):
+        """
+        This method is to be called to run the plugin
+        """
+
+        try:
+            self._create_cluster()
+            self._preflight_check()
+            self._test_k8s_cncf_conformance()
+            self._retrieve_result("cncf_result")
+            self._cleanup_sonobuoy_resources()
+            # self._test_scs_kaas_conformance()
+            # self._retrieve_result("scs_kaas_result")
+        except Exception as e:
+            logging.error(e)
+
+        try:
+            self._cleanup_sonobuoy_resources()
+        except Exception as e:
+            logging.error(e)
+        finally:
+            self._delete_cluster()

Tests/kaas/plugin/plugin_kind.py

@@ -0,0 +1,20 @@
+from interface import KubernetesClusterPlugin
+from pytest_kind import KindCluster
+import time
+
+
+class PluginKind(KubernetesClusterPlugin):
+    """
+    Plugin to handle the provisioning of kubernetes cluster for
+    conformance testing purpose with the use of Kind
+    """
+
+    def _create_cluster(self):
+        self.cluster_name = "scs-cluster"
+        self.cluster = KindCluster(self.cluster_name)
+        self.cluster.create()
+        time.sleep(30)
+        self.kubeconfig_path = str(self.cluster.kubeconfig_path.resolve())
+
+    def _delete_cluster(self):
+        self.cluster.delete()

Tests/kaas/plugin/plugin_static.py

@@ -0,0 +1,14 @@
+from interface import KubernetesClusterPlugin
+import os
+
+
+class PluginStatic(KubernetesClusterPlugin):
+    """
+    Plugin to handle the provisioning of kubernetes
+    using a kubeconfig file
+    """
+    def _create_cluster(self):
+        self.kubeconfig_path = os.environ["KUBECONFIG"]
+
+    def _delete_cluster(self):
+        pass

Tests/kaas/plugin/requirements.in

@@ -0,0 +1,3 @@
+pytest-kind
+kubernetes
+junitparser

Tests/kaas/plugin/requirements.txt

@@ -0,0 +1,62 @@
+#
+# This file is autogenerated by pip-compile with Python 3.10
+# by the following command:
+#
+#    pip-compile requirements.in
+#
+cachetools==5.5.0
+    # via google-auth
+certifi==2024.8.30
+    # via
+    #   kubernetes
+    #   requests
+charset-normalizer==3.3.2
+    # via requests
+google-auth==2.34.0
+    # via kubernetes
+idna==3.8
+    # via requests
+junitparser==3.2.0
+    # via -r requirements.in
+kubernetes==30.1.0
+    # via -r requirements.in
+oauthlib==3.2.2
+    # via
+    #   kubernetes
+    #   requests-oauthlib
+pyasn1==0.6.0
+    # via
+    #   pyasn1-modules
+    #   rsa
+pyasn1-modules==0.4.0
+    # via google-auth
+pykube-ng==23.6.0
+    # via pytest-kind
+pytest-kind==22.11.1
+    # via -r requirements.in
+python-dateutil==2.9.0.post0
+    # via kubernetes
+pyyaml==6.0.2
+    # via
+    #   kubernetes
+    #   pykube-ng
+requests==2.32.3
+    # via
+    #   kubernetes
+    #   pykube-ng
+    #   requests-oauthlib
+requests-oauthlib==2.0.0
+    # via kubernetes
+rsa==4.9
+    # via google-auth
+six==1.16.0
+    # via
+    #   kubernetes
+    #   python-dateutil
+urllib3==2.2.2
+    # via
+    #   kubernetes
+    #   pykube-ng
+    #   requests
+websocket-client==1.8.0
+    # via kubernetes

Tests/kaas/plugin/run.py

@@ -0,0 +1,15 @@
+from plugin_kind import PluginKind
+from plugin_static import PluginStatic
+import os
+
+if __name__ == "__main__":
+
+    match os.environ['CLUSTER_PROVIDER']:
+        case "static":
+            plugin = PluginStatic()
+        case "kind":
+            plugin = PluginKind()
+        case _:
+            raise NotImplementedError(f"{os.environ['CLUSTER_PROVIDER']} is not valid ")
+
+    plugin.run()