From 3d81a0a5bd642cea2a618e75d1c66c502c097772 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Xavier=20Dupr=C3=A9?= Date: Fri, 20 Aug 2021 11:29:52 +0200 Subject: [PATCH] Update CI to onnx==1.10.1 (#490) * check onnx==1.10.1 * better error message * change skipping condition * one condition less strict on lightgbm --- .azure-pipelines/linux-conda-CI.yml | 6 +++ .azure-pipelines/win32-conda-CI.yml | 37 +++++++++++++++++-- requirements-dev.txt | 1 - .../test_cml_DictVectorizerConverter.py | 19 +++++++++- .../coreml/test_cml_GLMClassifierConverter.py | 4 ++ .../coreml/test_cml_GLMRegressorConverter.py | 4 ++ tests/coreml/test_cml_ImputerConverter.py | 7 +++- .../coreml/test_cml_OneHotEncoderConverter.py | 6 +++ tests/coreml/test_cml_ScalerConverter.py | 4 ++ ...st_cml_SupportVectorClassifierConverter.py | 13 +++++++ ...est_cml_SupportVectorRegressorConverter.py | 4 ++ ...est_cml_TreeEnsembleClassifierConverter.py | 4 ++ ...test_cml_TreeEnsembleRegressorConverter.py | 4 ++ .../lightgbm/test_lightgbm_tree_structure.py | 3 +- tests/lightgbm/test_objective_functions.py | 2 +- 15 files changed, 108 insertions(+), 10 deletions(-) diff --git a/.azure-pipelines/linux-conda-CI.yml b/.azure-pipelines/linux-conda-CI.yml index 16f3b0a55..8a4c0a53d 100644 --- a/.azure-pipelines/linux-conda-CI.yml +++ b/.azure-pipelines/linux-conda-CI.yml @@ -13,6 +13,12 @@ jobs: vmImage: 'ubuntu-latest' strategy: matrix: + Python39-1101-RT181-xgb11: + python.version: '3.9' + ONNX_PATH: onnx==1.10.1 # '-i https://test.pypi.org/simple/ onnx==1.9.101' + ONNXRT_PATH: onnxruntime==1.8.1 + COREML_PATH: git+https://github.com/apple/coremltools@3.1 + xgboost.version: '>=1.2' Python39-190-RT180-xgb11: python.version: '3.9' ONNX_PATH: onnx==1.9.0 diff --git a/.azure-pipelines/win32-conda-CI.yml b/.azure-pipelines/win32-conda-CI.yml index 1a511762c..0e073eb5e 100644 --- a/.azure-pipelines/win32-conda-CI.yml +++ b/.azure-pipelines/win32-conda-CI.yml @@ -13,6 +13,20 @@ jobs: vmImage: 'windows-latest' strategy: matrix: + Python39-1101-RT181: + python.version: '3.9' + ONNX_PATH: 'onnx==1.10.1' # '-i https://test.pypi.org/simple/ onnx==1.9.101' + ONNXRT_PATH: onnxruntime==1.8.1 + COREML_PATH: git+https://github.com/apple/coremltools@3.1 + sklearn.version: '' + + Python39-190-RT181: + python.version: '3.9' + ONNX_PATH: 'onnx==1.9.0' + ONNXRT_PATH: onnxruntime==1.8.1 + COREML_PATH: git+https://github.com/apple/coremltools@3.1 + sklearn.version: '' + Python39-190-RT180: python.version: '3.9' ONNX_PATH: onnx==1.9.0 @@ -66,15 +80,32 @@ jobs: call activate py$(python.version) python -m pip install --upgrade pip numpy echo Test numpy installation... && python -c "import numpy" - python -m pip install %COREML_PATH% %ONNX_PATH% + python -m pip install scikit-learn + python -m pip install %ONNX_PATH% python -m pip install humming-bird-ml --no-deps python -m pip install -r requirements.txt python -m pip install torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio===0.8.1 -f https://download.pytorch.org/whl/torch_stable.html + displayName: 'Install dependencies (1)' + + - script: | + call activate py$(python.version) python -m pip install -r requirements-dev.txt + displayName: 'Install dependencies (2)' + + - script: | + call activate py$(python.version) + python -m pip install %COREML_PATH% + displayName: 'Install coremltools' + + - script: | + call activate py$(python.version) python -m pip install %ONNXRT_PATH% + displayName: 'Install onnxruntime' + + - script: | + call activate py$(python.version) python -m pip install scikit-learn$(sklearn.version) - python -m pip show pytest - displayName: 'Install dependencies' + displayName: 'Install scikit-learn' - script: | call activate py$(python.version) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4508bf97e..8f4d8ab00 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,7 +1,6 @@ -f https://download.pytorch.org/whl/torch_stable.html catboost codecov -coremltools cython dill flake8 diff --git a/tests/coreml/test_cml_DictVectorizerConverter.py b/tests/coreml/test_cml_DictVectorizerConverter.py index eb4fc371d..50204cc79 100644 --- a/tests/coreml/test_cml_DictVectorizerConverter.py +++ b/tests/coreml/test_cml_DictVectorizerConverter.py @@ -3,6 +3,11 @@ """ Tests CoreML DictVectorizer converter. """ +import sys +from distutils.version import StrictVersion +import unittest +import onnx +import sklearn try: from sklearn.impute import SimpleImputer as Imputer import sklearn.preprocessing @@ -12,7 +17,6 @@ except ImportError: from sklearn.preprocessing import Imputer import coremltools -import unittest from sklearn.feature_extraction import DictVectorizer from onnxmltools.convert.coreml.convert import convert from onnxmltools.utils import dump_data_and_model @@ -20,11 +24,22 @@ class TestCoreMLDictVectorizerConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_dict_vectorizer(self): model = DictVectorizer() data = [{'amy': 1., 'chin': 200.}, {'nice': 3., 'amy': 1.}] model.fit_transform(data) - model_coreml = coremltools.converters.sklearn.convert(model) + try: + model_coreml = coremltools.converters.sklearn.convert(model) + except NameError as e: + raise AssertionError( + "Unable to use coremltools, coremltools.__version__=%r, " + "onnx.__version__=%r, sklearn.__version__=%r, " + "sys.platform=%r." % ( + coremltools.__version__, onnx.__version__, + sklearn.__version__, sys.platform)) from e model_onnx = convert(model_coreml.get_spec()) self.assertTrue(model_onnx is not None) dump_data_and_model(data, model, model_onnx, basename="CmlDictVectorizer-OneOff-SkipDim1", diff --git a/tests/coreml/test_cml_GLMClassifierConverter.py b/tests/coreml/test_cml_GLMClassifierConverter.py index 5c71c71d5..61d36ca43 100644 --- a/tests/coreml/test_cml_GLMClassifierConverter.py +++ b/tests/coreml/test_cml_GLMClassifierConverter.py @@ -4,6 +4,7 @@ Tests CoreML GLMClassifier converter. """ import unittest +from distutils.version import StrictVersion import numpy from sklearn.datasets import load_iris from sklearn.linear_model import LogisticRegression @@ -31,6 +32,9 @@ def validate_zipmap(self, model): self.assertEqual(len(node.output), 1) self.assertTrue('classProbability' in node.output) + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_glm_classifier(self): iris = load_iris() X = iris.data[:, :2] diff --git a/tests/coreml/test_cml_GLMRegressorConverter.py b/tests/coreml/test_cml_GLMRegressorConverter.py index 4c79f9644..00289e034 100644 --- a/tests/coreml/test_cml_GLMRegressorConverter.py +++ b/tests/coreml/test_cml_GLMRegressorConverter.py @@ -4,6 +4,7 @@ Tests CoreML GLMRegressor converter. """ import unittest +from distutils.version import StrictVersion import numpy try: from sklearn.impute import SimpleImputer as Imputer @@ -23,6 +24,9 @@ class TestCoreMLGLMRegressorConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_glm_regressor(self): X, y = make_regression(n_features=4, random_state=0) diff --git a/tests/coreml/test_cml_ImputerConverter.py b/tests/coreml/test_cml_ImputerConverter.py index a6bb72f16..bf9ab71e7 100644 --- a/tests/coreml/test_cml_ImputerConverter.py +++ b/tests/coreml/test_cml_ImputerConverter.py @@ -3,8 +3,9 @@ """ Tests CoreML Imputer converter. """ -import numpy as np import unittest +from distutils.version import StrictVersion +import numpy as np try: from sklearn.impute import SimpleImputer as Imputer import sklearn.preprocessing @@ -14,11 +15,15 @@ except ImportError: from sklearn.preprocessing import Imputer import sklearn.preprocessing +import coremltools from onnxmltools.utils import dump_data_and_model class TestCoreMLImputerConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_imputer(self): try: model = Imputer(missing_values='NaN', strategy='mean', axis=0) diff --git a/tests/coreml/test_cml_OneHotEncoderConverter.py b/tests/coreml/test_cml_OneHotEncoderConverter.py index 019b67fda..f85e460e0 100644 --- a/tests/coreml/test_cml_OneHotEncoderConverter.py +++ b/tests/coreml/test_cml_OneHotEncoderConverter.py @@ -3,10 +3,13 @@ """ Main functions to convert machine learned model from *Core ML* model to *ONNX*. """ +import sys import os import unittest import warnings +from distutils.version import StrictVersion import numpy +import onnx try: from sklearn.impute import SimpleImputer as Imputer import sklearn.preprocessing @@ -23,6 +26,9 @@ class TestCoremlOneHotEncoderConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_one_hot_encoder(self): script_dir = os.path.dirname(__file__) relative_path = "../data/onehot_simple.mlmodel" diff --git a/tests/coreml/test_cml_ScalerConverter.py b/tests/coreml/test_cml_ScalerConverter.py index 88cb944f6..3220b1960 100644 --- a/tests/coreml/test_cml_ScalerConverter.py +++ b/tests/coreml/test_cml_ScalerConverter.py @@ -4,6 +4,7 @@ Tests CoreML Scaler converter. """ import unittest +from distutils.version import StrictVersion import numpy try: from sklearn.impute import SimpleImputer as Imputer @@ -21,6 +22,9 @@ class TestCoreMLScalerConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_scaler(self): model = StandardScaler() data = numpy.array([[0, 0, 3], [1, 1, 0], [0, 2, 1], [1, 0, 2]], dtype=numpy.float32) diff --git a/tests/coreml/test_cml_SupportVectorClassifierConverter.py b/tests/coreml/test_cml_SupportVectorClassifierConverter.py index 04a64cd8d..c3deb47ab 100644 --- a/tests/coreml/test_cml_SupportVectorClassifierConverter.py +++ b/tests/coreml/test_cml_SupportVectorClassifierConverter.py @@ -3,6 +3,7 @@ """ Tests CoreML SupportVectorClassifier converter. """ +from distutils.version import StrictVersion try: from sklearn.impute import SimpleImputer as Imputer import sklearn.preprocessing @@ -59,6 +60,9 @@ def validate_zipmap(self, model): self.assertEqual(len(node.output), 1) self.assertTrue('classProbability' in node.output) + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_support_vector_classifier_binary_no_prob(self): svm, X = self._fit_binary_classification(SVC(gamma=0.5)) svm_coreml = coremltools.converters.sklearn.convert(svm) @@ -71,6 +75,9 @@ def test_support_vector_classifier_binary_no_prob(self): dump_data_and_model(X, svm, svm_onnx, basename="CmlBinSVC-Out0", allow_failure=True) + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_support_vector_classifier_binary_with_prob(self): svm, X = self._fit_binary_classification(SVC(probability=True, gamma=0.5)) svm_coreml = coremltools.converters.sklearn.convert(svm) @@ -79,6 +86,9 @@ def test_support_vector_classifier_binary_with_prob(self): self.validate_zipmap(svm_onnx) self._check_model_outputs(svm_onnx, ['classLabel', 'classProbability']) + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_support_vector_classifier_multiclass_no_prob(self): svm, X = self._fit_multi_classification(SVC(gamma=0.5)) svm_coreml = coremltools.converters.sklearn.convert(svm) @@ -88,6 +98,9 @@ def test_support_vector_classifier_multiclass_no_prob(self): self.assertEqual(len(nodes), 1) self._check_model_outputs(svm_onnx, ['classLabel']) + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_support_vector_classifier_multiclass_with_prob(self): svm, X = self._fit_multi_classification(SVC(probability=True, gamma=0.5)) svm_coreml = coremltools.converters.sklearn.convert(svm) diff --git a/tests/coreml/test_cml_SupportVectorRegressorConverter.py b/tests/coreml/test_cml_SupportVectorRegressorConverter.py index f1fc3283e..57745a06c 100644 --- a/tests/coreml/test_cml_SupportVectorRegressorConverter.py +++ b/tests/coreml/test_cml_SupportVectorRegressorConverter.py @@ -3,6 +3,7 @@ """ Tests SupportVectorRegressor converter. """ +from distutils.version import StrictVersion try: from sklearn.impute import SimpleImputer as Imputer import sklearn.preprocessing @@ -22,6 +23,9 @@ class TestCoreMLSupportVectorRegressorConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_support_vector_regressor(self): X, y = make_regression(n_features=4, random_state=0) diff --git a/tests/coreml/test_cml_TreeEnsembleClassifierConverter.py b/tests/coreml/test_cml_TreeEnsembleClassifierConverter.py index 9dd370de1..7d453f8f6 100644 --- a/tests/coreml/test_cml_TreeEnsembleClassifierConverter.py +++ b/tests/coreml/test_cml_TreeEnsembleClassifierConverter.py @@ -4,6 +4,7 @@ Tests CoreML TreeEnsembleClassifier converter. """ import unittest +from distutils.version import StrictVersion import numpy try: from sklearn.impute import SimpleImputer as Imputer @@ -29,6 +30,9 @@ def validate_zipmap(self, model): self.assertEqual(len(node.output), 1) self.assertTrue('classProbability' in node.output) + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_tree_ensemble_classifier(self): X = numpy.array([[0, 1], [1, 1], [2, 0]], dtype=numpy.float32) y = [1, 0, 1] diff --git a/tests/coreml/test_cml_TreeEnsembleRegressorConverter.py b/tests/coreml/test_cml_TreeEnsembleRegressorConverter.py index 7fbe891bc..70dcb3aba 100644 --- a/tests/coreml/test_cml_TreeEnsembleRegressorConverter.py +++ b/tests/coreml/test_cml_TreeEnsembleRegressorConverter.py @@ -4,6 +4,7 @@ Tests CoreML TreeEnsembleRegressor converter. """ import unittest +from distutils.version import StrictVersion import numpy try: from sklearn.impute import SimpleImputer as Imputer @@ -22,6 +23,9 @@ class TestCoreMLTreeEnsembleRegressorConverter(unittest.TestCase): + @unittest.skipIf( + StrictVersion(coremltools.__version__) > StrictVersion("3.1"), + reason="untested") def test_tree_ensemble_regressor(self): X, y = make_regression(n_features=4, random_state=0) model = RandomForestRegressor().fit(X, y) diff --git a/tests/lightgbm/test_lightgbm_tree_structure.py b/tests/lightgbm/test_lightgbm_tree_structure.py index 778854023..c7adcabcb 100644 --- a/tests/lightgbm/test_lightgbm_tree_structure.py +++ b/tests/lightgbm/test_lightgbm_tree_structure.py @@ -3,8 +3,7 @@ import unittest import copy from onnxmltools.convert.lightgbm.operator_converters.LightGbm import ( - modify_tree_for_rule_in_set -) + modify_tree_for_rule_in_set) def count_nodes(tree, done=None): diff --git a/tests/lightgbm/test_objective_functions.py b/tests/lightgbm/test_objective_functions.py index 402f189e7..a49f6cac9 100644 --- a/tests/lightgbm/test_objective_functions.py +++ b/tests/lightgbm/test_objective_functions.py @@ -15,7 +15,7 @@ _N_ROWS=10_000 _N_COLS=10 _N_DECIMALS=5 -_FRAC = 0.9999 +_FRAC = 0.9997 _X = pd.DataFrame(np.random.random(size=(_N_ROWS, _N_COLS))) _Y = pd.Series(np.random.random(size=_N_ROWS))