From 21cd8d7db14ac47c3a3699710ae2ab684980957a Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sun, 12 Feb 2023 22:35:28 +0100 Subject: [PATCH 01/15] Add sklearn marker --- tests/conftest.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/conftest.py b/tests/conftest.py index cf3f33834..89da5fca4 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -174,6 +174,10 @@ def pytest_sessionfinish() -> None: logger.info("{} is killed".format(worker)) +def pytest_configure(config): + config.addinivalue_line("markers", "sklearn: marks tests that use scikit-learn") + + def pytest_addoption(parser): parser.addoption( "--long", From e2dcdc2a840559c9b5f18efa680e9ac1cca0f065 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sun, 12 Feb 2023 23:10:46 +0100 Subject: [PATCH 02/15] Mark tests that use scikit-learn --- .../test_sklearn_extension.py | 52 +++++++++++++++++++ tests/test_flows/test_flow.py | 10 ++++ tests/test_flows/test_flow_functions.py | 7 +++ tests/test_runs/test_run.py | 3 ++ tests/test_runs/test_run_functions.py | 29 +++++++++++ tests/test_setups/test_setup_functions.py | 5 ++ tests/test_study/test_study_examples.py | 2 + 7 files changed, 108 insertions(+) diff --git a/tests/test_extensions/test_sklearn_extension/test_sklearn_extension.py b/tests/test_extensions/test_sklearn_extension/test_sklearn_extension.py index 1046970f3..86ae419d2 100644 --- a/tests/test_extensions/test_sklearn_extension/test_sklearn_extension.py +++ b/tests/test_extensions/test_sklearn_extension/test_sklearn_extension.py @@ -15,6 +15,7 @@ import numpy as np import pandas as pd +import pytest import scipy.optimize import scipy.stats import sklearn.base @@ -176,6 +177,7 @@ def _serialization_test_helper( return serialization, new_model + @pytest.mark.sklearn def test_serialize_model(self): model = sklearn.tree.DecisionTreeClassifier( criterion="entropy", max_features="auto", max_leaf_nodes=2000 @@ -265,6 +267,7 @@ def test_serialize_model(self): self.assertEqual(serialization.dependencies, version_fixture) self.assertDictEqual(structure, structure_fixture) + @pytest.mark.sklearn def test_can_handle_flow(self): openml.config.server = self.production_server @@ -275,6 +278,7 @@ def test_can_handle_flow(self): openml.config.server = self.test_server + @pytest.mark.sklearn def test_serialize_model_clustering(self): model = sklearn.cluster.KMeans() @@ -367,6 +371,7 @@ def test_serialize_model_clustering(self): assert serialization.dependencies == version_fixture assert structure == fixture_structure + @pytest.mark.sklearn def test_serialize_model_with_subcomponent(self): model = sklearn.ensemble.AdaBoostClassifier( n_estimators=100, base_estimator=sklearn.tree.DecisionTreeClassifier() @@ -427,6 +432,7 @@ def test_serialize_model_with_subcomponent(self): ) self.assertDictEqual(structure, fixture_structure) + @pytest.mark.sklearn def test_serialize_pipeline(self): scaler = sklearn.preprocessing.StandardScaler(with_mean=False) dummy = sklearn.dummy.DummyClassifier(strategy="prior") @@ -496,6 +502,7 @@ def test_serialize_pipeline(self): self.assertIsNot(new_model.steps[0][1], model.steps[0][1]) self.assertIsNot(new_model.steps[1][1], model.steps[1][1]) + @pytest.mark.sklearn def test_serialize_pipeline_clustering(self): scaler = sklearn.preprocessing.StandardScaler(with_mean=False) km = sklearn.cluster.KMeans() @@ -564,6 +571,7 @@ def test_serialize_pipeline_clustering(self): self.assertIsNot(new_model.steps[0][1], model.steps[0][1]) self.assertIsNot(new_model.steps[1][1], model.steps[1][1]) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", @@ -622,6 +630,7 @@ def test_serialize_column_transformer(self): self.assertEqual(serialization.description, fixture_description) self.assertDictEqual(structure, fixture_structure) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", @@ -688,6 +697,7 @@ def test_serialize_column_transformer_pipeline(self): self.assertDictEqual(structure, fixture_structure) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="Pipeline processing behaviour updated" ) @@ -756,6 +766,7 @@ def test_serialize_feature_union(self): ) self.assertIs(new_model.transformer_list[1][1], "drop") + @pytest.mark.sklearn def test_serialize_feature_union_switched_names(self): ohe_params = {"categories": "auto"} if LooseVersion(sklearn.__version__) >= "0.20" else {} ohe = sklearn.preprocessing.OneHotEncoder(**ohe_params) @@ -796,6 +807,7 @@ def test_serialize_feature_union_switched_names(self): "ohe=sklearn.preprocessing.{}.StandardScaler)".format(module_name_encoder, scaler_name), ) + @pytest.mark.sklearn def test_serialize_complex_flow(self): ohe = sklearn.preprocessing.OneHotEncoder(handle_unknown="ignore") scaler = sklearn.preprocessing.StandardScaler(with_mean=False) @@ -856,6 +868,7 @@ def test_serialize_complex_flow(self): self.assertEqual(serialized.name, fixture_name) self.assertEqual(structure, fixture_structure) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.21", reason="Pipeline till 0.20 doesn't support 'passthrough'", @@ -951,6 +964,7 @@ def test_serialize_strings_as_pipeline_steps(self): self.assertIsInstance(extracted_info[2]["drop"], OpenMLFlow) self.assertEqual(extracted_info[2]["drop"].name, "drop") + @pytest.mark.sklearn def test_serialize_type(self): supported_types = [float, np.float32, np.float64, int, np.int32, np.int64] if LooseVersion(np.__version__) < "1.24": @@ -962,6 +976,7 @@ def test_serialize_type(self): deserialized = self.extension.flow_to_model(serialized) self.assertEqual(deserialized, supported_type) + @pytest.mark.sklearn def test_serialize_rvs(self): supported_rvs = [ scipy.stats.norm(loc=1, scale=5), @@ -977,11 +992,13 @@ def test_serialize_rvs(self): del supported_rv.dist self.assertEqual(deserialized.__dict__, supported_rv.__dict__) + @pytest.mark.sklearn def test_serialize_function(self): serialized = self.extension.model_to_flow(sklearn.feature_selection.chi2) deserialized = self.extension.flow_to_model(serialized) self.assertEqual(deserialized, sklearn.feature_selection.chi2) + @pytest.mark.sklearn def test_serialize_cvobject(self): methods = [sklearn.model_selection.KFold(3), sklearn.model_selection.LeaveOneOut()] fixtures = [ @@ -1031,6 +1048,7 @@ def test_serialize_cvobject(self): self.assertIsNot(m_new, m) self.assertIsInstance(m_new, type(method)) + @pytest.mark.sklearn def test_serialize_simple_parameter_grid(self): # We cannot easily test for scipy random variables in here, but they @@ -1078,6 +1096,7 @@ def test_serialize_simple_parameter_grid(self): del deserialized_params["estimator"] self.assertEqual(hpo_params, deserialized_params) + @pytest.mark.sklearn @unittest.skip( "This feature needs further reworking. If we allow several " "components, we need to register them all in the downstream " @@ -1132,6 +1151,7 @@ def test_serialize_advanced_grid(self): self.assertEqual(grid[1]["reduce_dim__k"], deserialized[1]["reduce_dim__k"]) self.assertEqual(grid[1]["classify__C"], deserialized[1]["classify__C"]) + @pytest.mark.sklearn def test_serialize_advanced_grid_fails(self): # This unit test is checking that the test we skip above would actually fail @@ -1151,6 +1171,7 @@ def test_serialize_advanced_grid_fails(self): ): self.extension.model_to_flow(clf) + @pytest.mark.sklearn def test_serialize_resampling(self): kfold = sklearn.model_selection.StratifiedKFold(n_splits=4, shuffle=True) serialized = self.extension.model_to_flow(kfold) @@ -1159,6 +1180,7 @@ def test_serialize_resampling(self): self.assertEqual(str(deserialized), str(kfold)) self.assertIsNot(deserialized, kfold) + @pytest.mark.sklearn def test_hypothetical_parameter_values(self): # The hypothetical parameter values of true, 1, 0.1 formatted as a # string (and their correct serialization and deserialization) an only @@ -1172,6 +1194,7 @@ def test_hypothetical_parameter_values(self): self.assertEqual(deserialized.get_params(), model.get_params()) self.assertIsNot(deserialized, model) + @pytest.mark.sklearn def test_gaussian_process(self): opt = scipy.optimize.fmin_l_bfgs_b kernel = sklearn.gaussian_process.kernels.Matern() @@ -1182,6 +1205,7 @@ def test_gaussian_process(self): ): self.extension.model_to_flow(gp) + @pytest.mark.sklearn def test_error_on_adding_component_multiple_times_to_flow(self): # this function implicitly checks # - openml.flows._check_multiple_occurence_of_component_in_flow() @@ -1206,6 +1230,7 @@ def test_error_on_adding_component_multiple_times_to_flow(self): with self.assertRaisesRegex(ValueError, fixture): self.extension.model_to_flow(pipeline2) + @pytest.mark.sklearn def test_subflow_version_propagated(self): this_directory = os.path.dirname(os.path.abspath(__file__)) tests_directory = os.path.abspath(os.path.join(this_directory, "..", "..")) @@ -1230,12 +1255,14 @@ def test_subflow_version_propagated(self): ), ) + @pytest.mark.sklearn @mock.patch("warnings.warn") def test_check_dependencies(self, warnings_mock): dependencies = ["sklearn==0.1", "sklearn>=99.99.99", "sklearn>99.99.99"] for dependency in dependencies: self.assertRaises(ValueError, self.extension._check_dependencies, dependency) + @pytest.mark.sklearn def test_illegal_parameter_names(self): # illegal name: estimators clf1 = sklearn.ensemble.VotingClassifier( @@ -1255,6 +1282,7 @@ def test_illegal_parameter_names(self): for case in cases: self.assertRaises(PyOpenMLError, self.extension.model_to_flow, case) + @pytest.mark.sklearn def test_paralizable_check(self): # using this model should pass the test (if param distribution is # legal) @@ -1304,6 +1332,7 @@ def test_paralizable_check(self): with self.assertRaises(PyOpenMLError): self.extension._prevent_optimize_n_jobs(model) + @pytest.mark.sklearn def test__get_fn_arguments_with_defaults(self): sklearn_version = LooseVersion(sklearn.__version__) if sklearn_version < "0.19": @@ -1361,6 +1390,7 @@ def test__get_fn_arguments_with_defaults(self): self.assertSetEqual(set(defaults.keys()), set(defaults.keys()) - defaultless) self.assertSetEqual(defaultless, defaultless - set(defaults.keys())) + @pytest.mark.sklearn def test_deserialize_with_defaults(self): # used the 'initialize_with_defaults' flag of the deserialization # method to return a flow that contains default hyperparameter @@ -1396,6 +1426,7 @@ def test_deserialize_with_defaults(self): self.extension.model_to_flow(pipe_deserialized), ) + @pytest.mark.sklearn def test_deserialize_adaboost_with_defaults(self): # used the 'initialize_with_defaults' flag of the deserialization # method to return a flow that contains default hyperparameter @@ -1434,6 +1465,7 @@ def test_deserialize_adaboost_with_defaults(self): self.extension.model_to_flow(pipe_deserialized), ) + @pytest.mark.sklearn def test_deserialize_complex_with_defaults(self): # used the 'initialize_with_defaults' flag of the deserialization # method to return a flow that contains default hyperparameter @@ -1477,6 +1509,7 @@ def test_deserialize_complex_with_defaults(self): self.extension.model_to_flow(pipe_deserialized), ) + @pytest.mark.sklearn def test_openml_param_name_to_sklearn(self): scaler = sklearn.preprocessing.StandardScaler(with_mean=False) boosting = sklearn.ensemble.AdaBoostClassifier( @@ -1511,6 +1544,7 @@ def test_openml_param_name_to_sklearn(self): openml_name = "%s(%s)_%s" % (subflow.name, subflow.version, splitted[-1]) self.assertEqual(parameter.full_name, openml_name) + @pytest.mark.sklearn def test_obtain_parameter_values_flow_not_from_server(self): model = sklearn.linear_model.LogisticRegression(solver="lbfgs") flow = self.extension.model_to_flow(model) @@ -1532,6 +1566,7 @@ def test_obtain_parameter_values_flow_not_from_server(self): with self.assertRaisesRegex(ValueError, msg): self.extension.obtain_parameter_values(flow) + @pytest.mark.sklearn def test_obtain_parameter_values(self): model = sklearn.model_selection.RandomizedSearchCV( @@ -1557,6 +1592,7 @@ def test_obtain_parameter_values(self): self.assertEqual(parameter["oml:value"], "5") self.assertEqual(parameter["oml:component"], 2) + @pytest.mark.sklearn def test_numpy_type_allowed_in_flow(self): """Simple numpy types should be serializable.""" dt = sklearn.tree.DecisionTreeClassifier( @@ -1564,6 +1600,7 @@ def test_numpy_type_allowed_in_flow(self): ) self.extension.model_to_flow(dt) + @pytest.mark.sklearn def test_numpy_array_not_allowed_in_flow(self): """Simple numpy arrays should not be serializable.""" bin = sklearn.preprocessing.MultiLabelBinarizer(classes=np.asarray([1, 2, 3])) @@ -1581,6 +1618,7 @@ def setUp(self): ################################################################################################ # Test methods for performing runs with this extension module + @pytest.mark.sklearn def test_run_model_on_task(self): task = openml.tasks.get_task(1) # anneal; crossvalidation # using most_frequent imputer since dataset has mixed types and to keep things simple @@ -1592,6 +1630,7 @@ def test_run_model_on_task(self): ) openml.runs.run_model_on_task(pipe, task, dataset_format="array") + @pytest.mark.sklearn def test_seed_model(self): # randomized models that are initialized without seeds, can be seeded randomized_clfs = [ @@ -1634,6 +1673,7 @@ def test_seed_model(self): if idx == 1: self.assertEqual(clf.cv.random_state, 56422) + @pytest.mark.sklearn def test_seed_model_raises(self): # the _set_model_seed_where_none should raise exception if random_state is # anything else than an int @@ -1646,6 +1686,7 @@ def test_seed_model_raises(self): with self.assertRaises(ValueError): self.extension.seed_model(model=clf, seed=42) + @pytest.mark.sklearn def test_run_model_on_fold_classification_1_array(self): task = openml.tasks.get_task(1) # anneal; crossvalidation @@ -1702,6 +1743,7 @@ def test_run_model_on_fold_classification_1_array(self): check_scores=False, ) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.21", reason="SimpleImputer, ColumnTransformer available only after 0.19 and " @@ -1773,6 +1815,7 @@ def test_run_model_on_fold_classification_1_dataframe(self): check_scores=False, ) + @pytest.mark.sklearn def test_run_model_on_fold_classification_2(self): task = openml.tasks.get_task(7) # kr-vs-kp; crossvalidation @@ -1826,6 +1869,7 @@ def test_run_model_on_fold_classification_2(self): check_scores=False, ) + @pytest.mark.sklearn def test_run_model_on_fold_classification_3(self): class HardNaiveBayes(sklearn.naive_bayes.GaussianNB): # class for testing a naive bayes classifier that does not allow soft @@ -1896,6 +1940,7 @@ def predict_proba(*args, **kwargs): X_test.shape[0] * len(task.class_labels), ) + @pytest.mark.sklearn def test_run_model_on_fold_regression(self): # There aren't any regression tasks on the test server openml.config.server = self.production_server @@ -1945,6 +1990,7 @@ def test_run_model_on_fold_regression(self): check_scores=False, ) + @pytest.mark.sklearn def test_run_model_on_fold_clustering(self): # There aren't any regression tasks on the test server openml.config.server = self.production_server @@ -1987,6 +2033,7 @@ def test_run_model_on_fold_clustering(self): check_scores=False, ) + @pytest.mark.sklearn def test__extract_trace_data(self): param_grid = { @@ -2038,6 +2085,7 @@ def test__extract_trace_data(self): param_value = json.loads(trace_iteration.parameters[param_in_trace]) self.assertTrue(param_value in param_grid[param]) + @pytest.mark.sklearn def test_trim_flow_name(self): import re @@ -2100,6 +2148,7 @@ def test_trim_flow_name(self): "weka.IsolationForest", SklearnExtension.trim_flow_name("weka.IsolationForest") ) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.21", reason="SimpleImputer, ColumnTransformer available only after 0.19 and " @@ -2189,6 +2238,7 @@ def test_run_on_model_with_empty_steps(self): self.assertEqual(len(new_model.named_steps), 3) self.assertEqual(new_model.named_steps["dummystep"], "passthrough") + @pytest.mark.sklearn def test_sklearn_serialization_with_none_step(self): msg = ( "Cannot serialize objects of None type. Please use a valid " @@ -2201,6 +2251,7 @@ def test_sklearn_serialization_with_none_step(self): with self.assertRaisesRegex(ValueError, msg): self.extension.model_to_flow(clf) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", @@ -2236,6 +2287,7 @@ def test_failed_serialization_of_custom_class(self): else: raise Exception(e) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", diff --git a/tests/test_flows/test_flow.py b/tests/test_flows/test_flow.py index 50d152192..c3c72f267 100644 --- a/tests/test_flows/test_flow.py +++ b/tests/test_flows/test_flow.py @@ -7,6 +7,7 @@ import re import time from unittest import mock +import pytest import scipy.stats import sklearn @@ -148,6 +149,7 @@ def test_from_xml_to_xml(self): self.assertEqual(new_xml, flow_xml) + @pytest.mark.sklearn def test_to_xml_from_xml(self): scaler = sklearn.preprocessing.StandardScaler(with_mean=False) boosting = sklearn.ensemble.AdaBoostClassifier( @@ -166,6 +168,7 @@ def test_to_xml_from_xml(self): openml.flows.functions.assert_flows_equal(new_flow, flow) self.assertIsNot(new_flow, flow) + @pytest.mark.sklearn def test_publish_flow(self): flow = openml.OpenMLFlow( name="sklearn.dummy.DummyClassifier", @@ -191,6 +194,7 @@ def test_publish_flow(self): TestBase.logger.info("collected from {}: {}".format(__file__.split("/")[-1], flow.flow_id)) self.assertIsInstance(flow.flow_id, int) + @pytest.mark.sklearn @mock.patch("openml.flows.functions.flow_exists") def test_publish_existing_flow(self, flow_exists_mock): clf = sklearn.tree.DecisionTreeClassifier(max_depth=2) @@ -206,6 +210,7 @@ def test_publish_existing_flow(self, flow_exists_mock): self.assertTrue("OpenMLFlow already exists" in context_manager.exception.message) + @pytest.mark.sklearn def test_publish_flow_with_similar_components(self): clf = sklearn.ensemble.VotingClassifier( [("lr", sklearn.linear_model.LogisticRegression(solver="lbfgs"))] @@ -259,6 +264,7 @@ def test_publish_flow_with_similar_components(self): TestBase._mark_entity_for_removal("flow", (flow3.flow_id, flow3.name)) TestBase.logger.info("collected from {}: {}".format(__file__.split("/")[-1], flow3.flow_id)) + @pytest.mark.sklearn def test_semi_legal_flow(self): # TODO: Test if parameters are set correctly! # should not throw error as it contains two differentiable forms of @@ -275,6 +281,7 @@ def test_semi_legal_flow(self): TestBase._mark_entity_for_removal("flow", (flow.flow_id, flow.name)) TestBase.logger.info("collected from {}: {}".format(__file__.split("/")[-1], flow.flow_id)) + @pytest.mark.sklearn @mock.patch("openml.flows.functions.get_flow") @mock.patch("openml.flows.functions.flow_exists") @mock.patch("openml._api_calls._perform_api_call") @@ -331,6 +338,7 @@ def test_publish_error(self, api_call_mock, flow_exists_mock, get_flow_mock): self.assertEqual(context_manager.exception.args[0], fixture) self.assertEqual(get_flow_mock.call_count, 2) + @pytest.mark.sklearn def test_illegal_flow(self): # should throw error as it contains two imputers illegal = sklearn.pipeline.Pipeline( @@ -359,6 +367,7 @@ def get_sentinel(): flow_id = openml.flows.flow_exists(name, version) self.assertFalse(flow_id) + @pytest.mark.sklearn def test_existing_flow_exists(self): # create a flow nb = sklearn.naive_bayes.GaussianNB() @@ -397,6 +406,7 @@ def test_existing_flow_exists(self): ) self.assertEqual(downloaded_flow_id, flow.flow_id) + @pytest.mark.sklearn def test_sklearn_to_upload_to_flow(self): iris = sklearn.datasets.load_iris() X = iris.data diff --git a/tests/test_flows/test_flow_functions.py b/tests/test_flows/test_flow_functions.py index fe058df23..532fb1d1b 100644 --- a/tests/test_flows/test_flow_functions.py +++ b/tests/test_flows/test_flow_functions.py @@ -271,6 +271,7 @@ def test_are_flows_equal_ignore_if_older(self): ) assert_flows_equal(flow, flow, ignore_parameter_values_on_older_children=None) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="OrdinalEncoder introduced in 0.20. " @@ -302,6 +303,7 @@ def test_get_flow1(self): flow = openml.flows.get_flow(1) self.assertIsNone(flow.external_version) + @pytest.mark.sklearn def test_get_flow_reinstantiate_model(self): model = ensemble.RandomForestClassifier(n_estimators=33) extension = openml.extensions.get_extension_by_model(model) @@ -323,6 +325,7 @@ def test_get_flow_reinstantiate_model_no_extension(self): reinstantiate=True, ) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) == "0.19.1", reason="Requires scikit-learn!=0.19.1, because target flow is from that version.", @@ -340,6 +343,7 @@ def test_get_flow_with_reinstantiate_strict_with_wrong_version_raises_exception( strict_version=True, ) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "1" and LooseVersion(sklearn.__version__) != "1.0.0", reason="Requires scikit-learn < 1.0.1." @@ -352,6 +356,7 @@ def test_get_flow_reinstantiate_flow_not_strict_post_1(self): assert flow.flow_id is None assert "sklearn==1.0.0" not in flow.dependencies + @pytest.mark.sklearn @unittest.skipIf( (LooseVersion(sklearn.__version__) < "0.23.2") or ("1.0" < LooseVersion(sklearn.__version__)), @@ -364,6 +369,7 @@ def test_get_flow_reinstantiate_flow_not_strict_023_and_024(self): assert flow.flow_id is None assert "sklearn==0.23.1" not in flow.dependencies + @pytest.mark.sklearn @unittest.skipIf( "0.23" < LooseVersion(sklearn.__version__), reason="Requires scikit-learn<=0.23, because the scikit-learn module structure changed.", @@ -374,6 +380,7 @@ def test_get_flow_reinstantiate_flow_not_strict_pre_023(self): assert flow.flow_id is None assert "sklearn==0.19.1" not in flow.dependencies + @pytest.mark.sklearn def test_get_flow_id(self): if self.long_version: list_all = openml.utils._list_all diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 88c998bc3..39a93bf4a 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -102,6 +102,7 @@ def _check_array(array, type_): else: self.assertIsNone(run_prime_trace_content) + @pytest.mark.sklearn def test_to_from_filesystem_vanilla(self): model = Pipeline( @@ -137,6 +138,7 @@ def test_to_from_filesystem_vanilla(self): "collected from {}: {}".format(__file__.split("/")[-1], run_prime.run_id) ) + @pytest.mark.sklearn @pytest.mark.flaky() def test_to_from_filesystem_search(self): @@ -189,6 +191,7 @@ def test_to_from_filesystem_no_model(self): with self.assertRaises(ValueError, msg="Could not find model.pkl"): openml.runs.OpenMLRun.from_filesystem(cache_path) + @pytest.mark.sklearn def test_publish_with_local_loaded_flow(self): """ Publish a run tied to a local flow after it has first been saved to diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1e92613c3..ca38750d8 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -20,6 +20,7 @@ import unittest import warnings import pandas as pd +import pytest import openml.extensions.sklearn from openml.testing import TestBase, SimpleImputer, CustomImputer @@ -387,6 +388,7 @@ def _check_sample_evaluations( self.assertGreater(evaluation, 0) self.assertLess(evaluation, max_time_allowed) + @pytest.mark.sklearn def test_run_regression_on_classif_task(self): task_id = 115 # diabetes; crossvalidation @@ -404,6 +406,7 @@ def test_run_regression_on_classif_task(self): dataset_format="array", ) + @pytest.mark.sklearn def test_check_erronous_sklearn_flow_fails(self): task_id = 115 # diabetes; crossvalidation task = openml.tasks.get_task(task_id) @@ -578,6 +581,7 @@ def _run_and_upload_regression( sentinel=sentinel, ) + @pytest.mark.sklearn def test_run_and_upload_logistic_regression(self): lr = LogisticRegression(solver="lbfgs", max_iter=1000) task_id = self.TEST_SERVER_TASK_SIMPLE["task_id"] @@ -585,6 +589,7 @@ def test_run_and_upload_logistic_regression(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(lr, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.sklearn def test_run_and_upload_linear_regression(self): lr = LinearRegression() task_id = self.TEST_SERVER_TASK_REGRESSION["task_id"] @@ -614,6 +619,7 @@ def test_run_and_upload_linear_regression(self): n_test_obs = self.TEST_SERVER_TASK_REGRESSION["n_test_obs"] self._run_and_upload_regression(lr, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.sklearn def test_run_and_upload_pipeline_dummy_pipeline(self): pipeline1 = Pipeline( @@ -627,6 +633,7 @@ def test_run_and_upload_pipeline_dummy_pipeline(self): n_test_obs = self.TEST_SERVER_TASK_SIMPLE["n_test_obs"] self._run_and_upload_classification(pipeline1, task_id, n_missing_vals, n_test_obs, "62501") + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", @@ -689,6 +696,7 @@ def get_ct_cf(nominal_indices, numeric_indices): sentinel=sentinel, ) + @pytest.mark.sklearn @unittest.skip("https://github.com/openml/OpenML/issues/1180") @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", @@ -740,6 +748,7 @@ def test_run_and_upload_knn_pipeline(self, warnings_mock): call_count += 1 self.assertEqual(call_count, 3) + @pytest.mark.sklearn def test_run_and_upload_gridsearch(self): gridsearch = GridSearchCV( BaggingClassifier(base_estimator=SVC()), @@ -758,6 +767,7 @@ def test_run_and_upload_gridsearch(self): ) self.assertEqual(len(run.trace.trace_iterations), 9) + @pytest.mark.sklearn def test_run_and_upload_randomsearch(self): randomsearch = RandomizedSearchCV( RandomForestClassifier(n_estimators=5), @@ -789,6 +799,7 @@ def test_run_and_upload_randomsearch(self): trace = openml.runs.get_run_trace(run.run_id) self.assertEqual(len(trace.trace_iterations), 5) + @pytest.mark.sklearn def test_run_and_upload_maskedarrays(self): # This testcase is important for 2 reasons: # 1) it verifies the correct handling of masked arrays (not all @@ -811,6 +822,7 @@ def test_run_and_upload_maskedarrays(self): ########################################################################## + @pytest.mark.sklearn def test_learning_curve_task_1(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -830,6 +842,7 @@ def test_learning_curve_task_1(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) + @pytest.mark.sklearn def test_learning_curve_task_2(self): task_id = 801 # diabates dataset num_test_instances = 6144 # for learning curve @@ -861,6 +874,7 @@ def test_learning_curve_task_2(self): ) self._check_sample_evaluations(run.sample_evaluations, num_repeats, num_folds, num_samples) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.21", reason="Pipelines don't support indexing (used for the assert check)", @@ -940,6 +954,7 @@ def _test_local_evaluations(self, run): self.assertGreaterEqual(alt_scores[idx], 0) self.assertLessEqual(alt_scores[idx], 1) + @pytest.mark.sklearn def test_local_run_swapped_parameter_order_model(self): clf = DecisionTreeClassifier() australian_task = 595 # Australian; crossvalidation @@ -955,6 +970,7 @@ def test_local_run_swapped_parameter_order_model(self): self._test_local_evaluations(run) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -984,6 +1000,7 @@ def test_local_run_swapped_parameter_order_flow(self): self._test_local_evaluations(run) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1021,6 +1038,7 @@ def test_online_run_metric_score(self): self._test_local_evaluations(run) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1082,6 +1100,7 @@ def test_initialize_model_from_run(self): self.assertEqual(flowS.components["Imputer"].parameters["strategy"], '"most_frequent"') self.assertEqual(flowS.components["VarianceThreshold"].parameters["threshold"], "0.05") + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="SimpleImputer doesn't handle mixed type DataFrame as input", @@ -1136,6 +1155,7 @@ def test__run_exists(self): run_ids = run_exists(task.task_id, setup_exists) self.assertTrue(run_ids, msg=(run_ids, clf)) + @pytest.mark.sklearn def test_run_with_illegal_flow_id(self): # check the case where the user adds an illegal flow id to a # non-existing flo @@ -1154,6 +1174,7 @@ def test_run_with_illegal_flow_id(self): avoid_duplicate_runs=True, ) + @pytest.mark.sklearn def test_run_with_illegal_flow_id_after_load(self): # Same as `test_run_with_illegal_flow_id`, but test this error is also # caught if the run is stored to and loaded from disk first. @@ -1182,6 +1203,7 @@ def test_run_with_illegal_flow_id_after_load(self): TestBase._mark_entity_for_removal("run", loaded_run.run_id) TestBase.logger.info("collected from test_run_functions: {}".format(loaded_run.run_id)) + @pytest.mark.sklearn def test_run_with_illegal_flow_id_1(self): # Check the case where the user adds an illegal flow id to an existing # flow. Comes to a different value error than the previous test @@ -1206,6 +1228,7 @@ def test_run_with_illegal_flow_id_1(self): avoid_duplicate_runs=True, ) + @pytest.mark.sklearn def test_run_with_illegal_flow_id_1_after_load(self): # Same as `test_run_with_illegal_flow_id_1`, but test this error is # also caught if the run is stored to and loaded from disk first. @@ -1239,6 +1262,7 @@ def test_run_with_illegal_flow_id_1_after_load(self): openml.exceptions.PyOpenMLError, expected_message_regex, loaded_run.publish ) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="OneHotEncoder cannot handle mixed type DataFrame as input", @@ -1455,6 +1479,7 @@ def test_get_runs_list_by_tag(self): runs = openml.runs.list_runs(tag="curves") self.assertGreaterEqual(len(runs), 1) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", @@ -1490,6 +1515,7 @@ def test_run_on_dataset_with_missing_labels_dataframe(self): # repeat, fold, row_id, 6 confidences, prediction and correct label self.assertEqual(len(row), 12) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.20", reason="columntransformer introduction in 0.20.0", @@ -1541,6 +1567,7 @@ def test_get_uncached_run(self): with self.assertRaises(openml.exceptions.OpenMLCacheException): openml.runs.functions._get_cached_run(10) + @pytest.mark.sklearn def test_run_flow_on_task_downloaded_flow(self): model = sklearn.ensemble.RandomForestClassifier(n_estimators=33) flow = self.extension.model_to_flow(model) @@ -1633,6 +1660,7 @@ def test_format_prediction_task_regression(self): res = format_prediction(regression, *ignored_input) self.assertListEqual(res, [0] * 5) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.21", reason="couldn't perform local tests successfully w/o bloating RAM", @@ -1686,6 +1714,7 @@ def test__run_task_get_arffcontent_2(self, parallel_mock): scores, expected_scores, decimal=2 if os.name == "nt" else 7 ) + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.21", reason="couldn't perform local tests successfully w/o bloating RAM", diff --git a/tests/test_setups/test_setup_functions.py b/tests/test_setups/test_setup_functions.py index 464431b94..73a691d84 100644 --- a/tests/test_setups/test_setup_functions.py +++ b/tests/test_setups/test_setup_functions.py @@ -10,6 +10,7 @@ from openml.testing import TestBase from typing import Dict import pandas as pd +import pytest import sklearn.tree import sklearn.naive_bayes @@ -34,6 +35,7 @@ def setUp(self): self.extension = openml.extensions.sklearn.SklearnExtension() super().setUp() + @pytest.mark.sklearn def test_nonexisting_setup_exists(self): # first publish a non-existing flow sentinel = get_sentinel() @@ -81,6 +83,7 @@ def _existing_setup_exists(self, classif): setup_id = openml.setups.setup_exists(flow) self.assertEqual(setup_id, run.setup_id) + @pytest.mark.sklearn def test_existing_setup_exists_1(self): def side_effect(self): self.var_smoothing = 1e-9 @@ -95,10 +98,12 @@ def side_effect(self): nb = sklearn.naive_bayes.GaussianNB() self._existing_setup_exists(nb) + @pytest.mark.sklearn def test_exisiting_setup_exists_2(self): # Check a flow with one hyperparameter self._existing_setup_exists(sklearn.naive_bayes.GaussianNB()) + @pytest.mark.sklearn def test_existing_setup_exists_3(self): # Check a flow with many hyperparameters self._existing_setup_exists( diff --git a/tests/test_study/test_study_examples.py b/tests/test_study/test_study_examples.py index 682359a61..cc3294085 100644 --- a/tests/test_study/test_study_examples.py +++ b/tests/test_study/test_study_examples.py @@ -3,6 +3,7 @@ from openml.testing import TestBase from openml.extensions.sklearn import cat, cont +import pytest import sklearn import unittest from distutils.version import LooseVersion @@ -12,6 +13,7 @@ class TestStudyFunctions(TestBase): _multiprocess_can_split_ = True """Test the example code of Bischl et al. (2018)""" + @pytest.mark.sklearn @unittest.skipIf( LooseVersion(sklearn.__version__) < "0.24", reason="columntransformer introduction in 0.24.0", From 9ad6daeac503bfa486bb9f693ec0793c063f0a9c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Sun, 12 Feb 2023 23:11:06 +0100 Subject: [PATCH 03/15] Only run scikit-learn tests multiple times The generic tests that don't use scikit-learn should only be tested once (per platform). --- .github/workflows/test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ac6d8dbb..f09bcf6cf 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -29,6 +29,7 @@ jobs: scikit-learn: 0.23.1 code-cov: true os: ubuntu-latest + all-tests: true - os: windows-latest scikit-learn: 0.24.* fail-fast: false @@ -62,7 +63,9 @@ jobs: if: matrix.os == 'ubuntu-latest' run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi - pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov --reruns 5 --reruns-delay 1 + # Most of the time, running only the scikit-learn tests is sufficient + if [ ! ${{ matrix.all-tests }} ]; then sklearn='-m "sklearn"'; fi + pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn_select --reruns 5 --reruns-delay 1 - name: Run tests on Windows if: matrix.os == 'windows-latest' run: | # we need a separate step because of the bash-specific if-statement in the previous one. From d67b7cccbc27e4b5fd77200a92769f1c9028869f Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 20 Feb 2023 16:54:09 +0100 Subject: [PATCH 04/15] Rename for correct variable --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index f09bcf6cf..b9516185f 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -65,7 +65,7 @@ jobs: if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi # Most of the time, running only the scikit-learn tests is sufficient if [ ! ${{ matrix.all-tests }} ]; then sklearn='-m "sklearn"'; fi - pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn_select --reruns 5 --reruns-delay 1 + pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 - name: Run tests on Windows if: matrix.os == 'windows-latest' run: | # we need a separate step because of the bash-specific if-statement in the previous one. From f72ff8b6412184b1989c0f53909982fd2e83518d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Mon, 20 Feb 2023 16:54:56 +0100 Subject: [PATCH 05/15] Add sklearn mark for filesystem test --- tests/test_runs/test_run.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py index 39a93bf4a..e64ffeed6 100644 --- a/tests/test_runs/test_run.py +++ b/tests/test_runs/test_run.py @@ -175,6 +175,7 @@ def test_to_from_filesystem_search(self): "collected from {}: {}".format(__file__.split("/")[-1], run_prime.run_id) ) + @pytest.mark.sklearn def test_to_from_filesystem_no_model(self): model = Pipeline( From e89db283d3f6fa54b36a509fef98b94d37e4a25c Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Tue, 21 Feb 2023 10:05:29 +0100 Subject: [PATCH 06/15] Remove quotes around sklearn --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b9516185f..8d5e94f3b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -64,7 +64,7 @@ jobs: run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi # Most of the time, running only the scikit-learn tests is sufficient - if [ ! ${{ matrix.all-tests }} ]; then sklearn='-m "sklearn"'; fi + if [ ! ${{ matrix.all-tests }} ]; then sklearn='-m sklearn'; fi pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 - name: Run tests on Windows if: matrix.os == 'windows-latest' From b547589364adf34dfeba53db8201405eda7af36e Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 22 Feb 2023 09:43:53 +0100 Subject: [PATCH 07/15] Instead include sklearn in the matrix definition --- .github/workflows/test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8d5e94f3b..b92e7cad0 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,7 @@ jobs: python-version: [3.6, 3.7, 3.8] scikit-learn: [0.21.2, 0.22.2, 0.23.1, 0.24] os: [ubuntu-latest] + sklearn-only: [true] exclude: # no scikit-learn 0.21.2 release for Python 3.8 - python-version: 3.8 scikit-learn: 0.21.2 @@ -28,9 +29,10 @@ jobs: - python-version: 3.8 scikit-learn: 0.23.1 code-cov: true + sklearn-only: false os: ubuntu-latest - all-tests: true - os: windows-latest + sklearn-only: false scikit-learn: 0.24.* fail-fast: false max-parallel: 4 @@ -64,7 +66,8 @@ jobs: run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi # Most of the time, running only the scikit-learn tests is sufficient - if [ ! ${{ matrix.all-tests }} ]; then sklearn='-m sklearn'; fi + if [ ${{ matrix.sklearn-only }} ]; then sklearn='-m sklearn'; fi + echo pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 - name: Run tests on Windows if: matrix.os == 'windows-latest' From 184c89cdb5cad6d58d58884f153e64bcd672435d Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 22 Feb 2023 09:50:06 +0100 Subject: [PATCH 08/15] Update jobnames --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index b92e7cad0..205b73aa3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: test: - name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}) + name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) runs-on: ${{ matrix.os }} strategy: matrix: From 7a2a85057cc8c5a25f49db3dd9bac44e549329bf Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 22 Feb 2023 11:41:58 +0100 Subject: [PATCH 09/15] Add explicit false to jobname --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 205b73aa3..6040f05b3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: test: - name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) + name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, format('sk-only: {0}', ${{ matrix.sklearn-only }} )) runs-on: ${{ matrix.os }} strategy: matrix: From 657f0d3b07b0951c7fc9f8755e2dbff7e622cdae Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Wed, 22 Feb 2023 11:44:54 +0100 Subject: [PATCH 10/15] Remove space --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6040f05b3..07214ed66 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: test: - name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, format('sk-only: {0}', ${{ matrix.sklearn-only }} )) + name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, format("sk-only:{0}", ${{ matrix.sklearn-only }} )) runs-on: ${{ matrix.os }} strategy: matrix: From f36a040ec13af7e8d271d6b966bcb8bbc2bc5af1 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Wed, 22 Feb 2023 11:46:09 +0100 Subject: [PATCH 11/15] Add function inside of expression? --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 07214ed66..71f23e1d2 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: test: - name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, format("sk-only:{0}", ${{ matrix.sklearn-only }} )) + name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, ${{ format("sk-only:{0}", ${{ matrix.sklearn-only }} ) }} ) runs-on: ${{ matrix.os }} strategy: matrix: From ff34515624589018106d151f1d327049bc54bb06 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 22 Feb 2023 11:55:34 +0100 Subject: [PATCH 12/15] Do string testing instead --- .github/workflows/test.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 71f23e1d2..5ad7d9e08 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,14 +4,14 @@ on: [push, pull_request] jobs: test: - name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, ${{ format("sk-only:{0}", ${{ matrix.sklearn-only }} ) }} ) + name: (matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) runs-on: ${{ matrix.os }} strategy: matrix: python-version: [3.6, 3.7, 3.8] scikit-learn: [0.21.2, 0.22.2, 0.23.1, 0.24] os: [ubuntu-latest] - sklearn-only: [true] + sklearn-only: ['true'] exclude: # no scikit-learn 0.21.2 release for Python 3.8 - python-version: 3.8 scikit-learn: 0.21.2 @@ -29,10 +29,10 @@ jobs: - python-version: 3.8 scikit-learn: 0.23.1 code-cov: true - sklearn-only: false + sklearn-only: 'false' os: ubuntu-latest - os: windows-latest - sklearn-only: false + sklearn-only: 'false' scikit-learn: 0.24.* fail-fast: false max-parallel: 4 @@ -66,7 +66,7 @@ jobs: run: | if [ ${{ matrix.code-cov }} ]; then codecov='--cov=openml --long --cov-report=xml'; fi # Most of the time, running only the scikit-learn tests is sufficient - if [ ${{ matrix.sklearn-only }} ]; then sklearn='-m sklearn'; fi + if [ ${{ matrix.sklearn-only }} = 'true' ]; then sklearn='-m sklearn'; fi echo pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 pytest -n 4 --durations=20 --timeout=600 --timeout-method=thread --dist load -sv $codecov $sklearn --reruns 5 --reruns-delay 1 - name: Run tests on Windows From e549aa90f1b2aa6f01cf33c6cbf9fa49fdadfba8 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 22 Feb 2023 11:56:57 +0100 Subject: [PATCH 13/15] Add missing ${{ --- .github/workflows/test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 5ad7d9e08..0f0b13731 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -4,7 +4,7 @@ on: [push, pull_request] jobs: test: - name: (matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) + name: (${{ matrix.os }}, Py${{ matrix.python-version }}, sk${{ matrix.scikit-learn }}, sk-only:${{ matrix.sklearn-only }}) runs-on: ${{ matrix.os }} strategy: matrix: From 1db1ff7348243ee9d0011e5dadab04de884d3f32 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Wed, 22 Feb 2023 12:08:26 +0100 Subject: [PATCH 14/15] Add explicit true to old sklearn tests --- .github/workflows/test.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 0f0b13731..5adfa3eac 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,12 +20,15 @@ jobs: scikit-learn: 0.18.2 scipy: 1.2.0 os: ubuntu-latest + sklearn-only: 'true' - python-version: 3.6 scikit-learn: 0.19.2 os: ubuntu-latest + sklearn-only: 'true' - python-version: 3.6 scikit-learn: 0.20.2 os: ubuntu-latest + sklearn-only: 'true' - python-version: 3.8 scikit-learn: 0.23.1 code-cov: true From d01b4b24c422daf1e7a4df9fb1d083b8d64c4910 Mon Sep 17 00:00:00 2001 From: PGijsbers Date: Thu, 23 Feb 2023 14:09:48 +0100 Subject: [PATCH 15/15] Add instruction to add pytest marker for sklearn tests --- CONTRIBUTING.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 688dbd7a9..87c8ae3c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -153,7 +153,8 @@ following rules before you submit a pull request: - Add [unit tests](https://github.com/openml/openml-python/tree/develop/tests) and [examples](https://github.com/openml/openml-python/tree/develop/examples) for any new functionality being introduced. - If an unit test contains an upload to the test server, please ensure that it is followed by a file collection for deletion, to prevent the test server from bulking up. For example, `TestBase._mark_entity_for_removal('data', dataset.dataset_id)`, `TestBase._mark_entity_for_removal('flow', (flow.flow_id, flow.name))`. - - Please ensure that the example is run on the test server by beginning with the call to `openml.config.start_using_configuration_for_example()`. + - Please ensure that the example is run on the test server by beginning with the call to `openml.config.start_using_configuration_for_example()`. + - Add the `@pytest.mark.sklearn` marker to your unit tests if they have a dependency on scikit-learn. - All tests pass when running `pytest`. On Unix-like systems, check with (from the toplevel source folder):