diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 2ff236b913177..4c24bcc2b1d48 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1081,6 +1081,13 @@ repos: pass_filenames: false require_serial: true additional_dependencies: ['packaging>=25', 'pyyaml', 'tomli>=2.0.1', 'rich>=13.6.0'] + - id: check-dependency-lower-bounds + name: Check that dependencies in pyproject.toml have lower bounds + language: python + entry: ./scripts/ci/prek/check_dependency_lower_bounds.py + files: (^|/)pyproject\.toml$ + require_serial: true + additional_dependencies: ['packaging>=25', 'tomli>=2.0.1', 'rich>=13.6.0'] - id: update-reproducible-source-date-epoch name: Update Source Date Epoch for reproducible builds language: python diff --git a/clients/python/pyproject.toml b/clients/python/pyproject.toml index 2e246c7778eb8..0614c5f4529ac 100644 --- a/clients/python/pyproject.toml +++ b/clients/python/pyproject.toml @@ -59,7 +59,7 @@ classifiers = [ dependencies = [ "pydantic >= 2.11.0", - "python-dateutil", + "python-dateutil>=2.7.0", "urllib3>=2.1.0,!=2.6.0", ] diff --git a/contributing-docs/13_airflow_dependencies_and_extras.rst b/contributing-docs/13_airflow_dependencies_and_extras.rst index 7c59ce25d17b1..1a64d5e2ca9d8 100644 --- a/contributing-docs/13_airflow_dependencies_and_extras.rst +++ b/contributing-docs/13_airflow_dependencies_and_extras.rst @@ -239,6 +239,21 @@ rules to remember: stopped working (like in case of ``amazon``, ``fab``). You are free to modify those versions to higher versions if you need to, and ``prek`` will remove those comments automatically. +* Every dependency we resolve from PyPI must have a lower bound. Without one the resolver is free to answer + with any version that has ever been published, so what our constraints pin - and what a user ends up + installing - depends on how the resolution went rather than on what the code needs. The + ``check-dependency-lower-bounds`` prek hook enforces this across ``project.dependencies``, + ``project.optional-dependencies``, ``dependency-groups`` and ``build-system.requires`` of every + ``pyproject.toml``. Use the oldest version you are willing to test against: + + .. code-block:: python + + "pyspark>=4.0.0", + + Two kinds of requirement are exempt: distributions that are members of our ``uv`` workspace (they are + resolved from the checkout, so a version range would say nothing) and direct URL requirements (the URL + already names the exact artifact). + Our CI system will do all the tests for you anyway - including running some lower-bind checks on dependencies. For example it will take each provider in a turn and will try to resolve lowest-possible dependencies defined for that provider and see if the tests are still passing, so we should be relatively protected against putting diff --git a/providers/common/ai/docs/index.rst b/providers/common/ai/docs/index.rst index 41a01074f44c9..4b0f82608b36d 100644 --- a/providers/common/ai/docs/index.rst +++ b/providers/common/ai/docs/index.rst @@ -241,11 +241,11 @@ Install them when installing from PyPI. For example: ============== ======================================================================================================================================= Extra Dependencies ============== ======================================================================================================================================= -``anthropic`` ``pydantic-ai-slim[anthropic]`` -``bedrock`` ``pydantic-ai-slim[bedrock]`` -``google`` ``pydantic-ai-slim[google]`` -``openai`` ``pydantic-ai-slim[openai]`` -``mcp`` ``pydantic-ai-slim[mcp]`` +``anthropic`` ``pydantic-ai-slim[anthropic]>=2.0.0`` +``bedrock`` ``pydantic-ai-slim[bedrock]>=2.0.0`` +``google`` ``pydantic-ai-slim[google]>=2.0.0`` +``openai`` ``pydantic-ai-slim[openai]>=2.0.0`` +``mcp`` ``pydantic-ai-slim[mcp]>=2.0.0`` ``code-mode`` ``pydantic-ai-harness[codemode]>=0.3.0`` ``shields`` ``pydantic-ai-shields>=0.3.4`` ``skills`` ``apache-airflow-providers-git>=0.4.0``, ``pydantic-ai-skills>=1.2.0`` diff --git a/providers/common/ai/pyproject.toml b/providers/common/ai/pyproject.toml index 07a65f18dd469..aa806092833df 100644 --- a/providers/common/ai/pyproject.toml +++ b/providers/common/ai/pyproject.toml @@ -76,11 +76,11 @@ dependencies = [ # The optional dependencies should be modified in place in the generated file # Any change in the dependencies is preserved when the file is regenerated [project.optional-dependencies] -"anthropic" = ["pydantic-ai-slim[anthropic]"] -"bedrock" = ["pydantic-ai-slim[bedrock]"] -"google" = ["pydantic-ai-slim[google]"] -"openai" = ["pydantic-ai-slim[openai]"] -"mcp" = ["pydantic-ai-slim[mcp]"] +"anthropic" = ["pydantic-ai-slim[anthropic]>=2.0.0"] +"bedrock" = ["pydantic-ai-slim[bedrock]>=2.0.0"] +"google" = ["pydantic-ai-slim[google]>=2.0.0"] +"openai" = ["pydantic-ai-slim[openai]>=2.0.0"] +"mcp" = ["pydantic-ai-slim[mcp]>=2.0.0"] # Code mode: collapse tool calls into a single `run_code` tool that the model # drives by writing Python, executed in the Monty sandbox (pydantic-monty). # Enables AgentOperator(code_mode=True). Monty is pre-1.0; pinned here as an diff --git a/scripts/ci/prek/check_dependency_lower_bounds.py b/scripts/ci/prek/check_dependency_lower_bounds.py new file mode 100755 index 0000000000000..4ec5863d2788b --- /dev/null +++ b/scripts/ci/prek/check_dependency_lower_bounds.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# /// script +# requires-python = ">=3.10,<3.11" +# dependencies = [ +# "packaging>=25", +# "rich>=13.6.0", +# "tomli>=2.0.1", +# ] +# /// +""" +Validate that every external dependency declared in a ``pyproject.toml`` has a lower bound. + +An unbounded requirement lets the resolver answer with any version that happens to be on +PyPI, so what a constraints file pins - and what a user ends up installing - depends on how +the resolution went rather than on what the code needs. Naming the oldest supported version +makes that answer deterministic and documents the floor the code is tested against. + +Checked locations: ``project.dependencies``, ``project.optional-dependencies``, +``dependency-groups`` and ``build-system.requires``. + +Requirements resolved from the uv workspace rather than from PyPI are exempt - their source +is the checkout, so a version range would say nothing. Direct URL requirements are exempt +too, since the URL already names the exact artifact. +""" + +from __future__ import annotations + +import sys +from functools import cache +from pathlib import Path + +from common_prek_utils import AIRFLOW_ROOT_PATH, console +from packaging.requirements import InvalidRequirement, Requirement +from packaging.utils import canonicalize_name +from rich.markup import escape + +try: + import tomllib +except ImportError: + import tomli as tomllib # type: ignore[no-redef] + +# Operators that place a floor under the resolved version; anything else (``!=``, ``<``, +# ``<=``) leaves the resolver free to reach back to the oldest release ever published. +LOWER_BOUND_OPERATORS = {">=", ">", "==", "===", "~="} + + +def _load_toml(path: Path) -> dict: + return tomllib.loads(path.read_text()) + + +@cache +def get_workspace_distribution_names() -> frozenset[str]: + """Return the canonical names of the distributions uv resolves from this checkout.""" + root_pyproject = AIRFLOW_ROOT_PATH / "pyproject.toml" + root_data = _load_toml(root_pyproject) + members = root_data.get("tool", {}).get("uv", {}).get("workspace", {}).get("members", []) + names = set() + for member in members: + for member_path in sorted(AIRFLOW_ROOT_PATH.glob(f"{member}/pyproject.toml")): + if name := _load_toml(member_path).get("project", {}).get("name"): + names.add(canonicalize_name(name)) + return frozenset(names) + + +def extract_requirements(data: dict) -> list[tuple[str, str]]: + """Return ``(section, requirement)`` pairs for every dependency table we guard.""" + requirements: list[tuple[str, str]] = [] + project = data.get("project") or {} + for dependency in project.get("dependencies") or []: + requirements.append(("project.dependencies", dependency)) + for extra, dependencies in (project.get("optional-dependencies") or {}).items(): + for dependency in dependencies: + requirements.append((f'project.optional-dependencies."{extra}"', dependency)) + for group, dependencies in (data.get("dependency-groups") or {}).items(): + for dependency in dependencies: + # A group may also pull in another group via ``{include-group = "..."}``. + if isinstance(dependency, str): + requirements.append((f'dependency-groups."{group}"', dependency)) + for dependency in (data.get("build-system") or {}).get("requires") or []: + requirements.append(("build-system.requires", dependency)) + return requirements + + +def check_requirement(section: str, dependency: str, workspace_names: frozenset[str]) -> str | None: + """Return an error message when ``dependency`` needs a lower bound, otherwise ``None``.""" + try: + requirement = Requirement(dependency) + except InvalidRequirement as error: + return f"[{section}] {dependency!r} is not a valid requirement: {error}" + if ( + requirement.url + or canonicalize_name(requirement.name) in workspace_names + or any(specifier.operator in LOWER_BOUND_OPERATORS for specifier in requirement.specifier) + ): + return None + return f"[{section}] {dependency!r} has no lower bound - add one, for example {requirement.name}>=X.Y.Z" + + +def check_pyproject_file(path: Path, workspace_names: frozenset[str]) -> list[str]: + return [ + error + for section, dependency in extract_requirements(_load_toml(path)) + if (error := check_requirement(section, dependency, workspace_names)) + ] + + +def main() -> int: + workspace_names = get_workspace_distribution_names() + failed = False + for file in sys.argv[1:]: + path = Path(file) + if errors := check_pyproject_file(path, workspace_names): + failed = True + console.print(f"\n[red]Missing lower bounds in {file}:[/]\n") + for error in errors: + console.print(f" {escape(error)}") + if failed: + console.print( + "\n[bright_yellow]Every dependency resolved from PyPI needs a lower bound.[/]\n" + "Without one the resolver may pick any published version, so constraints pin\n" + "whatever the resolution happened to produce rather than the oldest version the\n" + "code supports. Use the oldest version you are willing to test against.\n" + ) + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/tests/ci/prek/test_check_dependency_lower_bounds.py b/scripts/tests/ci/prek/test_check_dependency_lower_bounds.py new file mode 100644 index 0000000000000..4733e3b3dbc49 --- /dev/null +++ b/scripts/tests/ci/prek/test_check_dependency_lower_bounds.py @@ -0,0 +1,143 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import textwrap + +import pytest +from check_dependency_lower_bounds import check_pyproject_file, check_requirement, extract_requirements + +WORKSPACE_NAMES = frozenset({"apache-airflow-core", "apache-airflow-devel-common"}) + + +class TestCheckRequirement: + @pytest.mark.parametrize( + "dependency", + [ + pytest.param("pyspark>=4.0.0", id="greater-or-equal"), + pytest.param("pyspark>4.0.0", id="greater"), + pytest.param("hatchling==1.31.0", id="pinned"), + pytest.param("hatchling===1.31.0", id="arbitrary-equality"), + pytest.param("hatchling~=1.31", id="compatible-release"), + pytest.param("urllib3>=2.1.0,!=2.6.0", id="lower-bound-with-exclusion"), + pytest.param("pydantic-ai-slim[mcp]>=2.0.0", id="extras"), + pytest.param('fastavro>=1.10.0; python_version < "3.14"', id="marker"), + ], + ) + def test_no_error_when_lower_bound_present(self, dependency): + assert check_requirement("project.dependencies", dependency, WORKSPACE_NAMES) is None + + @pytest.mark.parametrize( + "dependency", + [ + pytest.param("pyspark", id="bare"), + pytest.param("pydantic-ai-slim[mcp]", id="extras"), + pytest.param("pyspark<5.0.0", id="upper-bound-only"), + pytest.param("pyspark!=4.1.0", id="exclusion-only"), + pytest.param('krb5; python_version < "3.14"', id="marker-only"), + ], + ) + def test_error_when_lower_bound_missing(self, dependency): + error = check_requirement("project.dependencies", dependency, WORKSPACE_NAMES) + assert error is not None + assert "has no lower bound" in error + assert "[project.dependencies]" in error + + @pytest.mark.parametrize( + "dependency", + [ + pytest.param("apache-airflow-core", id="plain"), + pytest.param("apache_airflow_core", id="non-canonical-name"), + pytest.param("apache-airflow-devel-common[mypy]", id="extras"), + ], + ) + def test_no_error_for_workspace_distribution(self, dependency): + assert check_requirement("dependency-groups.dev", dependency, WORKSPACE_NAMES) is None + + def test_no_error_for_direct_url_requirement(self): + dependency = ( + "sphinx-airflow-theme@https://airflow.apache.org/sphinx-airflow-theme/" + "sphinx_airflow_theme-0.3.13-py3-none-any.whl" + ) + assert check_requirement("project.optional-dependencies.docs", dependency, WORKSPACE_NAMES) is None + + def test_error_for_invalid_requirement(self): + error = check_requirement("project.dependencies", "not a requirement!", WORKSPACE_NAMES) + assert error is not None + assert "is not a valid requirement" in error + + +class TestExtractRequirements: + def test_extracts_every_guarded_table(self): + data = { + "build-system": {"requires": ["hatchling"]}, + "project": { + "dependencies": ["pyspark"], + "optional-dependencies": {"kerberos": ["krb5"]}, + }, + "dependency-groups": {"dev": ["pytest", {"include-group": "docs"}]}, + } + assert extract_requirements(data) == [ + ("project.dependencies", "pyspark"), + ('project.optional-dependencies."kerberos"', "krb5"), + ('dependency-groups."dev"', "pytest"), + ("build-system.requires", "hatchling"), + ] + + def test_no_requirements_when_tables_absent(self): + assert extract_requirements({"tool": {"uv": {"required-version": ">=0.9.0"}}}) == [] + + +class TestCheckPyprojectFile: + def _write(self, tmp_path, content): + path = tmp_path / "pyproject.toml" + path.write_text(textwrap.dedent(content)) + return path + + def test_reports_every_unbounded_dependency(self, tmp_path): + path = self._write( + tmp_path, + """ + [project] + name = "apache-airflow-providers-samba" + dependencies = ["smbprotocol>=1.5.0"] + + [project.optional-dependencies] + "kerberos" = ["krb5", "apache-airflow-core"] + + [dependency-groups] + dev = ["pytest"] + """, + ) + errors = check_pyproject_file(path, WORKSPACE_NAMES) + assert len(errors) == 2 + assert "krb5" in errors[0] + assert "pytest" in errors[1] + + def test_no_errors_when_all_bounded(self, tmp_path): + path = self._write( + tmp_path, + """ + [project] + name = "apache-airflow-providers-samba" + dependencies = ["smbprotocol>=1.5.0"] + + [dependency-groups] + dev = ["pytest>=9.1.1"] + """, + ) + assert check_pyproject_file(path, WORKSPACE_NAMES) == [] diff --git a/uv.lock b/uv.lock index 76b7408a20fae..940a928dcab5f 100644 --- a/uv.lock +++ b/uv.lock @@ -2958,7 +2958,7 @@ docs = [ [package.metadata] requires-dist = [ { name = "akeyless", specifier = ">=5.0.0" }, - { name = "akeyless-cloud-id", marker = "extra == 'cloud-id'" }, + { name = "akeyless-cloud-id", marker = "extra == 'cloud-id'", specifier = ">=0.3.0" }, { name = "apache-airflow", editable = "." }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, ] @@ -3968,7 +3968,7 @@ dev = [ { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-openlineage", editable = "providers/openlineage" }, { name = "apache-airflow-task-sdk", editable = "task-sdk" }, - { name = "pyspark" }, + { name = "pyspark", specifier = ">=4.0.0" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] @@ -4484,11 +4484,11 @@ requires-dist = [ { name = "pydantic-ai-shields", marker = "extra == 'shields'", specifier = ">=0.3.4" }, { name = "pydantic-ai-skills", marker = "extra == 'skills'", specifier = ">=1.2.0" }, { name = "pydantic-ai-slim", specifier = ">=2.0.0" }, - { name = "pydantic-ai-slim", extras = ["anthropic"], marker = "extra == 'anthropic'" }, - { name = "pydantic-ai-slim", extras = ["bedrock"], marker = "extra == 'bedrock'" }, - { name = "pydantic-ai-slim", extras = ["google"], marker = "extra == 'google'" }, - { name = "pydantic-ai-slim", extras = ["mcp"], marker = "extra == 'mcp'" }, - { name = "pydantic-ai-slim", extras = ["openai"], marker = "extra == 'openai'" }, + { name = "pydantic-ai-slim", extras = ["anthropic"], marker = "extra == 'anthropic'", specifier = ">=2.0.0" }, + { name = "pydantic-ai-slim", extras = ["bedrock"], marker = "extra == 'bedrock'", specifier = ">=2.0.0" }, + { name = "pydantic-ai-slim", extras = ["google"], marker = "extra == 'google'", specifier = ">=2.0.0" }, + { name = "pydantic-ai-slim", extras = ["mcp"], marker = "extra == 'mcp'", specifier = ">=2.0.0" }, + { name = "pydantic-ai-slim", extras = ["openai"], marker = "extra == 'openai'", specifier = ">=2.0.0" }, { name = "pypdf", marker = "extra == 'pdf'", specifier = ">=4.0.0" }, { name = "python-docx", marker = "extra == 'docx'", specifier = ">=1.0.0" }, { name = "sqlglot", marker = "extra == 'sql'", specifier = ">=30.0.0" }, @@ -4510,7 +4510,7 @@ dev = [ { name = "llama-index-embeddings-openai", specifier = ">=0.6.0" }, { name = "llama-index-llms-openai", specifier = ">=0.6.0" }, { name = "pydantic-ai-skills", specifier = ">=1.2.0" }, - { name = "pydantic-ai-slim", extras = ["mcp"] }, + { name = "pydantic-ai-slim", extras = ["mcp"], specifier = ">=2.0.0" }, { name = "sqlglot", specifier = ">=30.0.0" }, ] docs = [{ name = "apache-airflow-devel-common", extras = ["docs"], editable = "devel-common" }] @@ -7513,7 +7513,7 @@ requires-dist = [ { name = "apache-airflow", editable = "." }, { name = "apache-airflow-providers-common-compat", editable = "providers/common/compat" }, { name = "apache-airflow-providers-google", marker = "extra == 'google'", editable = "providers/google" }, - { name = "krb5", marker = "extra == 'kerberos'" }, + { name = "krb5", marker = "extra == 'kerberos'", specifier = ">=0.8.0" }, { name = "smbprotocol", specifier = ">=1.5.0" }, { name = "smbprotocol", extras = ["kerberos"], marker = "extra == 'kerberos'", specifier = ">=1.5.0" }, ] @@ -8474,7 +8474,7 @@ requires-dist = [ ] [package.metadata.requires-dev] -dev = [{ name = "pytest" }] +dev = [{ name = "pytest", specifier = ">=9.1.1" }] [[package]] name = "apache-airflow-scripts"