diff --git a/.github/workflows/code-health.yml b/.github/workflows/code-health.yml new file mode 100644 index 000000000..a3eb89cd5 --- /dev/null +++ b/.github/workflows/code-health.yml @@ -0,0 +1,118 @@ +# Periodic structural sweep: duplication, coupling, cohesion, dead code, and the +# maintainability trend. +# +# Advisory by design, and nothing here fails the build. These measures move over +# months rather than commits, and they report findings a reviewer has to judge +# rather than obey -- a metric can call good design a violation. The merge gates +# live in python-package.yml. See CONTRIBUTING.md for how to read the output. + +name: Code Health + +on: + schedule: + # 08:00 UTC Mondays: a week's worth of merges, read before the week starts. + - cron: "0 8 * * 1" + # On demand, for checking whether a refactor actually moved the numbers. + workflow_dispatch: + +concurrency: + group: ${{ github.workflow }} + cancel-in-progress: false + +permissions: + contents: read + +jobs: + health: + runs-on: ubuntu-latest + timeout-minutes: 20 + steps: + - uses: actions/checkout@v6 + with: + # wily indexes git history; a shallow clone gives it one revision to + # trend against, which is no trend at all. + fetch-depth: 0 + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + cache: "pip" + - name: Install analysis tooling + run: | + python -m pip install --upgrade pip + pip install -e .[metrics,health] + + - name: Structural analysis (pyscn) + # One output format per invocation, so run it twice: JSON to read the + # numbers, HTML to read the report. Never fails the job -- a non-zero + # exit here means "findings", which are the point. + continue-on-error: true + run: | + pyscn analyze --json --no-open dataretrieval | tee pyscn-summary.txt + pyscn analyze --html --no-open dataretrieval >/dev/null + + - name: Check the analysis actually resolved the package + # pyscn infers a project root, and when it guesses wrong it silently + # resolves only a fraction of the imports -- which *raises* the score, + # because most of what it grades is dependency-derived. A degraded run + # therefore looks like an improved one. Record the resolved edge count + # next to the score so that is visible rather than flattering. + if: always() + continue-on-error: true + run: | + python - <<'PY' > pyscn-sanity.txt + import glob, json, os + reports = sorted(glob.glob(".pyscn/reports/*.json")) + if not reports: + print("no pyscn JSON report found"); raise SystemExit + s = json.load(open(reports[-1]))["system"]["Summary"] + root, deps = s["ProjectRoot"], s["TotalDependencies"] + print(f"modules={s['TotalModules']} resolved_dependencies={deps} root={root}") + if os.path.realpath(root) != os.path.realpath(os.getcwd()): + print(f"WARNING: project root {root!r} is not the checkout; " + "import resolution is probably degraded and the scores " + "above are not comparable to previous runs.") + PY + cat pyscn-sanity.txt + + - name: Maintainability ranking (wily) + # Worst-maintained files today, and how the package has moved recently. + run: | + wily build dataretrieval --max-revisions 50 >/dev/null 2>&1 || true + { + echo '## Maintainability ranking' + echo '```' + wily rank dataretrieval maintainability.mi 2>&1 | head -25 \ + || echo 'wily index unavailable' + echo '```' + } > wily-summary.txt + + - name: Publish report + # ``always()`` so a partial report still lands rather than a bare red X. + if: always() + run: | + { + echo '## Structural analysis' + echo '```' + cat pyscn-summary.txt 2>/dev/null || echo 'pyscn produced no output' + cat pyscn-sanity.txt 2>/dev/null || true + echo '```' + cat wily-summary.txt 2>/dev/null || true + echo + echo 'Full reports are attached to this run as the' + echo '`code-health-report` artifact. Findings are advisory --' + echo 'read them as leads, not verdicts (see CONTRIBUTING.md).' + echo 'The merge gates live in the Python package workflow.' + } >> "$GITHUB_STEP_SUMMARY" + + - name: Upload reports + if: always() + uses: actions/upload-artifact@v4 + with: + name: code-health-report + path: | + .pyscn/reports/ + pyscn-summary.txt + wily-summary.txt + retention-days: 90 + if-no-files-found: warn diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index 6230850f4..c4e9b7d0f 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -41,15 +41,18 @@ jobs: python-version: "3.14" cache: "pip" - name: Install metrics tooling - # Versions pinned in the [metrics] extra so CI and the local - # pre-commit hook grade identically. + # Versions pinned in the extra so CI and pre-commit grade identically. run: pip install -e .[metrics] + # Ratchets, not aspirations: thresholds sit where the package already is, + # so they fail on regression rather than demanding a refactor. Each + # mirrors a pre-commit hook, so a clean local run means CI agrees. - name: Complexity gate - # Ratchet, not an aspiration: these are the tightest thresholds the - # package currently passes. A change that regresses complexity fails - # here with the offending block named. Mirrors the xenon pre-commit - # hook, so a contributor sees the same verdict before pushing. run: xenon --max-absolute C --max-modules B --max-average A dataretrieval + - name: Cognitive complexity gate + run: complexipy --max-complexity-allowed 27 --failed dataretrieval + - name: Dependency-direction contracts + # Rules and rationale live in .importlinter and the ADRs it cites. + run: lint-imports - name: Complexity trend vs base # Advisory: reports which files moved and by how much, so a reviewer # can see direction rather than a pass/fail. Never fails the build -- diff --git a/.gitignore b/.gitignore index 0b08fadaf..3dfbb55fa 100644 --- a/.gitignore +++ b/.gitignore @@ -115,3 +115,6 @@ ENV/ # wily metrics cache (rebuildable: `wily build dataretrieval`) .wily/ + +# pyscn analysis reports (rebuildable: `pyscn analyze dataretrieval`) +.pyscn/ diff --git a/.importlinter b/.importlinter new file mode 100644 index 000000000..0fb1cfb20 --- /dev/null +++ b/.importlinter @@ -0,0 +1,132 @@ +; Declarative dependency-direction contracts for ``dataretrieval``. +; +; This file owns every rule that is purely a statement about the import graph. +; ``tests/architecture_test.py`` owns the rules that are not: symbol-level +; claims, ``__all__`` surfaces, AST shape, and the one boundary that must be +; asserted positively. The split is deliberate and the two do not overlap -- +; a rule enforced twice is a rule that gets updated once. +; +; Run with ``lint-imports`` (installed by the ``[metrics]`` extra). + +[importlinter] +root_package = dataretrieval +; Contracts describe what runs, matching the AST suite. ``ogc.interruptions`` +; and ``ogc.chunking`` reference each other's types under ``TYPE_CHECKING``; +; that is a documentation edge, not a runtime one, and no cycle exists at +; import time. +exclude_type_checking_imports = True + +[importlinter:contract:layers] +name = Dependencies point toward stable policy (ADR 0003, ADR 0006) +type = layers +containers = + dataretrieval +layers = + ngwmn | nldi | nwis | streamstats | waterdata | wateruse | wqp + ogc + utils + transport + progress + codes | combining | rdb | response_metadata + credentials + exceptions +; Every top-level module must be placed in the stack deliberately. A new +; top-level module fails this contract until someone decides where it sits. +exhaustive = True +exhaustive_ignores = + _version + +[importlinter:contract:acyclic] +name = The runtime import graph is acyclic (ADR 0001) +type = acyclic_siblings +; Applied to the whole package, not just the two subsystems whose acyclicity +; was previously asserted by hand. The claim held package-wide already; stating +; it once at the root covers ``ogc`` and ``transport`` and every future +; subpackage without a new rule per package. +ancestors = + dataretrieval + +[importlinter:contract:ogc-consumers] +name = Only NGWMN and Water Data consume the OGC subsystem (ADR 0003) +type = protected +; Read as: nothing outside this list may reach into OGC, directly or through a +; submodule. An allowlist rather than a denylist, so a new service module is +; refused by default instead of being silently permitted until someone +; remembers to add it. +; +; The root ``dataretrieval`` package is deliberately NOT an allowed importer. +; ``allowed_importers`` is matched with ``as_packages``, so naming the root here +; would make every module in the distribution an allowed importer and the +; contract could never fail. Its two real imports are listed as explicit +; exceptions below instead -- narrow, visible, and they fail if they go stale. +protected_modules = + dataretrieval.ogc +allowed_importers = + dataretrieval.ngwmn + dataretrieval.waterdata +ignore_imports = +; The package __init__ re-exports the resumable-call and interruption types; +; they are part of the documented public surface, not a service reaching in. + dataretrieval -> dataretrieval.ogc.chunking + dataretrieval -> dataretrieval.ogc.interruptions + +[importlinter:contract:ogc-facade] +name = NGWMN consumes the OGC facade only, never its internals (ADR 0007) +type = forbidden +source_modules = + dataretrieval.ngwmn +; The wildcard is what makes this durable: a new ``ogc`` submodule is covered +; the day it is added, without editing this contract. +forbidden_modules = + dataretrieval.ogc.** +; Direct imports only. Reaching an internal *through* the facade is the design, +; not a violation -- ``ngwmn -> ogc -> ogc.engine`` is how the seam is supposed +; to work, and the default (indirect included) forbids exactly that. +allow_indirect_imports = True + +[importlinter:contract:ogc-internal-seams] +name = OGC internal seams (ADR 0003) +type = forbidden +source_modules = +; Feature shaping is downstream of execution. If shaping imports engine the +; subsystem gains a cycle and the schema fetch on an empty frame becomes +; reachable from request construction. + dataretrieval.ogc.shaping +forbidden_modules = + dataretrieval.ogc.engine + +[importlinter:contract:nwis-quarantine] +name = Deprecated NWIS has no dependents (ADR 0005) +type = forbidden +source_modules = + dataretrieval.codes + dataretrieval.combining + dataretrieval.credentials + dataretrieval.exceptions + dataretrieval.ngwmn + dataretrieval.nldi + dataretrieval.ogc + dataretrieval.progress + dataretrieval.rdb + dataretrieval.streamstats + dataretrieval.transport + dataretrieval.utils + dataretrieval.waterdata + dataretrieval.wateruse + dataretrieval.wqp +forbidden_modules = + dataretrieval.nwis + +[importlinter:contract:waterdata-families] +name = Water Data collection families do not reach through each other (ADR 0007) +type = independence +; Kept in step with ``_WATERDATA_FAMILIES`` in tests/architecture_test.py, which +; derives the facade's expected export union from the same six modules. A +; seventh family fails that test until it is listed there; add it here too. +modules = + dataretrieval.waterdata.cql + dataretrieval.waterdata.measurements + dataretrieval.waterdata.metadata + dataretrieval.waterdata.reference + dataretrieval.waterdata.samples + dataretrieval.waterdata.time_series diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9305a68d3..a4bca6fae 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -63,6 +63,40 @@ repos: files: ^dataretrieval/.*\.py$ pass_filenames: false + # Cognitive-complexity ratchet, the counterpart to xenon's cyclomatic one. + # The two measure different things: xenon counts branches, complexipy counts + # how hard the control flow is to hold in your head, so nesting and early + # exits weigh differently. 27 is the package's current maximum + # (``nldi.get_features``), so this holds the line rather than demanding a + # refactor. Unlike xenon this one is per-function, so scanning only the + # changed files gives the same verdict CI gives for the whole package. + - repo: https://github.com/rohaquinlop/complexipy-pre-commit + rev: v6.2.0 + hooks: + - id: complexipy + args: ["--max-complexity-allowed", "27", "--failed"] + files: ^dataretrieval/.*\.py$ + + # Dependency-direction contracts (ADR 0003, 0005, 0006, 0007). Complements + # tests/architecture_test.py, which enforces named pairwise claims by parsing + # the AST; this checks the whole layer stack over the *transitive* graph, so + # a service reaching transport through OGC fails here. Runs in about a + # second. Declared as a local hook rather than using the upstream one, which + # is ``language: system`` and so needs lint-imports already on PATH. + - repo: local + hooks: + - id: import-linter + name: import-linter + description: Enforce the dependency layers declared in .importlinter + entry: lint-imports + language: python + additional_dependencies: ["import-linter==2.13"] + # Reads the package tree from the repo root, not a file list; grimp + # resolves ``dataretrieval`` relative to the working directory, so the + # hook needs neither the package nor its runtime deps installed. + pass_filenames: false + files: ^(dataretrieval/.*\.py|\.importlinter)$ + # Strip cell outputs + execution_count from notebooks on commit so the # diff is the source, not the rendered run. Demos still execute fine # locally; clean commits keep PRs reviewable and avoid quota/timestamp diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3cda1ab22..efb676b56 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -123,14 +123,36 @@ mypy coverage run -m pytest tests/ coverage report -m xenon --max-absolute C --max-modules B --max-average A dataretrieval +complexipy --max-complexity-allowed 27 --failed dataretrieval +lint-imports ``` -The last one is a complexity ratchet: those thresholds are the tightest the -package passes today, so it fails only when a change makes complexity worse. It -names the offending block, so the fix is local -- usually extracting a branch -rather than restructuring. Install it with `pip install -e .[metrics]`; the -`xenon` pre-commit hook runs the identical check, so a clean pre-commit run means -CI agrees. +The last three come from `pip install -e .[metrics]`, and each has a pre-commit +hook running the identical check, so a clean pre-commit run means CI agrees. + +`xenon` and `complexipy` are complexity ratchets: the thresholds are the +tightest the package passes today, so they fail only when a change makes things +worse. They disagree usefully. `xenon` counts branches (cyclomatic complexity), +so a wide flat dispatch scores badly; `complexipy` counts how hard the control +flow is to follow (cognitive complexity), so it forgives the dispatch and +punishes nesting. Both name the offending block, so the fix is local -- usually +extracting a branch rather than restructuring. + +`lint-imports` checks the dependency contracts declared in +[`.importlinter`](.importlinter) against the *transitive* import graph: the +layer stack, which modules may consume OGC, NGWMN's facade-only seam, the NWIS +quarantine, collection-family independence, and package-wide acyclicity. + +**That file is the only place dependency direction is enforced.** These rules +were once asserted a second time in `tests/architecture_test.py` by hand-parsing +the AST; that duplication is gone, and re-adding it would mean one rule with two +homes that drift apart. What the tests still own is everything an import graph +cannot see -- which *symbols* cross a seam, declared `__all__` surfaces, the AST +shape of a facade, and the one boundary that must be asserted positively +(`lint-imports` can forbid an edge, never require one). If you are adding a rule +and it is purely "module A must not import module B", it belongs in +`.importlinter`. A boundary that legitimately moves is one edit there, plus the +ADR it cites. To see the *trend* rather than a pass/fail, that extra also installs [`wily`](https://github.com/tonybaloney/wily), which indexes metrics across git @@ -146,6 +168,38 @@ wily rank dataretrieval maintainability.mi # worst-maintained files today `wily` is advisory and is never a merge gate -- rising complexity in a file that gained a genuinely complex feature is information, not a failure. +#### The periodic deep sweep + +Duplication, coupling, cohesion, dependency depth, and dead code are tracked by +[`pyscn`](https://github.com/ludo-technologies/pyscn) on a weekly schedule +([code-health.yml](https://github.com/DOI-USGS/dataretrieval-python/blob/main/.github/workflows/code-health.yml)), +which attaches an HTML and a JSON report to each run. Nothing gates on it. These +measures move over months rather than commits, and a threshold nobody agreed to +is either noise or theatre. + +You do not need it to contribute, but it is the right tool for "what should we +clean up next?" -- including for an agent working on this repo, which gets a +whole-package structural picture from one command: + +```bash +pip install -e .[health] # its own extra: pyscn is a compiled binary with no + # linux/aarch64 wheel, so it is kept out of the + # extra the merge gates depend on +pyscn analyze dataretrieval # HTML report, or --json for the numbers +``` + +Read its findings as leads, not verdicts. Its clone detector flags this +package's per-collection getters -- thin, heavily documented wrappers whose +bodies necessarily rhyme -- and collapsing them into one parameterized function +would trade the documented public surface for a metric. Its +dependency-injection heuristics expect a class-oriented design this package +deliberately does not have. + +The same extra installs `pyscn-mcp`, a stdio MCP server exposing those analyses +as tools (`analyze_code`, `detect_clones`, `find_dead_code`, +`get_health_score`, and others). Registering it with an MCP-capable assistant is +a personal workflow choice, so this repository does not configure one. + For documentation changes, install `.[doc,nldi]` and run `make html` from `docs/`. The broader `make docs` target also runs doctests and network-dependent link checking. diff --git a/dataretrieval/ngwmn.py b/dataretrieval/ngwmn.py index 2b4ead160..73ed1f0ba 100644 --- a/dataretrieval/ngwmn.py +++ b/dataretrieval/ngwmn.py @@ -25,7 +25,7 @@ from dataretrieval.codes.states import apply_state from dataretrieval.credentials import WATERDATA_BASE_URL from dataretrieval.ogc import OgcDialect, get_ogc_data, prepare_request_args -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata __all__ = [ "get_sites", diff --git a/dataretrieval/nwis.py b/dataretrieval/nwis.py index 479743aa1..ce882fd73 100644 --- a/dataretrieval/nwis.py +++ b/dataretrieval/nwis.py @@ -17,7 +17,7 @@ import pandas as pd from dataretrieval.rdb import read_rdb -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from .utils import query diff --git a/dataretrieval/ogc/engine.py b/dataretrieval/ogc/engine.py index d438bf18a..5921f7eda 100644 --- a/dataretrieval/ogc/engine.py +++ b/dataretrieval/ogc/engine.py @@ -66,10 +66,10 @@ prepare_request_args, ) from dataretrieval.ogc.shaping import GEOPANDAS, _finalize_ogc, _get_resp_data +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.transport.pagination import paginate from dataretrieval.transport.sync import run_sync from dataretrieval.utils import ( - BaseMetadata, _default_headers, # noqa: F401 — compatibility re-export for tests _require_positive_int, ) diff --git a/dataretrieval/ogc/filters.py b/dataretrieval/ogc/filters.py index fdd08615b..508754097 100644 --- a/dataretrieval/ogc/filters.py +++ b/dataretrieval/ogc/filters.py @@ -63,6 +63,33 @@ def _quote_cql_str(value: str) -> str: return value.replace("'", "''") +def _skip_space(expr: str, i: int) -> int: + """Index of the first non-space character at or after ``i``.""" + while i < len(expr) and expr[i].isspace(): + i += 1 + return i + + +def _resume_after_or(expr: str, i: int) -> int | None: + """Where the clause after a top-level ``OR`` begins, if one starts at ``i``. + + ``i`` is the index of a space that may open a ``OR`` + separator. Returns the index of the next clause's first character, or + ``None`` when this space does not begin one -- so the caller's test is + "is this a separator?" rather than four nested boundary checks. + + The trailing space is required: without it ``A ORDER BY b`` would split on + the ``OR`` inside ``ORDER``. + """ + word_start = _skip_space(expr, i) + if expr[word_start : word_start + 2].lower() != "or": + return None + after_word = word_start + 2 + if after_word >= len(expr) or not expr[after_word].isspace(): + return None + return _skip_space(expr, after_word) + + def _split_top_level_or(expr: str) -> list[str]: """Split ``expr`` at each top-level ``OR``, respecting quotes and parens. @@ -81,34 +108,18 @@ def _split_top_level_or(expr: str) -> list[str]: if in_quote is not None: if ch == in_quote: in_quote = None - i += 1 - continue - if ch in ("'", '"'): + elif ch in ("'", '"'): in_quote = ch - i += 1 - continue - if ch == "(": + elif ch == "(": depth += 1 - i += 1 - continue - if ch == ")": + elif ch == ")": depth -= 1 - i += 1 - continue - if depth == 0 and ch.isspace(): - j = i + 1 - while j < n and expr[j].isspace(): - j += 1 - if j + 2 <= n and expr[j : j + 2].lower() == "or": - k = j + 2 - if k < n and expr[k].isspace(): - m = k + 1 - while m < n and expr[m].isspace(): - m += 1 - parts.append(expr[last:i].strip()) - last = m - i = m - continue + elif depth == 0 and ch.isspace(): + resume = _resume_after_or(expr, i + 1) + if resume is not None: + parts.append(expr[last:i].strip()) + last = i = resume + continue i += 1 parts.append(expr[last:].strip()) return [p for p in parts if p] diff --git a/dataretrieval/ogc/shaping.py b/dataretrieval/ogc/shaping.py index 16899c8f3..69229cc06 100644 --- a/dataretrieval/ogc/shaping.py +++ b/dataretrieval/ogc/shaping.py @@ -18,7 +18,7 @@ import pandas as pd from dataretrieval.ogc.policy import DEFAULT_DIALECT, OgcDialect -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata try: import geopandas as gpd diff --git a/dataretrieval/response_metadata.py b/dataretrieval/response_metadata.py new file mode 100644 index 000000000..134019dc4 --- /dev/null +++ b/dataretrieval/response_metadata.py @@ -0,0 +1,68 @@ +"""The metadata object every getter returns alongside its DataFrame. + +A dependency-free leaf on purpose. This class is the second half of the +``(DataFrame, metadata)`` return contract, so nearly every service module needs +it -- and while it lived in :mod:`dataretrieval.utils` beside the legacy query +machinery, needing it meant inheriting that module's whole HTTP stack +(transport, credentials, error policy) transitively. Here it costs its +consumers nothing but ``httpx``. + +``dataretrieval.utils.BaseMetadata`` remains a working import. +""" + +from __future__ import annotations + +from typing import Any + +import httpx + +__all__ = ["BaseMetadata"] + + +class BaseMetadata: + """Base class for metadata. + + Attributes + ---------- + url : str + Response url. + query_time: datetime.timedelta + Response elapsed time. + header: httpx.Headers + Response headers. + + """ + + def __init__(self, response: httpx.Response) -> None: + """Generate a standard set of metadata informed by the response. + + Parameters + ---------- + response: ``httpx.Response`` + Response object from the ``httpx`` module. + + """ + + # Coerce httpx.URL -> str: BaseMetadata.url has always been str. + self.url = str(response.url) + self.query_time = response.elapsed + self.header = response.headers + self.comment: str | None = None + + # # not sure what statistic_info is + # self.statistic_info = None + + # # disclaimer seems to be only part of importWaterML1 + # self.disclaimer = None + + # ``site_info`` is set by ``nwis`` / ``wqp``-specific metadata classes; the + # modern ``waterdata`` metadata leaves it unimplemented (use + # ``waterdata.get_monitoring_locations`` to retrieve site descriptions). + @property + def site_info(self) -> Any: + raise NotImplementedError( + "site_info must be implemented by BaseMetadata children" + ) + + def __repr__(self) -> str: + return f"{type(self).__name__}(url={self.url})" diff --git a/dataretrieval/utils.py b/dataretrieval/utils.py index 66f46d492..c33b3fa59 100644 --- a/dataretrieval/utils.py +++ b/dataretrieval/utils.py @@ -20,6 +20,9 @@ URLTooLong, error_for_status, ) +from dataretrieval.response_metadata import ( + BaseMetadata, # noqa: F401 — compatibility re-export; defined there now +) from dataretrieval.transport.retry import ( _GATEWAY_STATUSES, RetryPolicy, @@ -290,55 +293,6 @@ def _attach_datetime_columns(df: pd.DataFrame) -> pd.DataFrame: return df -class BaseMetadata: - """Base class for metadata. - - Attributes - ---------- - url : str - Response url. - query_time: datetime.timedelta - Response elapsed time. - header: httpx.Headers - Response headers. - - """ - - def __init__(self, response: httpx.Response) -> None: - """Generate a standard set of metadata informed by the response. - - Parameters - ---------- - response: ``httpx.Response`` - Response object from the ``httpx`` module. - - """ - - # Coerce httpx.URL -> str: BaseMetadata.url has always been str. - self.url = str(response.url) - self.query_time = response.elapsed - self.header = response.headers - self.comment: str | None = None - - # # not sure what statistic_info is - # self.statistic_info = None - - # # disclaimer seems to be only part of importWaterML1 - # self.disclaimer = None - - # ``site_info`` is set by ``nwis`` / ``wqp``-specific metadata classes; the - # modern ``waterdata`` metadata leaves it unimplemented (use - # ``waterdata.get_monitoring_locations`` to retrieve site descriptions). - @property - def site_info(self) -> Any: - raise NotImplementedError( - "site_info must be implemented by utils.BaseMetadata children" - ) - - def __repr__(self) -> str: - return f"{type(self).__name__}(url={self.url})" - - _URL_TOO_LONG_EXAMPLE = """ # n is the number of chunks to divide the query into \n split_list = np.array_split(site_list, n) diff --git a/dataretrieval/waterdata/cql.py b/dataretrieval/waterdata/cql.py index c4bee8ac9..0b69bf6c8 100644 --- a/dataretrieval/waterdata/cql.py +++ b/dataretrieval/waterdata/cql.py @@ -20,7 +20,7 @@ _construct_cql_request, _switch_properties_id, ) -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata.types import ( WATERDATA_SERVICES, ) diff --git a/dataretrieval/waterdata/measurements.py b/dataretrieval/waterdata/measurements.py index cfd615638..61d477036 100644 --- a/dataretrieval/waterdata/measurements.py +++ b/dataretrieval/waterdata/measurements.py @@ -13,7 +13,7 @@ import pandas as pd from dataretrieval.ogc.filters import FILTER_LANG -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata.utils import ( _get_args, get_ogc_data, diff --git a/dataretrieval/waterdata/metadata.py b/dataretrieval/waterdata/metadata.py index 97230167f..af5ad09ef 100644 --- a/dataretrieval/waterdata/metadata.py +++ b/dataretrieval/waterdata/metadata.py @@ -14,7 +14,7 @@ import pandas as pd from dataretrieval.ogc.filters import FILTER_LANG -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata.utils import ( _get_args, _with_state, diff --git a/dataretrieval/waterdata/nearest.py b/dataretrieval/waterdata/nearest.py index 24b72c4d2..f7ed94fa1 100644 --- a/dataretrieval/waterdata/nearest.py +++ b/dataretrieval/waterdata/nearest.py @@ -10,7 +10,7 @@ import pandas as pd -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata.time_series import get_continuous __all__ = ["get_nearest_continuous"] diff --git a/dataretrieval/waterdata/reference.py b/dataretrieval/waterdata/reference.py index 5d43ac357..22fd2a8bb 100644 --- a/dataretrieval/waterdata/reference.py +++ b/dataretrieval/waterdata/reference.py @@ -13,7 +13,7 @@ import pandas as pd from dataretrieval.ogc.schema import _check_ogc_requests -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata.types import ( METADATA_COLLECTIONS, ) diff --git a/dataretrieval/waterdata/samples.py b/dataretrieval/waterdata/samples.py index 864bc962b..be41f9ef2 100644 --- a/dataretrieval/waterdata/samples.py +++ b/dataretrieval/waterdata/samples.py @@ -19,6 +19,7 @@ import pandas as pd from dataretrieval.ogc.errors import _raise_for_non_200 +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.transport.http import ( HTTPX_DEFAULTS, ) @@ -28,7 +29,7 @@ from dataretrieval.transport.http import ( get as _get, ) -from dataretrieval.utils import BaseMetadata, _attach_datetime_columns, to_str +from dataretrieval.utils import _attach_datetime_columns, to_str from dataretrieval.waterdata.types import ( CODE_SERVICES, PROFILES, diff --git a/dataretrieval/waterdata/stats.py b/dataretrieval/waterdata/stats.py index 5666fb305..9ddac65dc 100644 --- a/dataretrieval/waterdata/stats.py +++ b/dataretrieval/waterdata/stats.py @@ -24,10 +24,10 @@ _attach_coordinates, _empty_feature_frame, ) +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.transport.http import default_headers from dataretrieval.transport.pagination import paginate from dataretrieval.transport.sync import run_sync -from dataretrieval.utils import BaseMetadata from dataretrieval.waterdata.utils import BASE_URL __all__ = ["get_data"] diff --git a/dataretrieval/waterdata/time_series.py b/dataretrieval/waterdata/time_series.py index 32657d3bc..44f490091 100644 --- a/dataretrieval/waterdata/time_series.py +++ b/dataretrieval/waterdata/time_series.py @@ -17,7 +17,7 @@ import pandas as pd from dataretrieval.ogc.filters import FILTER_LANG -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata import stats from dataretrieval.waterdata.utils import ( _get_args, diff --git a/dataretrieval/waterdata/utils.py b/dataretrieval/waterdata/utils.py index 18bb8129b..31a95ad63 100644 --- a/dataretrieval/waterdata/utils.py +++ b/dataretrieval/waterdata/utils.py @@ -30,7 +30,7 @@ from dataretrieval.credentials import WATERDATA_BASE_URL from dataretrieval.ogc import OgcDialect, prepare_request_args from dataretrieval.ogc import get_ogc_data as _facade_get_ogc_data -from dataretrieval.utils import BaseMetadata +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.waterdata.types import ( PROFILE_LOOKUP, PROFILES, diff --git a/dataretrieval/wateruse.py b/dataretrieval/wateruse.py index 5e09e3a17..be618d993 100644 --- a/dataretrieval/wateruse.py +++ b/dataretrieval/wateruse.py @@ -55,11 +55,12 @@ _combine_chunk_responses, ) from dataretrieval.exceptions import DataRetrievalError +from dataretrieval.response_metadata import BaseMetadata from dataretrieval.transport.http import default_headers, open_async_client from dataretrieval.transport.pagination import paginate from dataretrieval.transport.retry import RetryPolicy, retry_async from dataretrieval.transport.sync import run_sync -from dataretrieval.utils import BaseMetadata, _raise_for_status, to_str +from dataretrieval.utils import _raise_for_status, to_str __all__ = [ "get_wateruse", diff --git a/dataretrieval/wqp.py b/dataretrieval/wqp.py index b77348ea2..8249c76ea 100644 --- a/dataretrieval/wqp.py +++ b/dataretrieval/wqp.py @@ -16,7 +16,9 @@ import pandas as pd -from .utils import BaseMetadata, _attach_datetime_columns, _query_with_retry +from dataretrieval.response_metadata import BaseMetadata + +from .utils import _attach_datetime_columns, _query_with_retry __all__ = [ "get_results", diff --git a/docs/source/architecture/decisions/0001-modular-monolith.rst b/docs/source/architecture/decisions/0001-modular-monolith.rst index 35f2af4e4..006e912e1 100644 --- a/docs/source/architecture/decisions/0001-modular-monolith.rst +++ b/docs/source/architecture/decisions/0001-modular-monolith.rst @@ -43,7 +43,8 @@ Consequences Compliance ---------- -``tests/architecture_test.py`` prevents shared OGC infrastructure from -importing service adapters and prevents modern modules from depending on legacy -NWIS. The installed-wheel CI job verifies that the whole monolith ships as one -usable artifact. +``.importlinter`` prevents shared OGC infrastructure from importing service +adapters, prevents modern modules from depending on legacy NWIS, and requires +the runtime import graph to stay acyclic; ``lint-imports`` checks it in +pre-commit and CI. The installed-wheel CI job verifies that the whole monolith +ships as one usable artifact. diff --git a/docs/source/architecture/decisions/0003-dependency-direction.rst b/docs/source/architecture/decisions/0003-dependency-direction.rst index 8e24b2296..98d9f9e4b 100644 --- a/docs/source/architecture/decisions/0003-dependency-direction.rst +++ b/docs/source/architecture/decisions/0003-dependency-direction.rst @@ -44,16 +44,29 @@ Consequences Compliance ---------- -``tests/architecture_test.py`` parses runtime imports and enforces the rules -that hold today. Its allowlist is the authoritative inventory of exact temporary -cross-boundary imports; this ADR owns the direction and rationale rather than a -second copy of that mutable inventory. - -Focused fitness functions verify the current boundaries: NGWMN's only OGC -dependency is the facade, ``waterdata.utils`` does not bulk re-export private -OGC helpers, ``ogc.shaping`` does not depend on ``ogc.engine``, Water Use has -no OGC dependency, and both the OGC and transport runtime graphs are acyclic. - -The exact allowlist should shrink as private seams move. Any growth requires -explicit architecture review, and a change to the dependency policy requires -this ADR to be superseded. +``.importlinter`` is the single authority for dependency direction. It declares +the layer stack, the allowlist of OGC consumers, NGWMN's facade-only seam, the +NWIS quarantine, collection-family independence, and package-wide acyclicity; +``lint-imports`` checks all of it against the transitive import graph in +pre-commit and CI. A boundary that legitimately moves is one edit, in that file, +alongside the ADR it cites. + +These rules were previously asserted a second time in +``tests/architecture_test.py``, by hand-parsing the AST. That duplication is +gone. The tests now cover only what an import graph cannot express — which +symbols cross a seam, what a module's declared exports are, the AST shape of a +facade, and the one boundary that has to be asserted positively rather than +forbidden. A new rule that is purely about module-to-module direction belongs in +``.importlinter``. + +Named contracts verify the current boundaries: NGWMN's only OGC dependency is +the facade, ``ogc.shaping`` does not depend on ``ogc.engine``, Water Use and the +other non-OGC adapters cannot reach the OGC subsystem at all, and the runtime +graph is acyclic package-wide rather than only within ``ogc`` and ``transport``. +``waterdata.utils`` not bulk re-exporting private OGC helpers stays in the +fitness functions, because that claim is about the module's ``__all__``. + +The OGC consumer list is an allowlist, so a new service module is refused until +someone places it deliberately. It should shrink as private seams move. Any +growth requires explicit architecture review, and a change to the dependency +policy requires this ADR to be superseded. diff --git a/docs/source/architecture/decisions/0005-legacy-nwis.rst b/docs/source/architecture/decisions/0005-legacy-nwis.rst index c2d0d4d32..5d9be3b5e 100644 --- a/docs/source/architecture/decisions/0005-legacy-nwis.rst +++ b/docs/source/architecture/decisions/0005-legacy-nwis.rst @@ -44,5 +44,6 @@ Compliance ---------- Deprecation tests verify one warning per public call and validate named Water -Data replacements. ``tests/architecture_test.py`` prevents modern package -modules from importing ``dataretrieval.nwis``. +Data replacements. The ``nwis-quarantine`` contract in ``.importlinter`` +prevents modern package modules from importing ``dataretrieval.nwis``, directly +or through an intermediary. diff --git a/docs/source/architecture/decisions/0006-service-neutral-transport.rst b/docs/source/architecture/decisions/0006-service-neutral-transport.rst index 440ba5889..801f7eeff 100644 --- a/docs/source/architecture/decisions/0006-service-neutral-transport.rst +++ b/docs/source/architecture/decisions/0006-service-neutral-transport.rst @@ -101,10 +101,11 @@ Consequences Compliance ---------- -``tests/architecture_test.py`` enforces transport dependency direction, an -acyclic transport graph, Water Use isolation from OGC, that presentation and -frame-assembly modules do not reappear inside transport, and that only -``dataretrieval.credentials`` names the API-key host. Component and adapter +``.importlinter`` enforces transport dependency direction, an acyclic runtime +graph, and Water Use isolation from OGC. ``tests/architecture_test.py`` covers +what the import graph cannot see: that presentation and frame-assembly modules +do not reappear inside transport, and that only ``dataretrieval.credentials`` +names the API-key host. Component and adapter tests cover cursor termination, row caps, response aggregation, retry exhaustion, ``Retry-After`` limits, the no-progress budget, which failures are re-sent, cancellation, no-partial fan-out behavior, and credential host scoping. diff --git a/docs/source/architecture/decisions/0007-adapter-facades.rst b/docs/source/architecture/decisions/0007-adapter-facades.rst index d4cc90cf5..b63d41b0a 100644 --- a/docs/source/architecture/decisions/0007-adapter-facades.rst +++ b/docs/source/architecture/decisions/0007-adapter-facades.rst @@ -61,6 +61,7 @@ Compliance ``tests/contracts/public_api_test.py`` freezes Water Data imports, signatures, facade identity, and compatibility names. ``tests/architecture_test.py`` -requires a logic-free facade, exact active-service exports, isolated collection -families, no lateral adapter reach-through, and separate OGC request -construction and schema execution. +requires a logic-free facade, exact active-service exports, and separate OGC +request construction and schema execution. ``.importlinter`` keeps the +collection families independent of each other, holds NGWMN to the OGC facade, +and prevents lateral adapter reach-through. diff --git a/docs/source/architecture/index.rst b/docs/source/architecture/index.rst index 3f4e30f76..3f1fcbf8a 100644 --- a/docs/source/architecture/index.rst +++ b/docs/source/architecture/index.rst @@ -110,7 +110,8 @@ Shared components contract; ``retry`` classifies failures into OGC interruption types; and ``shaping``, ``dates``, ``filters``, and ``errors`` isolate their named protocol concerns. The full runtime OGC graph, including the facade, is - acyclic — enforced by ``tests/architecture_test.py``. + acyclic — enforced package-wide by the ``acyclic`` contract in + ``.importlinter``. ``dataretrieval.transport`` Internal service-neutral execution layer. Owns guarded client lifecycle and @@ -126,11 +127,18 @@ Shared components Stable error-policy leaf. It has no runtime third-party dependency, and every service can import it without creating an infrastructure cycle. +``dataretrieval.response_metadata`` + ``BaseMetadata``, the second half of every getter's ``(DataFrame, + metadata)`` return contract. A dependency-free leaf: nearly every service + module needs this class, and while it lived in ``utils`` beside the legacy + query machinery, importing it pulled that module's whole HTTP stack in + transitively. + ``dataretrieval.utils`` - Shared metadata, data-shaping helpers, ambient context support, legacy - request composition, and compatibility imports for transport names that - historically lived here. By default, do not add new service-specific - behavior there. + Data-shaping helpers, ambient context support, legacy request composition, + and compatibility imports for transport names that historically lived here + (including ``BaseMetadata``, so its original import path keeps working). By + default, do not add new service-specific behavior there. ``dataretrieval.codes`` and ``dataretrieval.rdb`` State/time-zone code conversion and RDB parsing leaves. @@ -142,8 +150,16 @@ The intended direction is:: -> third-party library / network Dependencies must not point from shared infrastructure back to a public service -adapter. The executable checks in ``tests/architecture_test.py`` enforce the -rules that hold today and explicitly list temporary variances. +adapter. ``.importlinter`` declares this as a layer stack and ``lint-imports`` +checks it over the transitive import graph, so a violation routed through an +intermediary fails as surely as a direct one. The stack is exhaustive: a new +top-level module fails the contract until it is placed, so where a module +belongs is decided when it is added rather than inferred later. + +``tests/architecture_test.py`` complements those contracts without repeating +them. It covers the rules an import graph cannot express — which symbols cross +a seam, declared ``__all__`` surfaces, the AST shape of a facade, and imports +that must exist rather than be forbidden. Interface view -------------- @@ -265,9 +281,8 @@ and request shapes differ. Known architectural debt ------------------------ -This view records categories and representative locations of debt. The fitness -functions in ``tests/architecture_test.py`` are authoritative for exact current -dependency allowlists. +This view records categories and representative locations of debt. +``.importlinter`` is authoritative for exact current dependency allowlists. - ``ogc/engine.py`` retains compatibility wrappers alongside OGC orchestration. - ``utils.py`` combines metadata, shaping, ambient configuration, legacy @@ -284,7 +299,8 @@ Architecturally significant changes should: #. add or supersede an ADR; #. identify affected characteristics and trade-offs; -#. add or update an executable fitness function; +#. add or update an executable fitness function, and the matching contract in + ``.importlinter`` when the change moves a dependency boundary; #. preserve public contracts or provide a deprecation path; and #. update this view when component responsibilities or dependency rules change. diff --git a/pyproject.toml b/pyproject.toml index d278cc333..9f26a4ceb 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -48,13 +48,20 @@ dataretrieval = ["py.typed"] type-check = [ "mypy", ] -# Complexity gates and history. ``xenon`` fails a build when complexity -# regresses; ``wily`` tracks the trend across commits so a review can say -# whether a change moved the codebase, not just whether it passed. Kept out of -# ``test`` so the test job stays lean -- the metrics job installs this instead. +# Structural gates (complexity ratchets, dependency contracts) and the wily +# trend history. Pure-Python, so it installs anywhere the package does. Kept out +# of ``test`` so the test job stays lean; see CONTRIBUTING.md for what each does. metrics = [ "xenon==0.9.3", "wily==1.25.0", + "complexipy==6.2.0", + "import-linter==2.13", +] +# Advisory weekly deep sweep -- see .github/workflows/code-health.yml. Separate +# from ``metrics`` because pyscn is a compiled binary with no linux/aarch64 +# wheel, and the extra the merge gates depend on should stay pure-Python. +health = [ + "pyscn==1.29.0", ] test = [ "pytest > 5.0.0", diff --git a/tests/architecture_test.py b/tests/architecture_test.py index 5fd62d550..88e8755aa 100644 --- a/tests/architecture_test.py +++ b/tests/architecture_test.py @@ -1,4 +1,28 @@ -"""Executable fitness functions for package dependency direction.""" +"""Executable fitness functions for architecture rules the import graph cannot +express. + +Plain dependency direction -- who may import whom, in which direction, without +cycles -- is declared in ``.importlinter`` and checked by ``lint-imports`` in +pre-commit and CI. Those rules used to be asserted here too, and are not any +more: one rule enforced in two places is one rule that gets updated in one +place. + +What remains is everything a boundary checker cannot see, because an import +graph has no opinion about it: + +* which *symbols* cross a boundary, not just which modules (``ogc.engine``'s + compatibility surface, ``ogc.requests`` borrowing header policy but not the + executing calls); +* the declared public surface -- ``__all__`` presence, ownership, and the + facade union; +* the AST shape of a module, such as a facade proving it contains no logic, or + a call proving it passes a destination URL; +* an import that must *exist*: ``lint-imports`` can forbid an edge, never + require one. + +Adding a rule here that is purely about module-to-module direction is a +regression -- put it in ``.importlinter`` instead. +""" from __future__ import annotations @@ -9,23 +33,6 @@ PACKAGE_ROOT = Path(__file__).parents[1] / "dataretrieval" -_SERVICE_PREFIXES = ( - "dataretrieval.ngwmn", - "dataretrieval.nldi", - "dataretrieval.nwis", - "dataretrieval.streamstats", - "dataretrieval.waterdata", - "dataretrieval.wateruse", - "dataretrieval.wqp", -) - -# NGWMN is the only top-level OGC consumer and uses the small facade -# (``dataretrieval.ogc``) exclusively. Exact equality makes growth or removal -# an intentional architecture change. -_ALLOWED_TOP_LEVEL_OGC_IMPORTS = { - "dataretrieval.ngwmn": {"dataretrieval.ogc"}, -} - #: How many names ``ogc.engine`` may import from ``ogc.requests``. A ceiling #: rather than an exact name list: the claim being enforced is "the legacy #: compatibility surface does not grow", and a name list also fails on every @@ -91,6 +98,8 @@ def visit_ImportFrom(self, node: ast.ImportFrom) -> None: self.modules.update(_resolve_from(self.current_module, self.path, node)) +# Pure over an unchanging tree and called from several tests; without caching +# the suite re-parses the same package files on each call. @functools.cache def _runtime_imports(path: Path) -> set[str]: module = _module_name(path) @@ -99,16 +108,6 @@ def _runtime_imports(path: Path) -> set[str]: return visitor.modules -# Both are pure over an unchanging tree and are called from a dozen places; -# without caching the suite re-parses every package file on each call. -@functools.cache -def _package_import_graph() -> dict[str, set[str]]: - return { - _module_name(path): _runtime_imports(path) - for path in sorted(PACKAGE_ROOT.rglob("*.py")) - } - - def _literal_exports(path: Path) -> set[str]: tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path)) for node in tree.body: @@ -131,60 +130,6 @@ def test_exceptions_has_no_runtime_third_party_dependency() -> None: ) -def test_ogc_does_not_depend_on_service_adapters() -> None: - """The reusable protocol subsystem must not point back to its callers.""" - violations: list[str] = [] - for module, imports in _package_import_graph().items(): - if not ( - module == "dataretrieval.ogc" or module.startswith("dataretrieval.ogc.") - ): - continue - for dependency in imports: - if dependency.startswith(_SERVICE_PREFIXES): - violations.append(f"{module} -> {dependency}") - assert not violations, "Forbidden OGC-to-service imports:\n" + "\n".join(violations) - - -def test_modern_modules_do_not_depend_on_legacy_nwis() -> None: - """NWIS stays quarantined during its deprecation window.""" - violations: list[str] = [] - for module, imports in _package_import_graph().items(): - if module in {"dataretrieval", "dataretrieval.nwis"}: - continue - for dependency in imports: - if dependency == "dataretrieval.nwis" or dependency.startswith( - "dataretrieval.nwis." - ): - violations.append(f"{module} -> {dependency}") - assert not violations, "Modern modules import legacy NWIS:\n" + "\n".join( - violations - ) - - -def test_top_level_ogc_consumers_match_documented_variances() -> None: - """No new top-level service may acquire an accidental OGC dependency.""" - observed: dict[str, set[str]] = {} - for path in sorted(PACKAGE_ROOT.glob("*.py")): - module = _module_name(path) - if module in {"dataretrieval", "dataretrieval.nwis"}: - continue - dependencies = { - dependency - for dependency in _runtime_imports(path) - if dependency == "dataretrieval.ogc" - or dependency.startswith("dataretrieval.ogc.") - } - if dependencies: - observed[module] = dependencies - - assert observed == _ALLOWED_TOP_LEVEL_OGC_IMPORTS, ( - "Top-level OGC dependencies differ from the architecture allowlist. " - "Update the code and allowlist; supersede ADR 0003 if the dependency " - "policy changes.\n" - f"expected={_ALLOWED_TOP_LEVEL_OGC_IMPORTS!r}\nobserved={observed!r}" - ) - - def test_engine_request_import_surface_does_not_grow() -> None: """Engine may preserve legacy request names but may not grow a new hub. @@ -218,80 +163,7 @@ def test_engine_request_import_surface_does_not_grow() -> None: ) -# --- Strengthened OGC boundary tests --- - - -def test_ogc_runtime_graph_is_acyclic() -> None: - """The OGC runtime import graph (including the facade) has no cycles. - - Now that no implementation module imports ``dataretrieval.ogc`` (the facade - ``__init__.py``), the full OGC graph — facade included — forms a DAG. - This is enforced without any documented exclusion. - """ - ogc_modules: dict[str, set[str]] = {} - for module, imports in _package_import_graph().items(): - if module == "dataretrieval.ogc" or module.startswith("dataretrieval.ogc."): - # Filter to intra-OGC dependencies - ogc_deps = { - dep - for dep in imports - if dep == "dataretrieval.ogc" or dep.startswith("dataretrieval.ogc.") - } - ogc_modules[module] = ogc_deps - - # DFS cycle detection - WHITE, GRAY, BLACK = 0, 1, 2 - color: dict[str, int] = {m: WHITE for m in ogc_modules} - path: list[str] = [] - - def dfs(node: str) -> list[str] | None: - color[node] = GRAY - path.append(node) - for dep in ogc_modules.get(node, set()): - if dep not in color: - continue - if color[dep] == GRAY: - cycle_start = path.index(dep) - return path[cycle_start:] + [dep] - if color[dep] == WHITE: - result = dfs(dep) - if result: - return result - path.pop() - color[node] = BLACK - return None - - for module in ogc_modules: - if color[module] == WHITE: - cycle = dfs(module) - if cycle: - raise AssertionError( - f"Cycle in OGC runtime graph: {' -> '.join(cycle)}" - ) - - -def test_shaping_has_no_engine_dependency() -> None: - """ogc.shaping must not import ogc.engine, even lazily.""" - shaping_imports = _runtime_imports(PACKAGE_ROOT / "ogc" / "shaping.py") - engine_deps = {dep for dep in shaping_imports if dep == "dataretrieval.ogc.engine"} - assert not engine_deps, ( - f"ogc.shaping must not depend on ogc.engine. Found: {engine_deps}" - ) - - -def test_ngwmn_uses_ogc_facade() -> None: - """NGWMN must use ONLY the ogc facade, not engine or other internals.""" - ngwmn_imports = _runtime_imports(PACKAGE_ROOT / "ngwmn.py") - ogc_deps = { - dep - for dep in ngwmn_imports - if dep == "dataretrieval.ogc" or dep.startswith("dataretrieval.ogc.") - } - # Exact equality: the ONLY OGC dependency is the facade package itself. - assert ogc_deps == {"dataretrieval.ogc"}, ( - "NGWMN must use ONLY the OGC facade (dataretrieval.ogc), not internals. " - f"Found: {ogc_deps}" - ) +# --- Seams that are about symbols and call sites, not module direction --- def test_waterdata_utils_is_not_an_ogc_reexport_hub() -> None: @@ -363,7 +235,7 @@ def test_default_header_calls_are_target_scoped() -> None: ) -# --- Shared execution-layer boundaries --- +# --- Where code and constants are allowed to live --- def test_transport_is_execution_policy_only() -> None: @@ -429,74 +301,6 @@ def test_credential_policy_has_one_definition() -> None: ) -def test_transport_does_not_depend_on_ogc_or_services() -> None: - """Transport policy must point inward, never back to protocol adapters.""" - violations: list[str] = [] - transport_root = PACKAGE_ROOT / "transport" - for path in sorted(transport_root.rglob("*.py")): - module = _module_name(path) - for dependency in _runtime_imports(path): - if ( - dependency == "dataretrieval.ogc" - or dependency.startswith("dataretrieval.ogc.") - or dependency.startswith(_SERVICE_PREFIXES) - ): - violations.append(f"{module} -> {dependency}") - assert not violations, "Transport crossed an adapter boundary:\n" + "\n".join( - violations - ) - - -def test_wateruse_has_no_ogc_dependency() -> None: - """The non-OGC Water Use adapter must consume transport directly.""" - imports = _runtime_imports(PACKAGE_ROOT / "wateruse.py") - ogc_dependencies = { - dependency - for dependency in imports - if dependency == "dataretrieval.ogc" - or dependency.startswith("dataretrieval.ogc.") - } - assert not ogc_dependencies, ( - f"Water Use imported OGC implementation modules: {sorted(ogc_dependencies)}" - ) - - -def test_transport_runtime_graph_is_acyclic() -> None: - """The service-neutral transport package must remain a directed acyclic graph.""" - graph = { - module: { - dependency - for dependency in imports - if dependency == "dataretrieval.transport" - or dependency.startswith("dataretrieval.transport.") - } - for module, imports in _package_import_graph().items() - if module == "dataretrieval.transport" - or module.startswith("dataretrieval.transport.") - } - visiting: set[str] = set() - visited: set[str] = set() - - def visit(module: str, path: tuple[str, ...]) -> None: - if module in visiting: - start = path.index(module) - cycle = (*path[start:], module) - raise AssertionError( - f"Cycle in transport runtime graph: {' -> '.join(cycle)}" - ) - if module in visited: - return - visiting.add(module) - for dependency in graph.get(module, set()): - if dependency in graph: - visit(dependency, (*path, module)) - visiting.remove(module) - visited.add(module) - - for module in graph: - visit(module, ()) - - # --- Adapter structure and public export boundaries --- #: The modules whose public surface must be declared, not inferred. This is a @@ -614,60 +418,6 @@ def test_waterdata_api_is_a_logic_free_compatibility_facade() -> None: assert not offenders, f"waterdata.api contains implementation: {offenders}" -def test_waterdata_collection_families_do_not_import_each_other() -> None: - """Families share through Water Data policy, OGC, and transport -- not laterally. - - A "family" is a module the ``waterdata.api`` facade re-exports from, so the - set comes from :data:`_WATERDATA_FAMILIES` rather than being restated. That - list is self-enforcing: a seventh family the facade re-exports must be added - to ``_EXPECTED_MODULE_EXPORTS`` or ``test_api_facade_exports_exactly_the_ - family_union`` fails, and it lands here on the same edit. - - Composed getters like ``nearest`` are deliberately outside it. They are not - peers of a family; ``get_nearest_continuous`` builds on ``get_continuous``, - which is ordinary layering rather than a lateral reach. - """ - graph = _package_import_graph() - families = { - "dataretrieval." + relative.removesuffix(".py").replace("/", ".") - for relative in _WATERDATA_FAMILIES - } - violations = [] - for module in sorted(families): - for dependency in graph[module]: - if dependency in families and dependency != module: - violations.append(f"{module} -> {dependency}") - assert not violations, "Lateral collection-family imports:\n" + "\n".join( - violations - ) - - -def test_service_adapters_do_not_reach_through_each_other() -> None: - # Every active service; NWIS is excluded because its deprecation is governed - # by its own fitness function. - adapters = set(_SERVICE_PREFIXES) - {"dataretrieval.nwis"} - - def owner(name: str) -> str | None: - """The adapter ``name`` belongs to, or None if it is shared code.""" - return next( - (a for a in adapters if name == a or name.startswith(a + ".")), - None, - ) - - violations: list[str] = [] - for module, imports in _package_import_graph().items(): - source = owner(module) - if source is None: - continue - for dependency in imports: - target = owner(dependency) - if target is not None and target != source: - violations.append(f"{module} -> {dependency}") - assert not violations, "Adapter-to-adapter imports:\n" + "\n".join( - sorted(set(violations)) - ) - - def test_ogc_request_construction_does_not_execute_http() -> None: """Building a request may borrow header policy, never the executing calls.