From 420ac92c19a4ffa6389059e9e9432252a38bb8d4 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:29:31 +0200 Subject: [PATCH 1/7] fix(alignment): align reordered shared-dim coords by label in merge (#550) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit §8 aligns by label, not position, so the same labels in a different order are the same coordinate. The constant path already reindexed a pure reorder, but the expression-merge path raised under v1 and silently misaligned under legacy. merge() now conforms shared user dims to the first operand's order before the §8/§11 checks, so a reorder reindexes (correct under both semantics) while a differing label set still raises. Aux coords ride along the reindex, so §11 conflicts are preserved. Spec: convention.md §8 now states order-independence explicitly and is retitled "Shared dimensions must carry the same labels" (was framed as xarray's order-sensitive `exact`). Supersedes #550. Co-Authored-By: Claude Opus 4.8 (1M context) --- arithmetics-design/convention.md | 19 +++++++++------ linopy/expressions.py | 40 ++++++++++++++++++++++++++++++++ test/test_legacy_violations.py | 26 +++++++++++++++++++++ 3 files changed, 78 insertions(+), 7 deletions(-) diff --git a/arithmetics-design/convention.md b/arithmetics-design/convention.md index 05c3f9c77..aca0dd1ea 100644 --- a/arithmetics-design/convention.md +++ b/arithmetics-design/convention.md @@ -127,16 +127,21 @@ array is treated as a scalar; a Python `list` is read as a numpy array (it carries values, not labels). Implemented in `linopy.alignment` ([#736]). -### §8. Shared dimensions must match exactly +### §8. Shared dimensions must carry the same labels -If two operands share a dimension, their coordinate labels must be identical, -or the operator raises `ValueError`. +If two operands share a dimension, their coordinate labels must be the same +*set*, or the operator raises `ValueError`. Order is immaterial: the same +labels in a different order are the same coordinate and align by label (a +reindex), following "by label, never by position" above — only a difference in +the label set raises. -This is xarray's model with `arithmetic_join="exact"` — deliberately stricter -than xarray's own default (`inner`). An inner join silently drops the +This is close to xarray's `arithmetic_join="exact"` — deliberately stricter +than xarray's own default (`inner`) — but order-independent, where xarray's +`exact` would reject a pure reorder. An inner join silently drops the non-overlapping labels, and in an optimization model a dropped coordinate is a -dropped term or constraint: a silent wrong answer. An exact match surfaces the -mismatch where it happens. (The [pyoframe] library uses the same model.) +dropped term or constraint: a silent wrong answer. Matching on the label set +surfaces a real mismatch where it happens. (The [pyoframe] library uses the +same model.) Because the rule is identical for every operator, the operator-alignment split ([#708]) — `*` aligning by label while `+`, `-`, `/` go by position — diff --git a/linopy/expressions.py b/linopy/expressions.py index 7ff13775a..e4c930c9c 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -2567,6 +2567,42 @@ def as_expression( Mergeable: TypeAlias = BaseExpression | variables.Variable | Dataset +def _conform_reordered_merge_dims( + data: list[Dataset], concat_dim: str +) -> list[Dataset]: + """ + Reindex shared user dims that carry the same labels in a different order. + + §8 aligns by label, not position: the same labels reordered are the same + coordinate, so each operand's shared user dims are conformed to the first + operand's order before the §8 / §11 checks and the concat. A pure reorder + introduces no new positions (the label set is unchanged) so no absence is + created; a genuinely different label *set* is left for the §8 mismatch + check to flag. Helper dims (``_term`` / ``_factor``) and the concat dim + are excluded — those legitimately vary across the merged operands. Mirrors + the constant path's ``_reindex_reordered_dims`` (#550). + """ + if len(data) < 2: + return data + skip = set(HELPER_DIMS) | {concat_dim} + ref = {d: data[0].indexes[d] for d in data[0].indexes if d not in skip} + if not ref: + return data + out = [data[0]] + for ds in data[1:]: + reindexer = { + d: idx + for d, idx in ref.items() + if d in ds.indexes + and not isinstance(ds.indexes[d], pd.MultiIndex) + and not ds.indexes[d].equals(idx) + and len(ds.indexes[d]) == len(idx) + and set(ds.indexes[d]) == set(idx) + } + out.append(ds.reindex(reindexer) if reindexer else ds) + return out + + @overload def merge( exprs: Sequence[Mergeable] | Mergeable, @@ -2659,6 +2695,10 @@ def merge( data = [e.data if isinstance(e, linopy_types) else e for e in exprs] data = [fill_missing_coords(ds, fill_helper_dims=True) for ds in data] + # §8 aligns by label, not position — a pure reorder is not a mismatch. + if join is None: + data = _conform_reordered_merge_dims(data, dim) + # §11: aux-coord conflict is independent of dim alignment — fires on # every join path. xr.concat(..., compat="override") silently drops # the conflicting aux coord, which is the #295 bug v1 closes; we must diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index 5482770ac..5c3ab0e84 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -804,6 +804,21 @@ def test_var_plus_var_broadcast_non_shared_dim_works( result = a + b assert set(result.coord_dims) == {"time", "scenario"} + def test_var_plus_var_reordered_labels_align(self, m: Model) -> None: + a = m.add_variables(coords=[pd.Index(["costs", "penalty"], name="e")], name="a") + b = m.add_variables(coords=[pd.Index(["penalty", "costs"], name="e")], name="b") + result = (1 * a) + (1 * b) + assert list(result.coeffs.coords["e"].values) == ["costs", "penalty"] + + def test_reordered_constants_pair_by_label_not_position(self, m: Model) -> None: + ea = pd.Index(["costs", "penalty"], name="e") + eb = pd.Index(["penalty", "costs"], name="e") + a = m.add_variables(coords=[ea], name="a") + pd.Series([100.0, 200.0], index=ea) + b = m.add_variables(coords=[eb], name="b") + pd.Series([1.0, 2.0], index=eb) + result = a + b + assert float(result.const.sel(e="costs")) == 102.0 + assert float(result.const.sel(e="penalty")) == 201.0 + @pytest.mark.legacy def test_var_plus_var_different_labels_silent( self, x: Variable, x_other: Variable @@ -1534,6 +1549,17 @@ def test_var_plus_var_aux_conflict_raises(self, m: Model, A: pd.Index) -> None: with pytest.raises(ValueError, match="Auxiliary coordinate"): v + w + @pytest.mark.v1 + def test_aux_conflict_survives_reordered_dim(self, m: Model) -> None: + v = m.add_variables( + lower=0, coords=[pd.Index(["x", "y", "z"], name="A")], name="v" + ).assign_coords(B=("A", [1, 2, 3])) + w = m.add_variables( + lower=0, coords=[pd.Index(["z", "y", "x"], name="A")], name="w" + ).assign_coords(B=("A", [1, 2, 3])) + with pytest.raises(ValueError, match="Auxiliary coordinate"): + v + w + @pytest.mark.v1 def test_mul_constant_aux_conflict_raises(self, m: Model, A: pd.Index) -> None: """Same rule on the multiplication path — not just ``+``.""" From c1f89a9788b7e48f3669f3e93a7409796a4c29d0 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 17:55:13 +0200 Subject: [PATCH 2/7] =?UTF-8?q?perf(merge):=20fold=20reorder-conform=20and?= =?UTF-8?q?=20=C2=A78=20mismatch=20detection=20into=20one=20pass?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The reorder fix added a second walk over the shared user dims (one to reindex reordered coords, one to detect a genuine mismatch), duplicating the per-dim .equals() work on every join=None merge — the hot path during model building. conform_merge_dims does both in a single pass and replaces merge_shared_user_coord_mismatch + the separate conform helper. Behaviour is unchanged (full suite 6476 passed under both semantics). Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/expressions.py | 72 +++++++++---------------------------------- linopy/semantics.py | 55 ++++++++++++++++++++++----------- 2 files changed, 52 insertions(+), 75 deletions(-) diff --git a/linopy/expressions.py b/linopy/expressions.py index e4c930c9c..658f88508 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -94,10 +94,10 @@ _shared_dim_mismatch_message, absorb_absence, check_user_nan, + conform_merge_dims, enforce_aux_conflict, first_mismatched_dim, is_v1, - merge_shared_user_coord_mismatch, warn_legacy, ) from linopy.types import ( @@ -2567,42 +2567,6 @@ def as_expression( Mergeable: TypeAlias = BaseExpression | variables.Variable | Dataset -def _conform_reordered_merge_dims( - data: list[Dataset], concat_dim: str -) -> list[Dataset]: - """ - Reindex shared user dims that carry the same labels in a different order. - - §8 aligns by label, not position: the same labels reordered are the same - coordinate, so each operand's shared user dims are conformed to the first - operand's order before the §8 / §11 checks and the concat. A pure reorder - introduces no new positions (the label set is unchanged) so no absence is - created; a genuinely different label *set* is left for the §8 mismatch - check to flag. Helper dims (``_term`` / ``_factor``) and the concat dim - are excluded — those legitimately vary across the merged operands. Mirrors - the constant path's ``_reindex_reordered_dims`` (#550). - """ - if len(data) < 2: - return data - skip = set(HELPER_DIMS) | {concat_dim} - ref = {d: data[0].indexes[d] for d in data[0].indexes if d not in skip} - if not ref: - return data - out = [data[0]] - for ds in data[1:]: - reindexer = { - d: idx - for d, idx in ref.items() - if d in ds.indexes - and not isinstance(ds.indexes[d], pd.MultiIndex) - and not ds.indexes[d].equals(idx) - and len(ds.indexes[d]) == len(idx) - and set(ds.indexes[d]) == set(idx) - } - out.append(ds.reindex(reindexer) if reindexer else ds) - return out - - @overload def merge( exprs: Sequence[Mergeable] | Mergeable, @@ -2695,30 +2659,24 @@ def merge( data = [e.data if isinstance(e, linopy_types) else e for e in exprs] data = [fill_missing_coords(ds, fill_helper_dims=True) for ds in data] - # §8 aligns by label, not position — a pure reorder is not a mismatch. + # §8: align shared user dims by label in one pass — a reorder reindexes, + # a differing label set is the mismatch raised/warned below. Runs before + # the §11 aux check so reordered aux coords compare in a consistent order. + mismatch = None if join is None: - data = _conform_reordered_merge_dims(data, dim) + data, mismatch = conform_merge_dims(data, concat_dim=dim) - # §11: aux-coord conflict is independent of dim alignment — fires on - # every join path. xr.concat(..., compat="override") silently drops - # the conflicting aux coord, which is the #295 bug v1 closes; we must - # raise (v1) / warn (legacy) before xr.concat sees the data, regardless - # of how the caller resolves the §8 dim mismatch. + # §11: aux-coord conflict fires on every join path — xr.concat(..., + # compat="override") silently drops it (the #295 bug v1 closes). enforce_aux_conflict(data) - # §8: shared *user* dimension coordinates must match exactly across all - # operands. Helper dims (_term, _factor) legitimately differ, so we - # validate user dims separately and keep xr.concat on join="outer" - # (which doesn't enforce "exact" — that's what this check is for). - if join is None: - mismatch = merge_shared_user_coord_mismatch(data, concat_dim=dim) - if is_v1() and mismatch is not None: - raise ValueError(_shared_dim_mismatch_message(*mismatch)) - # LEGACY: remove at 1.0 — warn-on-divergence is the migration signal. - if mismatch is not None: - warn_legacy( - _legacy_coord_mismatch_message(f"merge along dim {dim!r}", *mismatch), - ) + if is_v1() and mismatch is not None: + raise ValueError(_shared_dim_mismatch_message(*mismatch)) + # LEGACY: remove at 1.0 — warn-on-divergence is the migration signal. + if mismatch is not None: + warn_legacy( + _legacy_coord_mismatch_message(f"merge along dim {dim!r}", *mismatch) + ) if join is not None: override = join == "override" diff --git a/linopy/semantics.py b/linopy/semantics.py index 1ef43cd58..7c31eb937 100644 --- a/linopy/semantics.py +++ b/linopy/semantics.py @@ -299,31 +299,50 @@ def first_mismatched_dim(a: DataArray, b: DataArray) -> tuple[str, Any, Any] | N return None -def merge_shared_user_coord_mismatch( +def conform_merge_dims( datasets: Sequence[Dataset], concat_dim: str -) -> tuple[str, Any, Any] | None: +) -> tuple[list[Dataset], tuple[str, Any, Any] | None]: """ - Find a shared user dim where the operands' labels disagree. - - Returns ``(dim_name, left_labels, right_labels)`` for the first - mismatch found, or ``None`` if all operands agree. Helper dims - (``_term``, ``_factor``) and the concat dim itself are excluded — - those legitimately vary across the operands being merged. Compares - bare dimension indexes (``d.indexes[k]``) so non-dim (auxiliary) - coords are ignored — those are §11's job. + Align shared user dims for a merge, in a single pass over the operands. + + §8 aligns by label, not position: a shared user dim whose labels match the + first operand's as a *set* but in a different order is reindexed to that + order (returned in the conformed list). A dim whose label set differs is a + real mismatch, returned as ``(dim, first_labels, other_labels)`` for the + caller to raise (v1) / warn (legacy). Helper dims (``_term``, ``_factor``) + and the concat dim are excluded; bare dimension indexes are compared so + auxiliary coords are §11's job, and MultiIndex dims are left to §11. """ + datasets = list(datasets) + if len(datasets) < 2: + return datasets, None skip = set(HELPER_DIMS) | {concat_dim} - per_ds = [ + indexed = [ {k: d.indexes[k] for k in d.dims if k not in skip and k in d.indexes} for d in datasets ] - shared = set.intersection(*(set(p.keys()) for p in per_ds)) if per_ds else set() - for d_name in shared: - ref = per_ds[0][d_name] - for p in per_ds[1:]: - if not ref.equals(p[d_name]): - return str(d_name), ref.values, p[d_name].values - return None + shared = set.intersection(*(set(p) for p in indexed)) + if not shared: + return datasets, None + + out = [datasets[0]] + mismatch: tuple[str, Any, Any] | None = None + for i in range(1, len(datasets)): + reindexer = {} + for d in shared: + ref, idx = indexed[0][d], indexed[i][d] + if ref.equals(idx): + continue + if ( + not isinstance(idx, pd.MultiIndex) + and len(idx) == len(ref) + and set(idx) == set(ref) + ): + reindexer[d] = ref + elif mismatch is None: + mismatch = (str(d), ref.values, idx.values) + out.append(datasets[i].reindex(reindexer) if reindexer else datasets[i]) + return out, mismatch def conflicting_aux_coord( From 19e9b6c95e07f14207706f6ff0f0dea4fefd9c4a Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:15:21 +0200 Subject: [PATCH 3/7] test(merge): pin reorder behaviour on multi-operand, quadratic, and MultiIndex paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Regression guards for the §8 by-label alignment beyond the 2-operand case: multi-operand merge([a,b,c]) pairs by label, quadratic merge aligns reordered dims, and a reordered stacked MultiIndex raises (xarray cannot reindex it by tuple — left to §11; tied to the #744 MultiIndex-storage decision). Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_legacy_violations.py | 40 ++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index 5c3ab0e84..a997b0815 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -58,6 +58,7 @@ class corresponds to a section of ``arithmetics-design/convention.md`` from linopy import Model from linopy.config import LinopySemanticsWarning +from linopy.expressions import merge from linopy.testing import assert_linequal from linopy.variables import Variable @@ -819,6 +820,45 @@ def test_reordered_constants_pair_by_label_not_position(self, m: Model) -> None: assert float(result.const.sel(e="costs")) == 102.0 assert float(result.const.sel(e="penalty")) == 201.0 + def test_multi_operand_merge_reordered_pairs_by_label(self, m: Model) -> None: + ea = pd.Index(["x", "y", "z"], name="e") + er = pd.Index(["z", "y", "x"], name="e") + a = m.add_variables(coords=[ea], name="a") + pd.Series( + [1.0, 2.0, 3.0], index=ea + ) + b = m.add_variables(coords=[er], name="b") + pd.Series( + [10.0, 20.0, 30.0], index=er + ) + c = m.add_variables(coords=[ea], name="c") + pd.Series( + [100, 200, 300.0], index=ea + ) + result = merge([a, b, c], cls=type(a)) + assert float(result.const.sel(e="x")) == 131.0 + assert float(result.const.sel(e="z")) == 313.0 + + def test_quadratic_merge_reordered_aligns(self, m: Model) -> None: + ea = pd.Index(["x", "y", "z"], name="e") + er = pd.Index(["z", "y", "x"], name="e") + x = m.add_variables(coords=[ea], name="x") + y = m.add_variables(coords=[er], name="y") + result = (x * x) + (y * y) + assert list(result.coeffs.coords["e"].values) == ["x", "y", "z"] + + @pytest.mark.v1 + def test_reordered_multiindex_raises(self, m: Model) -> None: + mi1 = pd.MultiIndex.from_tuples( + [(1, "a"), (1, "b"), (2, "a")], names=["p", "s"] + ) + mi2 = pd.MultiIndex.from_tuples( + [(2, "a"), (1, "b"), (1, "a")], names=["p", "s"] + ) + mi1.name = "snap" + mi2.name = "snap" + x = m.add_variables(coords=[mi1], name="x") + y = m.add_variables(coords=[mi2], name="y") + with pytest.raises(ValueError, match="Auxiliary coordinate"): + (1 * x) + (1 * y) + @pytest.mark.legacy def test_var_plus_var_different_labels_silent( self, x: Variable, x_other: Variable From 9bc157c9f54c68fe8ea234a72e89d5b0819bfb6b Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 18:32:41 +0200 Subject: [PATCH 4/7] =?UTF-8?q?fix(merge):=20align=20reordered=20stacked?= =?UTF-8?q?=20MultiIndex=20by=20tuple=20(resolve=20=C2=A78/=C2=A711=20gap)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A reordered full MultiIndex is "the same coordinate spelled differently" and per §8/§11 should align — but reindex cannot reorder a stacked MI by tuple, so it previously fell through to a confusing §11 aux-coord raise. conform_merge_dims now permutes via positional isel using get_indexer, which works uniformly for a plain index and a MultiIndex's tuples (and get_indexer doubles as the same-set test, replacing the set() comparison — cheaper). A genuinely different label set still raises the §8 mismatch. convention.md §11 now states order-independence for the full-MI case explicitly. Only the MultiIndex case was affected; a plain dim with aux coords already reordered correctly (the aux coord rides along the permute). Co-Authored-By: Claude Opus 4.8 (1M context) --- arithmetics-design/convention.md | 4 +++- linopy/semantics.py | 30 +++++++++++++++--------------- test/test_legacy_violations.py | 16 ++++++++++------ 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/arithmetics-design/convention.md b/arithmetics-design/convention.md index aca0dd1ea..9eb743187 100644 --- a/arithmetics-design/convention.md +++ b/arithmetics-design/convention.md @@ -207,7 +207,9 @@ values: An input that reconstructs the *entire* MultiIndex (all levels, every combination) is not a conflict — it is the same coordinate spelled -differently, and aligns element-wise under §8. +differently, and aligns by tuple under §8, in any order. Order is +immaterial here exactly as for a plain dimension: the same tuples in a +different order are reordered to match, not rejected. (Legacy projects implicitly and warns — scenario B of the [#732]/[#737] discussion; the implicit projection is removed at 1.0.) diff --git a/linopy/semantics.py b/linopy/semantics.py index 7c31eb937..e498fd2a4 100644 --- a/linopy/semantics.py +++ b/linopy/semantics.py @@ -305,13 +305,16 @@ def conform_merge_dims( """ Align shared user dims for a merge, in a single pass over the operands. - §8 aligns by label, not position: a shared user dim whose labels match the - first operand's as a *set* but in a different order is reindexed to that - order (returned in the conformed list). A dim whose label set differs is a - real mismatch, returned as ``(dim, first_labels, other_labels)`` for the - caller to raise (v1) / warn (legacy). Helper dims (``_term``, ``_factor``) - and the concat dim are excluded; bare dimension indexes are compared so - auxiliary coords are §11's job, and MultiIndex dims are left to §11. + §8 aligns by label, not position: a shared user dim whose labels are the + first operand's reordered (same set) is permuted to that order. The permute + uses positional ``isel`` rather than ``reindex`` so it works uniformly for a + plain index and a stacked MultiIndex's tuples (``reindex`` cannot reorder a + MultiIndex by tuple); ``get_indexer`` also doubles as the same-set test. A + dim whose label set differs is a real mismatch, returned as + ``(dim, first_labels, other_labels)`` for the caller to raise (v1) / warn + (legacy). Helper dims (``_term``, ``_factor``) and the concat dim are + excluded; bare dimension indexes are compared, so auxiliary coords (which + ride along the permute) stay §11's job. """ datasets = list(datasets) if len(datasets) < 2: @@ -328,20 +331,17 @@ def conform_merge_dims( out = [datasets[0]] mismatch: tuple[str, Any, Any] | None = None for i in range(1, len(datasets)): - reindexer = {} + permute: dict[Any, Any] = {} for d in shared: ref, idx = indexed[0][d], indexed[i][d] if ref.equals(idx): continue - if ( - not isinstance(idx, pd.MultiIndex) - and len(idx) == len(ref) - and set(idx) == set(ref) - ): - reindexer[d] = ref + positions = idx.get_indexer(ref) if len(idx) == len(ref) else None + if positions is not None and (positions >= 0).all(): + permute[d] = positions elif mismatch is None: mismatch = (str(d), ref.values, idx.values) - out.append(datasets[i].reindex(reindexer) if reindexer else datasets[i]) + out.append(datasets[i].isel(permute) if permute else datasets[i]) return out, mismatch diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index a997b0815..8d4a4d86f 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -844,8 +844,7 @@ def test_quadratic_merge_reordered_aligns(self, m: Model) -> None: result = (x * x) + (y * y) assert list(result.coeffs.coords["e"].values) == ["x", "y", "z"] - @pytest.mark.v1 - def test_reordered_multiindex_raises(self, m: Model) -> None: + def test_reordered_multiindex_aligns_by_tuple(self, m: Model) -> None: mi1 = pd.MultiIndex.from_tuples( [(1, "a"), (1, "b"), (2, "a")], names=["p", "s"] ) @@ -854,10 +853,15 @@ def test_reordered_multiindex_raises(self, m: Model) -> None: ) mi1.name = "snap" mi2.name = "snap" - x = m.add_variables(coords=[mi1], name="x") - y = m.add_variables(coords=[mi2], name="y") - with pytest.raises(ValueError, match="Auxiliary coordinate"): - (1 * x) + (1 * y) + x = m.add_variables(coords=[mi1], name="x") + pd.Series( + [1.0, 2.0, 3.0], index=mi1 + ) + y = m.add_variables(coords=[mi2], name="y") + pd.Series( + [10.0, 20.0, 30.0], index=mi2 + ) + result = x + y + got = dict(zip(map(tuple, result.const.indexes["snap"]), result.const.values)) + assert got == {(1, "a"): 31.0, (1, "b"): 22.0, (2, "a"): 13.0} @pytest.mark.legacy def test_var_plus_var_different_labels_silent( From 42620b9412549c2eb2838a8edee2c37a24e7caf6 Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:23:49 +0200 Subject: [PATCH 5/7] fix(merge): raise dim mismatch before the aux-coord check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A shared-dim label mismatch is the root cause, so report it as a dim conflict rather than letting the §11 aux check fire first — a different stacked MultiIndex otherwise surfaced as its level coords conflicting (the wrong message). Aux conflicts still raise once the dims agree. Adds a routing test. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/expressions.py | 24 +++++++++--------------- test/test_legacy_violations.py | 15 +++++++++++++++ 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/linopy/expressions.py b/linopy/expressions.py index 658f88508..7c48e54ea 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -2659,24 +2659,18 @@ def merge( data = [e.data if isinstance(e, linopy_types) else e for e in exprs] data = [fill_missing_coords(ds, fill_helper_dims=True) for ds in data] - # §8: align shared user dims by label in one pass — a reorder reindexes, - # a differing label set is the mismatch raised/warned below. Runs before - # the §11 aux check so reordered aux coords compare in a consistent order. - mismatch = None + # §8: a dim mismatch is the root cause, so raise it before the §11 aux + # check (else a MultiIndex mismatch reads as a level-coord conflict). if join is None: data, mismatch = conform_merge_dims(data, concat_dim=dim) + if is_v1() and mismatch is not None: + raise ValueError(_shared_dim_mismatch_message(*mismatch)) + if mismatch is not None: # LEGACY: remove at 1.0 + warn_legacy( + _legacy_coord_mismatch_message(f"merge along dim {dim!r}", *mismatch) + ) - # §11: aux-coord conflict fires on every join path — xr.concat(..., - # compat="override") silently drops it (the #295 bug v1 closes). - enforce_aux_conflict(data) - - if is_v1() and mismatch is not None: - raise ValueError(_shared_dim_mismatch_message(*mismatch)) - # LEGACY: remove at 1.0 — warn-on-divergence is the migration signal. - if mismatch is not None: - warn_legacy( - _legacy_coord_mismatch_message(f"merge along dim {dim!r}", *mismatch) - ) + enforce_aux_conflict(data) # §11 if join is not None: override = join == "override" diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index 8d4a4d86f..bdbdbe829 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -863,6 +863,21 @@ def test_reordered_multiindex_aligns_by_tuple(self, m: Model) -> None: got = dict(zip(map(tuple, result.const.indexes["snap"]), result.const.values)) assert got == {(1, "a"): 31.0, (1, "b"): 22.0, (2, "a"): 13.0} + @pytest.mark.v1 + def test_different_multiindex_raises_dim_mismatch(self, m: Model) -> None: + mi1 = pd.MultiIndex.from_tuples( + [(1, "a"), (1, "b"), (2, "a")], names=["p", "s"] + ) + mi2 = pd.MultiIndex.from_tuples( + [(1, "a"), (1, "b"), (3, "c")], names=["p", "s"] + ) + mi1.name = "snap" + mi2.name = "snap" + x = m.add_variables(coords=[mi1], name="x") + y = m.add_variables(coords=[mi2], name="y") + with pytest.raises(ValueError, match="shared dimension 'snap'"): + (1 * x) + (1 * y) + @pytest.mark.legacy def test_var_plus_var_different_labels_silent( self, x: Variable, x_other: Variable From 32fe9c4945a32d144abf58ab60b539b3917805db Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:28:43 +0200 Subject: [PATCH 6/7] test(merge): assert via public .indexes, not .coeffs.coords The reorder tests reached through the internal term storage (.coeffs.coords) for coordinates that the expression exposes directly via .indexes. Co-Authored-By: Claude Opus 4.8 (1M context) --- test/test_legacy_violations.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index bdbdbe829..7130b0d3f 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -809,7 +809,7 @@ def test_var_plus_var_reordered_labels_align(self, m: Model) -> None: a = m.add_variables(coords=[pd.Index(["costs", "penalty"], name="e")], name="a") b = m.add_variables(coords=[pd.Index(["penalty", "costs"], name="e")], name="b") result = (1 * a) + (1 * b) - assert list(result.coeffs.coords["e"].values) == ["costs", "penalty"] + assert list(result.indexes["e"]) == ["costs", "penalty"] def test_reordered_constants_pair_by_label_not_position(self, m: Model) -> None: ea = pd.Index(["costs", "penalty"], name="e") @@ -842,7 +842,7 @@ def test_quadratic_merge_reordered_aligns(self, m: Model) -> None: x = m.add_variables(coords=[ea], name="x") y = m.add_variables(coords=[er], name="y") result = (x * x) + (y * y) - assert list(result.coeffs.coords["e"].values) == ["x", "y", "z"] + assert list(result.indexes["e"]) == ["x", "y", "z"] def test_reordered_multiindex_aligns_by_tuple(self, m: Model) -> None: mi1 = pd.MultiIndex.from_tuples( @@ -860,7 +860,7 @@ def test_reordered_multiindex_aligns_by_tuple(self, m: Model) -> None: [10.0, 20.0, 30.0], index=mi2 ) result = x + y - got = dict(zip(map(tuple, result.const.indexes["snap"]), result.const.values)) + got = dict(zip(map(tuple, result.indexes["snap"]), result.const.values)) assert got == {(1, "a"): 31.0, (1, "b"): 22.0, (2, "a"): 13.0} @pytest.mark.v1 From cde347ce60e0c9f5273f551d3dd0d88817d2feea Mon Sep 17 00:00:00 2001 From: FBumann <117816358+FBumann@users.noreply.github.com> Date: Thu, 4 Jun 2026 19:54:45 +0200 Subject: [PATCH 7/7] fix(merge): make reorder-align v1-only; legacy keeps positional + warns MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per the transitioning contract, legacy must not change: reordering coords on a shared dim was always positional in expression merges (the constant path, by contrast, has always aligned labelled operands by label — that asymmetry is genuine legacy and is what v1 unifies). conform_merge_dims now permutes only under v1; under legacy it leaves the operands positional and the caller warns with a reorder-specific message (v1 would align by label, a different result). Tests: the align cases are now @pytest.mark.v1; added legacy guards for the positional result and the full warning text. Co-Authored-By: Claude Opus 4.8 (1M context) --- linopy/expressions.py | 19 ++++++++---- linopy/semantics.py | 57 ++++++++++++++++++++++------------ test/test_legacy_violations.py | 40 ++++++++++++++++++++++++ 3 files changed, 91 insertions(+), 25 deletions(-) diff --git a/linopy/expressions.py b/linopy/expressions.py index 7c48e54ea..7e46af39b 100644 --- a/linopy/expressions.py +++ b/linopy/expressions.py @@ -90,6 +90,7 @@ ) from linopy.semantics import ( _legacy_coord_mismatch_message, + _legacy_coord_reorder_message, _legacy_nan_rhs_constraint_message, _shared_dim_mismatch_message, absorb_absence, @@ -2659,16 +2660,22 @@ def merge( data = [e.data if isinstance(e, linopy_types) else e for e in exprs] data = [fill_missing_coords(ds, fill_helper_dims=True) for ds in data] - # §8: a dim mismatch is the root cause, so raise it before the §11 aux - # check (else a MultiIndex mismatch reads as a level-coord conflict). + # §8: v1 aligns a reorder by label and raises a dim mismatch (before the + # §11 aux check, else a MultiIndex mismatch reads as a level-coord + # conflict). Legacy keeps positional alignment and only warns. if join is None: - data, mismatch = conform_merge_dims(data, concat_dim=dim) - if is_v1() and mismatch is not None: - raise ValueError(_shared_dim_mismatch_message(*mismatch)) - if mismatch is not None: # LEGACY: remove at 1.0 + data, mismatch, reorder = conform_merge_dims(data, concat_dim=dim) + if is_v1(): + if mismatch is not None: + raise ValueError(_shared_dim_mismatch_message(*mismatch)) + elif mismatch is not None: # LEGACY: remove at 1.0 warn_legacy( _legacy_coord_mismatch_message(f"merge along dim {dim!r}", *mismatch) ) + elif reorder is not None: # LEGACY: remove at 1.0 + warn_legacy( + _legacy_coord_reorder_message(f"merge along dim {dim!r}", *reorder) + ) enforce_aux_conflict(data) # §11 diff --git a/linopy/semantics.py b/linopy/semantics.py index e498fd2a4..38e066956 100644 --- a/linopy/semantics.py +++ b/linopy/semantics.py @@ -152,6 +152,19 @@ def _legacy_coord_mismatch_message( ) +def _legacy_coord_reorder_message(context: str, dim: str, left: Any, right: Any) -> str: + """Same labels, different order — aligned positionally by legacy; v1 reindexes.""" + return ( + f"Coordinate order mismatch in {context} aligned positionally by legacy." + " Under v1 the same labels in a different order align by label (a" + " reindex), giving a different result." + f"\n Dim: {dim!r}: left={_short_repr(left)}, right={_short_repr(right)}" + "\n Resolve: `.sel(...)` / `.reindex(...)` to align" + "\n `.assign_coords(...)` to relabel one side" + "\n or pass an explicit `join=` argument." + _OPT_IN_HINT + ) + + def _legacy_aux_conflict_message(name: str, left: Any, right: Any, kind: str) -> str: """ Conflicting aux coord silently dropped by xarray under legacy. @@ -301,24 +314,25 @@ def first_mismatched_dim(a: DataArray, b: DataArray) -> tuple[str, Any, Any] | N def conform_merge_dims( datasets: Sequence[Dataset], concat_dim: str -) -> tuple[list[Dataset], tuple[str, Any, Any] | None]: +) -> tuple[list[Dataset], tuple[str, Any, Any] | None, tuple[str, Any, Any] | None]: """ - Align shared user dims for a merge, in a single pass over the operands. - - §8 aligns by label, not position: a shared user dim whose labels are the - first operand's reordered (same set) is permuted to that order. The permute - uses positional ``isel`` rather than ``reindex`` so it works uniformly for a - plain index and a stacked MultiIndex's tuples (``reindex`` cannot reorder a - MultiIndex by tuple); ``get_indexer`` also doubles as the same-set test. A - dim whose label set differs is a real mismatch, returned as - ``(dim, first_labels, other_labels)`` for the caller to raise (v1) / warn - (legacy). Helper dims (``_term``, ``_factor``) and the concat dim are - excluded; bare dimension indexes are compared, so auxiliary coords (which - ride along the permute) stay §11's job. + Inspect shared user dims for a merge, in a single pass over the operands. + + Returns ``(data, mismatch, reorder)``. A shared user dim whose labels are + the first operand's in a different order (same set, including a stacked + MultiIndex's tuples) is a *reorder*; one whose label set differs is a + *mismatch* — each reported as ``(dim, first_labels, other_labels)`` (first + found). Under v1, reorders are aligned to the first operand's order in the + returned data (via positional ``isel`` — ``reindex`` cannot reorder a + MultiIndex by tuple) and ``reorder`` is ``None``; the caller raises on + ``mismatch``. Under legacy, nothing is aligned and the caller warns: + ``reorder`` (v1 would align by label) or ``mismatch`` (v1 would raise). + Helper dims (``_term``, ``_factor``) and the concat dim are excluded; bare + dimension indexes are compared, so auxiliary coords stay §11's job. """ datasets = list(datasets) if len(datasets) < 2: - return datasets, None + return datasets, None, None skip = set(HELPER_DIMS) | {concat_dim} indexed = [ {k: d.indexes[k] for k in d.dims if k not in skip and k in d.indexes} @@ -326,23 +340,28 @@ def conform_merge_dims( ] shared = set.intersection(*(set(p) for p in indexed)) if not shared: - return datasets, None + return datasets, None, None + permute = is_v1() out = [datasets[0]] mismatch: tuple[str, Any, Any] | None = None + reorder: tuple[str, Any, Any] | None = None for i in range(1, len(datasets)): - permute: dict[Any, Any] = {} + plan: dict[Any, Any] = {} for d in shared: ref, idx = indexed[0][d], indexed[i][d] if ref.equals(idx): continue positions = idx.get_indexer(ref) if len(idx) == len(ref) else None if positions is not None and (positions >= 0).all(): - permute[d] = positions + if permute: + plan[d] = positions + elif reorder is None: + reorder = (str(d), ref.values, idx.values) elif mismatch is None: mismatch = (str(d), ref.values, idx.values) - out.append(datasets[i].isel(permute) if permute else datasets[i]) - return out, mismatch + out.append(datasets[i].isel(plan) if plan else datasets[i]) + return out, mismatch, reorder def conflicting_aux_coord( diff --git a/test/test_legacy_violations.py b/test/test_legacy_violations.py index 7130b0d3f..c4f0ecebe 100644 --- a/test/test_legacy_violations.py +++ b/test/test_legacy_violations.py @@ -805,12 +805,14 @@ def test_var_plus_var_broadcast_non_shared_dim_works( result = a + b assert set(result.coord_dims) == {"time", "scenario"} + @pytest.mark.v1 def test_var_plus_var_reordered_labels_align(self, m: Model) -> None: a = m.add_variables(coords=[pd.Index(["costs", "penalty"], name="e")], name="a") b = m.add_variables(coords=[pd.Index(["penalty", "costs"], name="e")], name="b") result = (1 * a) + (1 * b) assert list(result.indexes["e"]) == ["costs", "penalty"] + @pytest.mark.v1 def test_reordered_constants_pair_by_label_not_position(self, m: Model) -> None: ea = pd.Index(["costs", "penalty"], name="e") eb = pd.Index(["penalty", "costs"], name="e") @@ -820,6 +822,7 @@ def test_reordered_constants_pair_by_label_not_position(self, m: Model) -> None: assert float(result.const.sel(e="costs")) == 102.0 assert float(result.const.sel(e="penalty")) == 201.0 + @pytest.mark.v1 def test_multi_operand_merge_reordered_pairs_by_label(self, m: Model) -> None: ea = pd.Index(["x", "y", "z"], name="e") er = pd.Index(["z", "y", "x"], name="e") @@ -836,6 +839,7 @@ def test_multi_operand_merge_reordered_pairs_by_label(self, m: Model) -> None: assert float(result.const.sel(e="x")) == 131.0 assert float(result.const.sel(e="z")) == 313.0 + @pytest.mark.v1 def test_quadratic_merge_reordered_aligns(self, m: Model) -> None: ea = pd.Index(["x", "y", "z"], name="e") er = pd.Index(["z", "y", "x"], name="e") @@ -844,6 +848,7 @@ def test_quadratic_merge_reordered_aligns(self, m: Model) -> None: result = (x * x) + (y * y) assert list(result.indexes["e"]) == ["x", "y", "z"] + @pytest.mark.v1 def test_reordered_multiindex_aligns_by_tuple(self, m: Model) -> None: mi1 = pd.MultiIndex.from_tuples( [(1, "a"), (1, "b"), (2, "a")], names=["p", "s"] @@ -863,6 +868,41 @@ def test_reordered_multiindex_aligns_by_tuple(self, m: Model) -> None: got = dict(zip(map(tuple, result.indexes["snap"]), result.const.values)) assert got == {(1, "a"): 31.0, (1, "b"): 22.0, (2, "a"): 13.0} + @pytest.mark.legacy + def test_reordered_merge_positional_legacy(self, m: Model) -> None: + ea = pd.Index(["costs", "penalty"], name="e") + eb = pd.Index(["penalty", "costs"], name="e") + a = m.add_variables(coords=[ea], name="a") + pd.Series([100.0, 200.0], index=ea) + b = m.add_variables(coords=[eb], name="b") + pd.Series([1.0, 2.0], index=eb) + result = a + b + assert float(result.const.sel(e="costs")) == 101.0 + assert float(result.const.sel(e="penalty")) == 202.0 + + @pytest.mark.legacy + def test_reordered_merge_warns_legacy(self, m: Model, unsilenced: None) -> None: + ea = pd.Index(["costs", "penalty"], name="e") + eb = pd.Index(["penalty", "costs"], name="e") + a = m.add_variables(coords=[ea], name="a") + b = m.add_variables(coords=[eb], name="b") + with pytest.warns(LinopySemanticsWarning) as record: + (1 * a) + (1 * b) + msg = next( + str(w.message) for w in record if w.category is LinopySemanticsWarning + ) + assert msg == ( + "Coordinate order mismatch in merge along dim '_term' aligned " + "positionally by legacy. Under v1 the same labels in a different " + "order align by label (a reindex), giving a different result." + "\n Dim: 'e': left=['costs', 'penalty'], " + "right=['penalty', 'costs']" + "\n Resolve: `.sel(...)` / `.reindex(...)` to align" + "\n `.assign_coords(...)` to relabel one side" + "\n or pass an explicit `join=` argument." + "\n Opt in: linopy.options['semantics'] = 'v1'" + "\n Silence: warnings.filterwarnings('ignore', " + "category=LinopySemanticsWarning)" + ) + @pytest.mark.v1 def test_different_multiindex_raises_dim_mismatch(self, m: Model) -> None: mi1 = pd.MultiIndex.from_tuples(