From 89b5714bd894afabdf9a8363c8903a05ba84a992 Mon Sep 17 00:00:00 2001 From: Jared Yu Date: Fri, 31 Jul 2026 22:57:01 -0700 Subject: [PATCH 1/2] IO: Fix Windows drive letters misidentified as URI schemes by urlparse On Windows, Python's urlparse treats paths like 'C:\Users\...' as having scheme='c', causing 'Unrecognized filesystem type in URI: c' errors. Uses os.path.splitdrive to detect Windows drive-letter paths before urlparse is called, avoiding the misparse entirely. splitdrive is inherently platform-aware (no-op on Linux/macOS) so no sys.platform check is needed. Fixes all three parse sites (_infer_file_io_from_scheme, PyArrowFileIO.parse_location, FsspecFileIO._get_fs_from_uri) and includes platform-conditional tests. Related: #2477, #1005 --- pyiceberg/io/__init__.py | 26 ++++++++++++++++++++++---- pyiceberg/io/fsspec.py | 11 +++++++---- pyiceberg/io/pyarrow.py | 10 +++++++++- tests/io/test_io.py | 22 ++++++++++++++++++++++ tests/io/test_pyarrow.py | 12 ++++++++++++ 5 files changed, 72 insertions(+), 9 deletions(-) diff --git a/pyiceberg/io/__init__.py b/pyiceberg/io/__init__.py index 7dbc651214..6ff1674740 100644 --- a/pyiceberg/io/__init__.py +++ b/pyiceberg/io/__init__.py @@ -27,6 +27,7 @@ import importlib import logging +import os import warnings from abc import ABC, abstractmethod from io import SEEK_SET @@ -41,6 +42,18 @@ logger = logging.getLogger(__name__) + +def _is_local_path(path: str) -> bool: + r"""Check if a path is a local filesystem path rather than a URI. + + Uses os.path.splitdrive to detect Windows drive letters (e.g. 'C:\\...') + in a platform-aware way. On non-Windows systems, splitdrive always returns + an empty drive component, so this only triggers on Windows. + """ + drive, _ = os.path.splitdrive(path) + return drive != "" + + AWS_PROFILE_NAME = "client.profile-name" AWS_REGION = "client.region" AWS_ACCESS_KEY_ID = "client.access-key-id" @@ -335,14 +348,19 @@ def _import_file_io(io_impl: str, properties: Properties) -> FileIO | None: def _infer_file_io_from_scheme(path: str, properties: Properties) -> FileIO | None: - parsed_url = urlparse(path) - if parsed_url.scheme: - if file_ios := SCHEMA_TO_FILE_IO.get(parsed_url.scheme): + if _is_local_path(path): + scheme = "file" + else: + parsed_url = urlparse(path) + scheme = parsed_url.scheme + + if scheme: + if file_ios := SCHEMA_TO_FILE_IO.get(scheme): for file_io_path in file_ios: if file_io := _import_file_io(file_io_path, properties): return file_io else: - warnings.warn(f"No preferred file implementation for scheme: {parsed_url.scheme}", stacklevel=2) + warnings.warn(f"No preferred file implementation for scheme: {scheme}", stacklevel=2) return None diff --git a/pyiceberg/io/fsspec.py b/pyiceberg/io/fsspec.py index 7749268ff5..b28ffd0699 100644 --- a/pyiceberg/io/fsspec.py +++ b/pyiceberg/io/fsspec.py @@ -89,6 +89,7 @@ InputStream, OutputFile, OutputStream, + _is_local_path, ) from pyiceberg.typedef import Properties from pyiceberg.types import strtobool @@ -443,7 +444,7 @@ def new_input(self, location: str) -> FsspecInputFile: FsspecInputFile: An FsspecInputFile instance for the given location. """ uri = urlparse(location) - fs = self._get_fs_from_uri(uri) + fs = self._get_fs_from_uri(uri, location) return FsspecInputFile(location=location, fs=fs) @override @@ -457,7 +458,7 @@ def new_output(self, location: str) -> FsspecOutputFile: FsspecOutputFile: An FsspecOutputFile instance for the given location. """ uri = urlparse(location) - fs = self._get_fs_from_uri(uri) + fs = self._get_fs_from_uri(uri, location) return FsspecOutputFile(location=location, fs=fs) @override @@ -475,11 +476,13 @@ def delete(self, location: str | InputFile | OutputFile) -> None: str_location = location uri = urlparse(str_location) - fs = self._get_fs_from_uri(uri) + fs = self._get_fs_from_uri(uri, str_location) fs.rm(str_location) - def _get_fs_from_uri(self, uri: "ParseResult") -> AbstractFileSystem: + def _get_fs_from_uri(self, uri: "ParseResult", location: str = "") -> AbstractFileSystem: """Get a filesystem from a parsed URI, using hostname for ADLS account resolution.""" + if _is_local_path(location): + return self.get_fs("file") if uri.scheme in _ADLS_SCHEMES: return self.get_fs(uri.scheme, uri.hostname) return self.get_fs(uri.scheme) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index 29ede3ce9e..c36f1639d9 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -121,6 +121,7 @@ InputStream, OutputFile, OutputStream, + _is_local_path, ) from pyiceberg.io.fileformat import DataFileStatistics as DataFileStatistics from pyiceberg.io.fileformat import FileFormatFactory, FileFormatModel, FileFormatWriter @@ -401,10 +402,17 @@ def __init__(self, properties: Properties = EMPTY_DICT): @staticmethod def parse_location(location: str, properties: Properties = EMPTY_DICT) -> tuple[str, str, str]: - """Return (scheme, netloc, path) for the given location. + r"""Return (scheme, netloc, path) for the given location. Uses DEFAULT_SCHEME and DEFAULT_NETLOC if scheme/netloc are missing. + On Windows, paths with drive letters (e.g. 'C:\\...') are treated as + local file paths rather than URIs. """ + if _is_local_path(location): + default_scheme = properties.get("DEFAULT_SCHEME", "file") + default_netloc = properties.get("DEFAULT_NETLOC", "") + return default_scheme, default_netloc, os.path.abspath(location) + uri = urlparse(location) if not uri.scheme: diff --git a/tests/io/test_io.py b/tests/io/test_io.py index d9bee33f8b..a3dd9765c8 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -17,6 +17,7 @@ import os import pickle +import sys import tempfile from typing import Any @@ -27,6 +28,7 @@ PY_IO_IMPL, _import_file_io, _infer_file_io_from_scheme, + _is_local_path, load_file_io, ) from pyiceberg.io.pyarrow import PyArrowFileIO @@ -339,3 +341,23 @@ def test_infer_file_io_from_schema_unknown() -> None: _infer_file_io_from_scheme("unknown://bucket/path/", {}) assert str(w[0].message) == "No preferred file implementation for scheme: unknown" + + +@pytest.mark.parametrize( + "path", + ["s3://bucket/key", "hdfs://cluster/path", "gs://bucket/obj", "/tmp/foo", ""], +) +def test_is_local_path_false_for_uris_and_posix(path: str) -> None: + assert _is_local_path(path) is False + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +@pytest.mark.parametrize("path", [r"C:\Users\test", r"D:\data\iceberg", "E:/warehouse"]) +def test_is_local_path_true_on_windows(path: str) -> None: + assert _is_local_path(path) is True + + +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +def test_infer_file_io_from_scheme_windows_path() -> None: + result = _infer_file_io_from_scheme(r"C:\Users\test\warehouse", {}) + assert isinstance(result, PyArrowFileIO) diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index fef33a25c7..407f925ed2 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -17,6 +17,7 @@ # pylint: disable=protected-access,unused-argument,redefined-outer-name import logging import os +import sys import tempfile import uuid import warnings @@ -2326,6 +2327,17 @@ def check_results(location: str, expected_schema: str, expected_netloc: str, exp check_results("/root/tmp/foo.txt", "file", "", "/root/tmp/foo.txt") +@pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") +def test_parse_location_windows_drive_letter() -> None: + """Windows drive letters should be treated as local file paths, not URL schemes.""" + for drive in ("C", "D", "c", "d"): + path = f"{drive}:\\Users\\test\\file.avro" + scheme, netloc, result_path = PyArrowFileIO.parse_location(path) + assert scheme == "file" + assert netloc == "" + assert result_path == os.path.abspath(path) + + def test_make_compatible_name() -> None: assert make_compatible_name("label/abc") == "label_x2Fabc" assert make_compatible_name("label?abc") == "label_x3Fabc" From c55b2ac4370819f94f798b6e4295f623a48bfc61 Mon Sep 17 00:00:00 2001 From: Jared Yu Date: Sun, 2 Aug 2026 11:28:54 -0700 Subject: [PATCH 2/2] CI: Add Windows unit test job and fix test compatibility Adds a windows-latest job to python-ci.yml and fixes platform assumptions in tests that prevented the suite from passing on Windows. Test fixes: - Reduce RANDOM_LENGTH to 8 (Windows MAX_PATH is 260 chars) - Use warehouse.as_posix() in SQLite URIs - Use context managers for file handles (Windows file locking) - Use os.path.abspath() in path assertions - Use raw paths instead of file: URIs in pyarrow fixtures - Skip Rich box-rendering tests on Windows (different terminal chars) - Skip Kerberos tests on Windows (puresasl C lib unavailable) - Relax error message assertion ([WinError 2] vs [Errno 2]) - Normalize backslashes in Hive path assertions Result: 3795+ tests pass on Windows with 0 failures. Depends on #3721. Closes #2477. --- .github/workflows/python-ci.yml | 24 +++++++++++ tests/catalog/test_hive.py | 8 +++- tests/catalog/test_sql.py | 20 ++++----- tests/conftest.py | 12 +++--- tests/io/test_io.py | 16 +++---- tests/io/test_pyarrow.py | 74 +++++++++++++++++---------------- tests/utils/test_manifest.py | 2 +- 7 files changed, 94 insertions(+), 62 deletions(-) diff --git a/.github/workflows/python-ci.yml b/.github/workflows/python-ci.yml index 5eadc98ad7..678cad603b 100644 --- a/.github/workflows/python-ci.yml +++ b/.github/workflows/python-ci.yml @@ -79,6 +79,30 @@ jobs: - name: Generate coverage report (85%) # Coverage threshold should only increase over time — never decrease it! run: COVERAGE_FAIL_UNDER=85 make coverage-report + windows-unit-test: + runs-on: windows-latest + timeout-minutes: 15 + strategy: + fail-fast: true + matrix: + python: ['3.12'] + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 + with: + python-version: ${{ matrix.python }} + - name: Install UV + uses: astral-sh/setup-uv@fac544c07dec837d0ccb6301d7b5580bf5edae39 # v8.2.0 + with: + enable-cache: true + - name: Install + run: uv sync --all-extras --no-extra hive-kerberos + - name: Run unit tests + run: uv run python -m pytest tests/ -m "(unmarked or parametrize) and not integration" --ignore=tests/integration -v -x + cibw-dev-env-smoke-test: runs-on: ubuntu-latest steps: diff --git a/tests/catalog/test_hive.py b/tests/catalog/test_hive.py index 09bb5ab920..f594aa876e 100644 --- a/tests/catalog/test_hive.py +++ b/tests/catalog/test_hive.py @@ -18,6 +18,7 @@ import base64 import copy import struct +import sys import threading import uuid from collections.abc import Generator @@ -300,7 +301,7 @@ def test_create_table( # it to construct the assert_called_with metadata_location: str = called_hive_table.parameters["metadata_location"] assert metadata_location.endswith(".metadata.json") - assert "/database/table/metadata/" in metadata_location + assert "/database/table/metadata/" in metadata_location.replace("\\", "/") catalog._client.__enter__().create_table.assert_called_with( HiveTable( tableName="table", @@ -481,7 +482,7 @@ def test_create_table_with_given_location_removes_trailing_slash( # it to construct the assert_called_with metadata_location: str = called_hive_table.parameters["metadata_location"] assert metadata_location.endswith(".metadata.json") - assert "/database/table-given-location/metadata/" in metadata_location + assert "/database/table-given-location/metadata/" in metadata_location.replace("\\", "/") catalog._client.__enter__().create_table.assert_called_with( HiveTable( tableName="table", @@ -1369,6 +1370,7 @@ def test_create_hive_client_failure() -> None: assert mock_hive_client.call_count == 2 +@pytest.mark.skipif(sys.platform == "win32", reason="Kerberos/puresasl not available on Windows") def test_create_hive_client_with_kerberos( kerberized_hive_metastore_fake_url: str, ) -> None: @@ -1381,6 +1383,7 @@ def test_create_hive_client_with_kerberos( assert client is not None +@pytest.mark.skipif(sys.platform == "win32", reason="Kerberos/puresasl not available on Windows") def test_create_hive_client_with_kerberos_using_context_manager( kerberized_hive_metastore_fake_url: str, ) -> None: @@ -1412,6 +1415,7 @@ def test_create_hive_client_with_kerberos_using_context_manager( assert open_client._iprot.trans.isOpen() +@pytest.mark.skipif(sys.platform == "win32", reason="Kerberos/puresasl not available on Windows") def test_kerberized_client_uses_fresh_transport_on_reuse( kerberized_hive_metastore_fake_url: str, ) -> None: diff --git a/tests/catalog/test_sql.py b/tests/catalog/test_sql.py index 1eabc56a9e..4ba63d7837 100644 --- a/tests/catalog/test_sql.py +++ b/tests/catalog/test_sql.py @@ -61,7 +61,7 @@ def catalog_memory(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog, @pytest.fixture(scope="module") def catalog_sqlite(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog, None, None]: props = { - "uri": f"sqlite:////{warehouse}/sql-catalog", + "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog", "warehouse": f"file://{warehouse}", } catalog = SqlCatalog(catalog_name, **props) @@ -72,7 +72,7 @@ def catalog_sqlite(catalog_name: str, warehouse: Path) -> Generator[SqlCatalog, @pytest.fixture(scope="module") def catalog_uri(warehouse: Path) -> str: - return f"sqlite:////{warehouse}/sql-catalog" + return f"sqlite:///{warehouse.as_posix()}/sql-catalog" @pytest.fixture(scope="module") @@ -96,7 +96,7 @@ def test_creation_with_echo_parameter(catalog_name: str, warehouse: Path) -> Non for echo_param, expected_echo_value in test_cases: props = { - "uri": f"sqlite:////{warehouse}/sql-catalog", + "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog", "warehouse": f"file://{warehouse}", } # None is for default value @@ -119,7 +119,7 @@ def test_creation_with_pool_pre_ping_parameter(catalog_name: str, warehouse: Pat for pool_pre_ping_param, expected_pool_pre_ping_value in test_cases: props = { - "uri": f"sqlite:////{warehouse}/sql-catalog", + "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog", "warehouse": f"file://{warehouse}", } # None is for default value @@ -139,7 +139,7 @@ def test_creation_from_impl(catalog_name: str, warehouse: Path) -> None: catalog_name, **{ "py-catalog-impl": "pyiceberg.catalog.sql.SqlCatalog", - "uri": f"sqlite:////{warehouse}/sql-catalog", + "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog", "warehouse": f"file://{warehouse}", }, ), @@ -268,7 +268,7 @@ def get_columns(engine: Engine) -> set[str]: def test_adds_iceberg_type_column_to_old_schema(warehouse: Path) -> None: - uri = f"sqlite:////{warehouse}/test-migration-add-col" + uri = f"sqlite:///{warehouse.as_posix()}/test-migration-add-col" engine = _create_v0_db(uri) # Verify the column does not exist in the old schema @@ -334,7 +334,7 @@ def test_list_tables_filters_by_iceberg_type(warehouse: Path) -> None: def test_migration_to_v1_with_property_set(warehouse: Path) -> None: - uri = f"sqlite:////{warehouse}/test-v1-migrate" + uri = f"sqlite:///{warehouse.as_posix()}/test-v1-migrate" engine = _create_v0_db(uri) assert "iceberg_type" not in get_columns(engine) @@ -361,7 +361,7 @@ def test_invalid_schema_version_raises(warehouse: Path) -> None: def test_list_tables_works_on_v0_schema(warehouse: Path) -> None: """list_tables should work on V0 schemas without iceberg_type column.""" - uri = f"sqlite:////{warehouse}/test-v0-list" + uri = f"sqlite:///{warehouse.as_posix()}/test-v0-list" _create_v0_db(uri) catalog = SqlCatalog( @@ -441,7 +441,7 @@ def _make_v0_catalog(uri: str, warehouse: Path) -> SqlCatalog: def test_namespace_exists_on_v0_schema(warehouse: Path) -> None: """namespace_exists should not fail on V0 schema (no iceberg_type column).""" - catalog = _make_v0_catalog(f"sqlite:////{warehouse}/test-v0-ns-exists", warehouse) + catalog = _make_v0_catalog(f"sqlite:///{warehouse.as_posix()}/test-v0-ns-exists", warehouse) catalog.create_namespace("ns") assert catalog.namespace_exists("ns") assert not catalog.namespace_exists("missing") @@ -449,7 +449,7 @@ def test_namespace_exists_on_v0_schema(warehouse: Path) -> None: def test_create_and_load_table_on_v0_schema(warehouse: Path) -> None: """create_table and load_table should work on V0 schema without iceberg_type column.""" - catalog = _make_v0_catalog(f"sqlite:////{warehouse}/test-v0-create", warehouse) + catalog = _make_v0_catalog(f"sqlite:///{warehouse.as_posix()}/test-v0-create", warehouse) catalog.create_namespace("ns") schema = Schema(NestedField(1, "id", StringType(), required=True)) tbl = catalog.create_table(("ns", "tbl"), schema) diff --git a/tests/conftest.py b/tests/conftest.py index 2b1f2366ed..51a663434f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2388,7 +2388,7 @@ def empty_home_dir_path(tmp_path_factory: pytest.TempPathFactory) -> str: return home_path -RANDOM_LENGTH = 20 +RANDOM_LENGTH = 8 # Keep short to stay within Windows MAX_PATH (260 chars) NUM_TABLES = 2 @@ -2445,15 +2445,15 @@ def hierarchical_namespace_list(hierarchical_namespace_name: str) -> list[str]: BUCKET_NAME = "test_bucket" TABLE_METADATA_LOCATION_REGEX = re.compile( - r"""s3://test_bucket/my_iceberg_database-[a-z]{20}.db/ - my_iceberg_table-[a-z]{20}/metadata/ + r"""s3://test_bucket/my_iceberg_database-[a-z]{8}.db/ + my_iceberg_table-[a-z]{8}/metadata/ [0-9]{5}-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""", re.X, ) BQ_TABLE_METADATA_LOCATION_REGEX = re.compile( - r"""gs://alexstephen-test-bq-bucket/my_iceberg_database_[a-z]{20}.db/ - my_iceberg_table-[a-z]{20}/metadata/ + r"""gs://alexstephen-test-bq-bucket/my_iceberg_database_[a-z]{8}.db/ + my_iceberg_table-[a-z]{8}/metadata/ [0-9]{5}-[a-f0-9]{8}-?[a-f0-9]{4}-?4[a-f0-9]{3}-?[89ab][a-f0-9]{3}-?[a-f0-9]{12}.metadata.json""", re.X, ) @@ -3138,7 +3138,7 @@ def _create_sql_without_rowcount_catalog(name: str, warehouse: Path) -> Catalog: from pyiceberg.catalog.sql import SqlCatalog props = { - "uri": f"sqlite:////{warehouse}/sql-catalog", + "uri": f"sqlite:///{warehouse.as_posix()}/sql-catalog", "warehouse": f"file://{warehouse}", } catalog = SqlCatalog(name, **props) diff --git a/tests/io/test_io.py b/tests/io/test_io.py index a3dd9765c8..51aa86c67f 100644 --- a/tests/io/test_io.py +++ b/tests/io/test_io.py @@ -49,10 +49,10 @@ def test_custom_local_input_file() -> None: input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file - f = input_file.open() - data = f.read() - assert data == b"foo" - assert len(input_file) == 3 + with input_file.open() as f: + data = f.read() + assert data == b"foo" + assert len(input_file) == 3 def test_custom_local_output_file() -> None: @@ -87,10 +87,10 @@ def test_pickled_pyarrow_round_trip() -> None: f.write(b"foo") input_file = deserialized_file_io.new_input(location=f"{absolute_file_location}") - f = input_file.open() - data = f.read() - assert data == b"foo" - assert len(input_file) == 3 + with input_file.open() as f: + data = f.read() + assert data == b"foo" + assert len(input_file) == 3 deserialized_file_io.delete(location=f"{absolute_file_location}") diff --git a/tests/io/test_pyarrow.py b/tests/io/test_pyarrow.py index 407f925ed2..b31c18949b 100644 --- a/tests/io/test_pyarrow.py +++ b/tests/io/test_pyarrow.py @@ -163,14 +163,14 @@ def test_pyarrow_input_file() -> None: input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file - r = input_file.open(seekable=False) - assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol - data = r.read() - assert data == b"foo" - assert len(input_file) == 3 - with pytest.raises(OSError) as exc_info: - r.seek(0, 0) - assert "only valid on seekable files" in str(exc_info.value) + with input_file.open(seekable=False) as r: + assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol + data = r.read() + assert data == b"foo" + assert len(input_file) == 3 + with pytest.raises(OSError) as exc_info: + r.seek(0, 0) + assert "only valid on seekable files" in str(exc_info.value) def test_pyarrow_input_file_seekable() -> None: @@ -189,15 +189,15 @@ def test_pyarrow_input_file_seekable() -> None: input_file = PyArrowFileIO().new_input(location=f"{absolute_file_location}") # Test opening and reading the file - r = input_file.open(seekable=True) - assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol - data = r.read() - assert data == b"foo" - assert len(input_file) == 3 - r.seek(0, 0) - data = r.read() - assert data == b"foo" - assert len(input_file) == 3 + with input_file.open(seekable=True) as r: + assert isinstance(r, InputStream) # Test that the file object abides by the InputStream protocol + data = r.read() + assert data == b"foo" + assert len(input_file) == 3 + r.seek(0, 0) + data = r.read() + assert data == b"foo" + assert len(input_file) == 3 def test_pyarrow_output_file() -> None: @@ -211,9 +211,9 @@ def test_pyarrow_output_file() -> None: output_file = PyArrowFileIO().new_output(location=f"{absolute_file_location}") # Create the output file and write to it - f = output_file.create() - assert isinstance(f, OutputStream) # Test that the file object abides by the OutputStream protocol - f.write(b"foo") + with output_file.create() as output_stream: + assert isinstance(output_stream, OutputStream) # Test that the file object abides by the OutputStream protocol + output_stream.write(b"foo") # Confirm that bytes were written with open(file_location, "rb") as f: @@ -284,7 +284,7 @@ def test_raise_on_opening_a_local_file_not_found() -> None: with pytest.raises(FileNotFoundError) as exc_info: f.open() - assert "[Errno 2] Failed to open local file" in str(exc_info.value) + assert "Failed to open local file" in str(exc_info.value) def test_raise_on_opening_an_s3_file_no_permission() -> None: @@ -1003,7 +1003,7 @@ def _write_table_to_data_file(filepath: str, schema: pa.Schema, table: pa.Table) def file_int(schema_int: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_int, metadata={ICEBERG_SCHEMA: bytes(schema_int.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/a.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema) + f"{tmpdir}/a.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema) ) @@ -1011,7 +1011,7 @@ def file_int(schema_int: Schema, tmpdir: str) -> str: def file_int_str(schema_int_str: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_int_str, metadata={ICEBERG_SCHEMA: bytes(schema_int_str.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/a.parquet", + f"{tmpdir}/a.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2]), pa.array(["0", "1", "2"])], schema=pyarrow_schema), ) @@ -1021,7 +1021,7 @@ def file_int_str(schema_int_str: Schema, tmpdir: str) -> str: def file_string(schema_str: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_str, metadata={ICEBERG_SCHEMA: bytes(schema_str.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/b.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array(["0", "1", "2"])], schema=pyarrow_schema) + f"{tmpdir}/b.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array(["0", "1", "2"])], schema=pyarrow_schema) ) @@ -1029,7 +1029,7 @@ def file_string(schema_str: Schema, tmpdir: str) -> str: def file_long(schema_long: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_long, metadata={ICEBERG_SCHEMA: bytes(schema_long.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/c.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema) + f"{tmpdir}/c.parquet", pyarrow_schema, pa.Table.from_arrays([pa.array([0, 1, 2])], schema=pyarrow_schema) ) @@ -1037,7 +1037,7 @@ def file_long(schema_long: Schema, tmpdir: str) -> str: def file_struct(schema_struct: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_struct, metadata={ICEBERG_SCHEMA: bytes(schema_struct.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/d.parquet", + f"{tmpdir}/d.parquet", pyarrow_schema, pa.Table.from_pylist( [ @@ -1054,7 +1054,7 @@ def file_struct(schema_struct: Schema, tmpdir: str) -> str: def file_list(schema_list: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_list, metadata={ICEBERG_SCHEMA: bytes(schema_list.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/e.parquet", + f"{tmpdir}/e.parquet", pyarrow_schema, pa.Table.from_pylist( [ @@ -1073,7 +1073,7 @@ def file_list_of_structs(schema_list_of_structs: Schema, tmpdir: str) -> str: schema_list_of_structs, metadata={ICEBERG_SCHEMA: bytes(schema_list_of_structs.model_dump_json(), UTF8)} ) return _write_table_to_file( - f"file:{tmpdir}/e.parquet", + f"{tmpdir}/e.parquet", pyarrow_schema, pa.Table.from_pylist( [ @@ -1092,7 +1092,7 @@ def file_map_of_structs(schema_map_of_structs: Schema, tmpdir: str) -> str: schema_map_of_structs, metadata={ICEBERG_SCHEMA: bytes(schema_map_of_structs.model_dump_json(), UTF8)} ) return _write_table_to_file( - f"file:{tmpdir}/e.parquet", + f"{tmpdir}/e.parquet", pyarrow_schema, pa.Table.from_pylist( [ @@ -1109,7 +1109,7 @@ def file_map_of_structs(schema_map_of_structs: Schema, tmpdir: str) -> str: def file_map(schema_map: Schema, tmpdir: str) -> str: pyarrow_schema = schema_to_pyarrow(schema_map, metadata={ICEBERG_SCHEMA: bytes(schema_map.model_dump_json(), UTF8)}) return _write_table_to_file( - f"file:{tmpdir}/e.parquet", + f"{tmpdir}/e.parquet", pyarrow_schema, pa.Table.from_pylist( [ @@ -2323,8 +2323,8 @@ def check_results(location: str, expected_schema: str, expected_netloc: str, exp check_results("hdfs://127.0.0.1/root/foo.txt", "hdfs", "127.0.0.1", "/root/foo.txt") check_results("hdfs://clusterA/root/foo.txt", "hdfs", "clusterA", "/root/foo.txt") - check_results("/root/foo.txt", "file", "", "/root/foo.txt") - check_results("/root/tmp/foo.txt", "file", "", "/root/tmp/foo.txt") + check_results("/root/foo.txt", "file", "", os.path.abspath("/root/foo.txt")) + check_results("/root/tmp/foo.txt", "file", "", os.path.abspath("/root/tmp/foo.txt")) @pytest.mark.skipif(sys.platform != "win32", reason="Windows-only behavior") @@ -2492,6 +2492,7 @@ def tracking_iter() -> Iterator[pa.RecordBatch]: assert len(consumed) == arrow_table_with_null.num_rows +@pytest.mark.skipif(sys.platform == "win32", reason="Rich renders different box characters on Windows terminals") def test_schema_mismatch_type(table_schema_simple: Schema) -> None: other_schema = pa.schema( ( @@ -2515,6 +2516,7 @@ def test_schema_mismatch_type(table_schema_simple: Schema) -> None: _check_pyarrow_schema_compatible(table_schema_simple, other_schema) +@pytest.mark.skipif(sys.platform == "win32", reason="Rich renders different box characters on Windows terminals") def test_schema_mismatch_nullability(table_schema_simple: Schema) -> None: other_schema = pa.schema( ( @@ -2553,6 +2555,7 @@ def test_schema_compatible_nullability_diff(table_schema_simple: Schema) -> None pytest.fail("Unexpected Exception raised when calling `_check_pyarrow_schema_compatible`") +@pytest.mark.skipif(sys.platform == "win32", reason="Rich renders different box characters on Windows terminals") def test_schema_mismatch_missing_field(table_schema_simple: Schema) -> None: other_schema = pa.schema( ( @@ -2595,6 +2598,7 @@ def test_schema_compatible_missing_nullable_field_nested(table_schema_nested: Sc pytest.fail("Unexpected Exception raised when calling `_check_pyarrow_schema_compatible`") +@pytest.mark.skipif(sys.platform == "win32", reason="Rich renders different box characters on Windows terminals") def test_schema_mismatch_missing_required_field_nested(table_schema_nested: Schema) -> None: other_schema = table_schema_nested.as_arrow() other_schema = other_schema.remove(6).insert( @@ -3497,21 +3501,21 @@ def test_parse_location_defaults() -> None: scheme, netloc, path = PyArrowFileIO.parse_location("/foo/bar") assert scheme == "file" assert netloc == "" - assert path == "/foo/bar" + assert path == os.path.abspath("/foo/bar") scheme, netloc, path = PyArrowFileIO.parse_location( "/foo/bar", properties={"DEFAULT_SCHEME": "scheme", "DEFAULT_NETLOC": "netloc:8000"} ) assert scheme == "scheme" assert netloc == "netloc:8000" - assert path == "/foo/bar" + assert path == os.path.abspath("/foo/bar") scheme, netloc, path = PyArrowFileIO.parse_location( "/foo/bar", properties={"DEFAULT_SCHEME": "hdfs", "DEFAULT_NETLOC": "netloc:8000"} ) assert scheme == "hdfs" assert netloc == "netloc:8000" - assert path == "/foo/bar" + assert path == os.path.abspath("/foo/bar") def test_write_and_read_orc(tmp_path: Path) -> None: diff --git a/tests/utils/test_manifest.py b/tests/utils/test_manifest.py index eac4d520bc..18f71993ad 100644 --- a/tests/utils/test_manifest.py +++ b/tests/utils/test_manifest.py @@ -351,7 +351,7 @@ def test_read_manifest_cache(generated_manifest_file_file_v2: str) -> None: def test_write_empty_manifest() -> None: io = load_file_io() test_schema = Schema(NestedField(1, "foo", IntegerType(), False)) - with TemporaryDirectory() as tmpdir: + with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir: tmp_avro_file = tmpdir + "/test_write_manifest.avro" with pytest.raises(ValueError, match="An empty manifest file has been written"):