diff --git a/.github/actions/build_wheels/action.yml b/.github/actions/build_wheels/action.yml new file mode 100644 index 00000000..4d8ff9c9 --- /dev/null +++ b/.github/actions/build_wheels/action.yml @@ -0,0 +1,71 @@ +name: "Build Package" +description: "Build wheels for all platforms" + +inputs: + python-version: + description: "Python version to use for building" + required: false + default: "3.12" + target: + description: "Target architecture" + required: true + manylinux: + description: "Manylinux version" + required: false + default: "" + is-musl: + description: "Whether this is a musl build" + required: false + default: "false" + before-script: + description: "Script to run before building" + required: false + default: "" + platform-name: + description: "Platform name for artifact identification" + required: true + +runs: + using: "composite" + steps: + # Cache Rust build artifacts + - name: Cache Rust build + uses: actions/cache@v4 + with: + path: | + target + key: ${{ runner.os }}-build-${{ inputs.platform-name }}-${{ hashFiles('**/Cargo.lock', 'src/**') }} + restore-keys: | + ${{ runner.os }}-build-${{ inputs.platform-name }}- + + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: ${{ inputs.is-musl != 'true' && 'pip' || '' }} + + # Install python dependencies to run cargo tests + - name: Install required python dependencies + if: ${{ inputs.is-musl != 'true' }} + run: pip install numpy fsspec s3fs xarray + shell: bash + + - name: Run cargo tests + if: ${{ inputs.is-musl != 'true' }} + run: cargo test --no-default-features + shell: bash + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ inputs.target }} + args: --release --out dist --find-interpreter + sccache: "true" + manylinux: ${{ inputs.manylinux }} + container: ${{ inputs.is-musl == 'true' && 'off' || '' }} + before-script-linux: ${{ inputs.before-script }} + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ inputs.platform-name }} + path: dist diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml deleted file mode 100644 index f6284e9e..00000000 --- a/.github/actions/run-tests/action.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: "Run Tests" -description: "Install package and run tests" - -inputs: - skip_tests: - description: "Skip running tests" - required: false - default: "false" - is_musl: - description: "Whether this is a musl build" - required: false - default: "false" - -runs: - using: "composite" - steps: - - name: Run regular tests - if: ${{ inputs.skip_tests != 'true' && inputs.is_musl != 'true' }} - shell: bash - run: | - python -m pip install pytest numpy - WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" - pytest tests/ - - - name: Run musl tests - if: ${{ inputs.skip_tests != 'true' && inputs.is_musl == 'true' }} - uses: addnab/docker-run-action@v3 - with: - image: alpine:latest - options: -v ${{ github.workspace }}:/io -w /io - run: | - apk add python3 py3-pip - python -m venv .venv - source .venv/bin/activate - pip install --upgrade pip - WHEEL_PATH=$(ls dist/*.whl) - pip install --force-reinstall "$WHEEL_PATH" - pip install pytest - pytest tests/ diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000..188885ff --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,74 @@ +name: "Test Package" +description: "Run tests on built wheels" + +inputs: + python-version: + description: "Python version to use for testing" + required: false + default: "3.12" + is-musl: + description: "Whether this is a musl build" + required: false + default: "false" + test-type: + description: "Type of test to run (standard or min_deps)" + required: false + default: "standard" + platform-name: + description: "Platform name for artifact identification" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Download wheels + uses: actions/download-artifact@v4 + with: + name: wheels-${{ inputs.platform-name }} + path: dist + + # Regular tests (non-musl platforms) + - name: Run regular tests + if: ${{ inputs.test-type == 'standard' && inputs.is-musl != 'true' }} + run: | + python -m pip install pytest + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" + pytest tests/ + shell: bash + + # Musl tests + - name: Run musl tests + if: ${{ inputs.test-type == 'standard' && inputs.is-musl == 'true' }} + uses: addnab/docker-run-action@v3 + with: + image: alpine:latest + options: -v ${{ github.workspace }}:/io -w /io + run: | + apk add python3 py3-pip + python -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + WHEEL_PATH=$(ls dist/*.whl) + pip install --force-reinstall "$WHEEL_PATH" + pip install pytest + pytest tests/ + + # Minimum dependencies test + - name: Run tests with minimum dependencies + if: ${{ inputs.test-type == 'min_deps' }} + run: | + python -m pip install pytest==6.0 + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" \ + "numpy==1.20.0" \ + "fsspec==2023.1.0" \ + "s3fs==2023.1.0" \ + "xarray==2023.1.0" + pytest tests/ -v + shell: bash diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index 80bfcf56..00000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,130 +0,0 @@ -# This file is autogenerated by maturin v1.7.4 -# To update, run -# -# maturin generate-ci github -# -name: CI - -on: - push: - branches: - - main - tags: - - "*" - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - build: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - # Linux x86_64 - - runner: ubuntu-latest - target: x86_64 - before-script: python3 -m ensurepip && cat /etc/os-release && yum install clang -y - manylinux: "2_28" - - # Linux aarch64 - - runner: ubuntu-latest - target: aarch64-unknown-linux-gnu - skip_tests: true - before-script: | - apt-get update && \ - apt-get install --assume-yes --no-install-recommends crossbuild-essential-arm64 - manylinux: "2_28" - - # Musl x86_64 - - runner: ubuntu-22.04 - target: x86_64-unknown-linux-musl - container: docker://messense/rust-musl-cross:x86_64-musl - before-script: cat /etc/os-release && apt install clang -y - manylinux: musllinux_1_2 - is_musl: true - - # Windows - - runner: windows-latest - target: x64 - - # macOS - - runner: macos-13 - target: x86_64 - - runner: macos-14 - target: aarch64 - - container: ${{ matrix.platform.container }} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: 3.x - - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: "true" - manylinux: ${{ matrix.platform.manylinux }} - container: ${{ matrix.platform.is_musl && 'off' || '' }} - before-script-linux: ${{ matrix.platform.before-script }} - - - uses: ./.github/actions/run-tests - with: - skip_tests: ${{ matrix.platform.skip_tests }} - is_musl: ${{ matrix.platform.is_musl }} - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} - path: dist - - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist - - name: Upload sdist - uses: actions/upload-artifact@v4 - with: - name: wheels-sdist - path: dist - - release: - name: Release - runs-on: ubuntu-latest - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: - - build - - sdist - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@v4 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-path: "wheels-*/*" - - name: Publish to PyPI - if: "startsWith(github.ref, 'refs/tags/')" - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* diff --git a/.github/workflows/build-test.yml b/.github/workflows/build-test.yml new file mode 100644 index 00000000..49b22508 --- /dev/null +++ b/.github/workflows/build-test.yml @@ -0,0 +1,189 @@ +name: Build and Test + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + name: Build - ${{ matrix.platform.name }} + runs-on: ${{ matrix.platform.runner }} + container: ${{ matrix.platform.container }} + strategy: + matrix: + platform: + - name: linux-x86_64 + runner: ubuntu-latest + target: x86_64 + before-script: python3 -m ensurepip && cat /etc/os-release && yum install clang -y + manylinux: "2_28" + is-musl: false + + - name: linux-aarch64 + runner: ubuntu-latest + target: aarch64-unknown-linux-gnu + skip_tests: true + before-script: | + apt-get update && \ + apt-get install --assume-yes --no-install-recommends crossbuild-essential-arm64 + manylinux: "2_28" + is-musl: false + + - name: linux-musl-x86_64 + runner: ubuntu-22.04 + target: x86_64-unknown-linux-musl + container: docker://messense/rust-musl-cross:x86_64-musl + before-script: cat /etc/os-release && apt install clang -y + manylinux: musllinux_1_2 + is-musl: true + + - name: windows-x64 + runner: windows-latest + target: x64 + is-musl: false + + - name: macos-x86_64 + runner: macos-13 + target: x86_64 + is-musl: false + + - name: macos-arm64 + runner: macos-14 + target: aarch64 + is-musl: false + + steps: + # Use the build composite action + - uses: actions/checkout@v4 + + - name: Build wheels + uses: ./.github/actions/build_wheels + with: + python-version: "3.12" + target: ${{ matrix.platform.target }} + manylinux: ${{ matrix.platform.manylinux }} + is-musl: ${{ matrix.platform.is-musl }} + before-script: ${{ matrix.platform.before-script }} + platform-name: ${{ matrix.platform.name }} + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist + + test: + needs: build + name: Test - ${{ matrix.config.name }} + runs-on: ${{ matrix.config.runner }} + strategy: + fail-fast: false + matrix: + config: + # Platform tests with Python 3.12 + - name: Linux x86_64 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.12" + test-type: standard + + - name: Linux musl + runner: ubuntu-22.04 + platform-name: linux-musl-x86_64 + is-musl: true + python-version: "3.12" + test-type: standard + + - name: Windows + runner: windows-latest + platform-name: windows-x64 + is-musl: false + python-version: "3.12" + test-type: standard + + - name: macOS x86_64 + runner: macos-13 + platform-name: macos-x86_64 + is-musl: false + python-version: "3.12" + test-type: standard + + - name: macOS ARM64 + runner: macos-14 + platform-name: macos-arm64 + is-musl: false + python-version: "3.12" + test-type: standard + + # Additional Python versions for Linux x86_64 + - name: Python 3.8 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.8" + test-type: standard + + - name: Python 3.9 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.9" + test-type: standard + + - name: Python 3.10 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.10" + test-type: standard + + - name: Python 3.11 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.11" + test-type: standard + + - name: Python 3.13 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.13" + test-type: standard + + # Minimum dependencies test + - name: Min Dependencies + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.8" + test-type: min_deps + steps: + # Use the test composite action + - uses: actions/checkout@v4 + + - name: Run tests + uses: ./.github/actions/test + with: + python-version: ${{ matrix.config.python-version }} + is-musl: ${{ matrix.config.is-musl }} + test-type: ${{ matrix.config.test-type }} + platform-name: ${{ matrix.config.platform-name }} diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5315b6a1..79a8a0e9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,46 +13,22 @@ on: - "true" jobs: - # # First check if CI workflow succeeded - # check-ci: - # runs-on: ubuntu-latest - # steps: - # - name: Check CI workflow status - # uses: actions/github-script@v7 - # id: check-status - # with: - # script: | - # const ref = context.sha; - # const workflow_name = 'Build and Test'; - - # const workflow_runs = await github.rest.actions.listWorkflowRuns({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # workflow_id: workflow_name, - # head_sha: ref, - # }); - - # const latest_run = workflow_runs.data.workflow_runs[0]; - # if (!latest_run || latest_run.conclusion !== 'success') { - # core.setFailed('CI workflow has not succeeded for this commit'); - # return false; - # } - # return true; - publish: if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - # needs: check-ci runs-on: ubuntu-latest permissions: id-token: write contents: write attestations: write + actions: read steps: - - name: Download all wheel artifacts - uses: actions/download-artifact@v4 + - name: Download artifacts from CI + uses: dawidd6/action-download-artifact@v9 with: - workflow: build-and-test.yml - commit: ${{ github.sha }} + github_token: ${{ secrets.GITHUB_TOKEN }} + workflow: build-test.yml + commit: ${{ github.event.workflow_run.head_sha }} + workflow_conclusion: success path: artifact-download # Create dist directory @@ -77,7 +53,6 @@ jobs: - name: Publish to TestPyPI uses: PyO3/maturin-action@v1 env: - MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} MATURIN_REPOSITORY: testpypi with: command: upload @@ -87,8 +62,6 @@ jobs: - name: Publish to PyPI if: ${{ github.event.inputs.publish_to_pypi == 'true' }} uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} with: command: upload args: --non-interactive --skip-existing dist/* diff --git a/.gitignore b/.gitignore index 107d90b0..e809d8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,6 @@ docs/_build/ # Pyenv .python-version + +# CSpell Spell Checker +cspell.json diff --git a/Cargo.lock b/Cargo.lock index d9582018..1a80671a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "omfilesrspy" -version = "0.1.0" +version = "0.0.1" dependencies = [ "delegate", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 44edec8f..80f17227 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "omfilesrspy" -version = "0.1.0" +version = "0.0.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] -name = "omfilesrspy" +name = "omfiles" crate-type = ["cdylib"] [dependencies] diff --git a/README.md b/README.md index a115531b..a3ce083a 100644 --- a/README.md +++ b/README.md @@ -1,31 +1,27 @@ -# Omfiles-rs Python bindings +# Python bindings for Open Meteo file format -## Development +[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) +[![Build and Test](https://github.com/terraputix/omfilesrspy/actions/workflows/build-test.yml/badge.svg)](https://github.com/terraputix/omfilesrspy/actions/workflows/build-test.yml) +[![Project Status: WIP – Initial development is in progress, but there has not yet been a stable, usable release suitable for the public.](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip) +[![Publish](https://github.com/terraputix/omfilesrspy/actions/workflows/publish.yml/badge.svg)](https://github.com/terraputix/omfilesrspy/actions/workflows/publish.yml) -```bash -# setup python virtual environment with pyenv -python -m venv .venv -source .venv/bin/activate -# To always activate this environment in this directory run `pyenv local pyo3` -pip install maturin -maturin develop --extras=dev -# if you encounter an error: Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them -unset CONDA_PREFIX -``` +> **Note:** This package is currently under active development and not yet ready for production use. APIs may change without notice until the first stable release. -### Tests +## Features -```bash -cargo test --no-default-features -``` +- Fast reading and writing of multi-dimensional arrays +- Hierarchical data structure support +- Integration with [NumPy](https://github.com/numpy/numpy) arrays +- Chunked data access for efficient I/O +- Support for [fsspec](https://github.com/fsspec/filesystem_spec) and [xarray](https://github.com/pydata/xarray) ### Basic Reading OM files are [structured like a tree of variables](https://github.com/open-meteo/om-file-format?tab=readme-ov-file#data-hierarchy-model). The following example assumes that the file `test_file.om` contains an array variable as a root variable which has a dimensionality greater than 2 and a size of at least 2x100: ```python -from omfilesrspy import OmFilePyReader +from omfiles import OmFilePyReader reader = OmFilePyReader("test_file.om") data = reader[0:2, 0:100, ...] @@ -36,7 +32,7 @@ data = reader[0:2, 0:100, ...] #### Simple Array ```python import numpy as np -from omfilesrspy import OmFilePyWriter +from omfiles import OmFilePyWriter # Create sample data data = np.random.rand(100, 100).astype(np.float32) @@ -61,7 +57,7 @@ writer.close(variable) #### Hierarchical Structure ```python import numpy as np -from omfilesrspy import OmFilePyWriter +from omfiles import OmFilePyWriter # Create sample data features = np.random.rand(1000, 64).astype(np.float32) @@ -100,6 +96,27 @@ root_var = writer.write_scalar( writer.close(root_var) ``` + +## Development + +```bash +# setup python virtual environment with pyenv +python -m venv .venv +source .venv/bin/activate +# To always activate this environment in this directory run `pyenv local pyo3` +pip install maturin + +maturin develop --extras=dev +# if you encounter an error: Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them +unset CONDA_PREFIX +``` + +### Tests + +```bash +cargo test --no-default-features +``` + ## Benchmarks Before running the benchmarks, make sure to compile the release version of the library: diff --git a/benchmarks/helpers/args.py b/benchmarks/helpers/args.py index b62b12bb..d755aac7 100644 --- a/benchmarks/helpers/args.py +++ b/benchmarks/helpers/args.py @@ -1,6 +1,6 @@ import argparse -from omfilesrspy.types import BasicSelection +from omfiles.types import BasicSelection def parse_args() -> argparse.Namespace: diff --git a/benchmarks/helpers/io/readers.py b/benchmarks/helpers/io/readers.py index d49b4061..7557cecd 100644 --- a/benchmarks/helpers/io/readers.py +++ b/benchmarks/helpers/io/readers.py @@ -4,10 +4,10 @@ import h5py import netCDF4 as nc import numpy as np -import omfilesrspy as om +import omfiles as om import xarray as xr import zarr -from omfilesrspy.types import BasicSelection +from omfiles.types import BasicSelection class BaseReader(ABC): diff --git a/benchmarks/helpers/io/writers.py b/benchmarks/helpers/io/writers.py index 36d47777..b5e7b7af 100644 --- a/benchmarks/helpers/io/writers.py +++ b/benchmarks/helpers/io/writers.py @@ -5,7 +5,7 @@ import h5py import netCDF4 as nc import numpy as np -import omfilesrspy as om +import omfiles as om import zarr from zarr.core.buffer import NDArrayLike diff --git a/pyproject.toml b/pyproject.toml index 8266b05e..4de12732 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["maturin>=1.7,<2.0"] build-backend = "maturin" [project] -name = "omfilesrspy" +name = "omfiles" requires-python = ">=3.8" classifiers = [ "Programming Language :: Rust", @@ -13,9 +13,9 @@ classifiers = [ dynamic = ["version"] dependencies = [ "numpy>=1.20.0", - "fsspec>=2024.10.0", - "s3fs>=2024.10.0", - "xarray>=0.19.0", + "fsspec>=2023.1.0", + "s3fs>=2023.1.0", + "xarray>=2023.1.0", ] [tool.maturin] @@ -27,4 +27,4 @@ bindings = "pyo3" # pyo3 bindings is actually the default for maturin dev = ["pytest>=6.0", "hidefix", "h5py", "netCDF4", "zarr"] [project.entry-points."xarray.backends"] -om = "omfilesrspy.xarray_backend:OmXarrayEntrypoint" +om = "omfiles.xarray_backend:OmXarrayEntrypoint" diff --git a/python/omfilesrspy/__init__.py b/python/omfiles/__init__.py similarity index 66% rename from python/omfilesrspy/__init__.py rename to python/omfiles/__init__.py index c4cc74a0..9ce9d4c8 100644 --- a/python/omfilesrspy/__init__.py +++ b/python/omfiles/__init__.py @@ -1,4 +1,4 @@ from . import types, xarray_backend -from .omfilesrspy import OmFilePyReader, OmFilePyWriter +from .omfiles import OmFilePyReader, OmFilePyWriter __all__ = ["OmFilePyReader", "OmFilePyWriter", "xarray_backend", "types"] diff --git a/python/omfilesrspy/py.typed b/python/omfiles/py.typed similarity index 100% rename from python/omfilesrspy/py.typed rename to python/omfiles/py.typed diff --git a/python/omfilesrspy/omfilesrspy.pyi b/python/omfiles/pyomfiles.pyi similarity index 99% rename from python/omfilesrspy/omfilesrspy.pyi rename to python/omfiles/pyomfiles.pyi index 4759f185..d9546393 100644 --- a/python/omfilesrspy/omfilesrspy.pyi +++ b/python/omfiles/pyomfiles.pyi @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Tuple, Union import numpy as np diff --git a/python/omfiles/types.py b/python/omfiles/types.py new file mode 100644 index 00000000..776d9681 --- /dev/null +++ b/python/omfiles/types.py @@ -0,0 +1,9 @@ +try: + from types import EllipsisType +except ImportError: + EllipsisType = type(Ellipsis) +from typing import Tuple, Union + +# This is from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 +BasicSelector = Union[int, slice, EllipsisType] +BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] # also used for BlockIndex diff --git a/python/omfilesrspy/xarray_backend.py b/python/omfiles/xarray_backend.py similarity index 98% rename from python/omfilesrspy/xarray_backend.py rename to python/omfiles/xarray_backend.py index 79109721..11991d72 100644 --- a/python/omfilesrspy/xarray_backend.py +++ b/python/omfiles/xarray_backend.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import numpy as np from xarray.backends.common import BackendArray, BackendEntrypoint, WritableCFDataStore, _normalize_path from xarray.backends.store import StoreBackendEntrypoint @@ -5,7 +7,7 @@ from xarray.core.utils import FrozenDict from xarray.core.variable import Variable -from .omfilesrspy import OmFilePyReader, OmVariable +from .omfiles import OmFilePyReader, OmVariable # need some special secret attributes to tell us the dimensions DIMENSION_KEY = "_ARRAY_DIMENSIONS" diff --git a/python/omfilesrspy/types.py b/python/omfilesrspy/types.py deleted file mode 100644 index a8dbadbd..00000000 --- a/python/omfilesrspy/types.py +++ /dev/null @@ -1,5 +0,0 @@ -from types import EllipsisType - -# This is from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 -BasicSelector = int | slice | EllipsisType -BasicSelection = BasicSelector | tuple[BasicSelector, ...] # also used for BlockIndex diff --git a/src/lib.rs b/src/lib.rs index c912e3b0..17b8dd1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ mod writer; /// A Python module implemented in Rust. #[pymodule(gil_used = false)] -fn omfilesrspy<'py>(m: &Bound<'py, PyModule>) -> PyResult<()> { +fn omfiles<'py>(m: &Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..8b137891 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 2ac6b210..71b7556c 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -1,11 +1,11 @@ import fsspec import numpy as np -import omfilesrspy +import omfiles # def test_fsspec_backend(): # fsspec_object = fsspec.open("test_files/read_test.om", "rb") -# file = omfilesrspy.FsSpecBackend(fsspec_object) +# file = omfiles.FsSpecBackend(fsspec_object) # assert file.file_size == 144 @@ -15,7 +15,7 @@ def test_s3_reader(): backend = fs.open(file_path, mode="rb") # Create reader over fs spec backend - reader = omfilesrspy.OmFilePyReader(backend) + reader = omfiles.OmFilePyReader(backend) data = reader[57812:57813, 0:100] # Verify the data @@ -29,7 +29,7 @@ def test_s3_reader_with_cache(): backend = fs.open(file_path, mode="rb", cache_type="mmap", block_size=1024, cache_options={"location": "cache"}) # Create reader over fs spec backend - reader = omfilesrspy.OmFilePyReader(backend) + reader = omfiles.OmFilePyReader(backend) data = reader[57812:57813, 0:100] # Verify the data diff --git a/tests/test_omfilesrspy.py b/tests/test_omfilesrspy.py index bc197cd6..e5b3298f 100644 --- a/tests/test_omfilesrspy.py +++ b/tests/test_omfilesrspy.py @@ -1,7 +1,7 @@ import os import numpy as np -import omfilesrspy +import omfiles from .test_utils import create_test_om_file @@ -12,7 +12,7 @@ def test_write_om_roundtrip(): try: create_test_om_file(temp_file) - reader = omfilesrspy.OmFilePyReader(temp_file) + reader = omfiles.OmFilePyReader(temp_file) data = reader[0:5, 0:5] del reader @@ -54,14 +54,14 @@ def test_round_trip_array_datatypes(): try: # Write data - writer = omfilesrspy.OmFilePyWriter(temp_file) + writer = omfiles.OmFilePyWriter(temp_file) variable = writer.write_array(test_data, chunks=chunks, scale_factor=10000.0, add_offset=0.0) writer.close(variable) del writer # Read data back - reader = omfilesrspy.OmFilePyReader(temp_file) + reader = omfiles.OmFilePyReader(temp_file) read_data = reader[:] del reader @@ -85,7 +85,7 @@ def test_write_hierarchical_file(): child2_data = np.random.rand(3, 3).astype(np.float32) # Write hierarchical structure - writer = omfilesrspy.OmFilePyWriter(temp_file) + writer = omfiles.OmFilePyWriter(temp_file) # Write child2 array child2_var = writer.write_array( @@ -123,7 +123,7 @@ def test_write_hierarchical_file(): del writer # Read and verify the data using OmFilePyReader - reader = omfilesrspy.OmFilePyReader(temp_file) + reader = omfiles.OmFilePyReader(temp_file) # Verify root data read_root = reader[:] diff --git a/tests/test_utils.py b/tests/test_utils.py index 1e3bbacd..db90cdcb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,5 +1,7 @@ +from __future__ import annotations + import numpy as np -from omfilesrspy import OmFilePyWriter +from omfiles import OmFilePyWriter def create_test_om_file(filename: str = "test_file.om", shape=(5, 5), dtype=np.float32) -> tuple[str, np.ndarray]: diff --git a/tests/test_xarray.py b/tests/test_xarray.py index 41e65c7e..1bf0a038 100644 --- a/tests/test_xarray.py +++ b/tests/test_xarray.py @@ -1,9 +1,15 @@ import os +# for some reason xr.open_dataset triggers a warning: +# "RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility. Expected 16 from C header, got 96 from PyObject" +# We will just filter it out for now... +# https://github.com/pydata/xarray/issues/7259 +import warnings + import numpy as np +import omfiles.omfiles as om +import omfiles.xarray_backend as om_xarray import xarray as xr -from omfilesrspy.omfilesrspy import OmFilePyReader, OmFilePyWriter -from omfilesrspy.xarray_backend import OmBackendArray from xarray.core import indexing from .test_utils import create_test_om_file @@ -27,8 +33,8 @@ def test_om_backend_xarray_dtype(): try: create_test_om_file(temp_file, dtype=dtype) - reader = OmFilePyReader(temp_file) - backend_array = OmBackendArray(reader=reader) + reader = om.OmFilePyReader(temp_file) + backend_array = om_xarray.OmBackendArray(reader=reader) assert isinstance(backend_array.dtype, np.dtype) assert backend_array.dtype == dtype @@ -47,6 +53,7 @@ def test_xarray_backend(): try: create_test_om_file(temp_file) + warnings.filterwarnings("ignore", message="numpy.ndarray size changed", category=RuntimeWarning) ds = xr.open_dataset(temp_file, engine="om") data = ds["data"][:].values del ds @@ -78,7 +85,7 @@ def test_xarray_hierarchical_file(): precipitation_data = np.random.rand(5, 5, 10).astype(np.float32) # Write hierarchical structure - writer = OmFilePyWriter(temp_file) + writer = om.OmFilePyWriter(temp_file) # dimensionality metadata temperature_dimension_var = writer.write_scalar("LATITUDE,LONGITUDE,ALTITUDE,TIME", name="_ARRAY_DIMENSIONS") @@ -125,10 +132,9 @@ def test_xarray_hierarchical_file(): writer.close(root_var) del writer - # Read using xarray backend + warnings.filterwarnings("ignore", message="numpy.ndarray size changed", category=RuntimeWarning) ds = xr.open_dataset(temp_file, engine="om") - # Check temperature data temp = ds["temperature"] np.testing.assert_array_almost_equal(temp.values, temperature_data, decimal=4)