From 236b79f3b69cb7fc231eb00cf638c60a5f6d0319 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 28 Feb 2025 11:12:09 +0100 Subject: [PATCH 01/37] test pypi workflow --- .github/workflows/CI.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 80bfcf56..d7831067 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -101,7 +101,7 @@ jobs: path: dist release: - name: Release + name: Release to PyPI runs-on: ubuntu-latest if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} needs: @@ -128,3 +128,32 @@ jobs: with: command: upload args: --non-interactive --skip-existing wheels-*/* + + test-pypi-release: + name: Release to TestPyPI + runs-on: ubuntu-latest + # Run on pushes to main or manual workflow dispatch + if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push') + needs: + - build + - sdist + permissions: + # Use to sign the release artifacts + id-token: write + # Used to upload release artifacts + contents: write + # Used to generate artifact attestation + attestations: write + steps: + - uses: actions/download-artifact@v4 + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: "wheels-*/*" + - name: Publish to TestPyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + with: + command: upload + args: --repository testpypi --non-interactive --skip-existing wheels-*/* From 5cb2c65e610296feb26fb608fae5a5c840df7de2 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 28 Feb 2025 12:03:26 +0100 Subject: [PATCH 02/37] fix repository --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index d7831067..82163107 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -154,6 +154,7 @@ jobs: uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + MATURIN_REPOSITORY: testpypi with: command: upload - args: --repository testpypi --non-interactive --skip-existing wheels-*/* + args: --non-interactive --skip-existing wheels-*/* From cbdf78cc96f4e13dda106a06cf264832a287754d Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 28 Feb 2025 12:16:53 +0100 Subject: [PATCH 03/37] fix automatic run --- .github/workflows/CI.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 82163107..0d240dbc 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -132,8 +132,8 @@ jobs: test-pypi-release: name: Release to TestPyPI runs-on: ubuntu-latest - # Run on pushes to main or manual workflow dispatch - if: github.event_name == 'workflow_dispatch' || (github.event_name == 'push') + # Run on pushes or manual workflow dispatch + if: ${{github.event_name == 'workflow_dispatch' || (github.event_name == 'push') }} needs: - build - sdist From d44a161eb6e93ba05efc4ba31c6791bb80f5878e Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 11:59:38 +0100 Subject: [PATCH 04/37] relax dependencies and filter xr related warning in test --- pyproject.toml | 8 ++++---- tests/__init__.py | 1 + tests/test_xarray.py | 20 +++++++++++++------- 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 8266b05e..349d515d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,10 +12,10 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "numpy>=1.20.0", - "fsspec>=2024.10.0", - "s3fs>=2024.10.0", - "xarray>=0.19.0", + "numpy>=1.26.0", + "fsspec>=2023.1.0", + "s3fs>=2023.1.0", + "xarray>=2023.1.0", ] [tool.maturin] diff --git a/tests/__init__.py b/tests/__init__.py index e69de29b..8b137891 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -0,0 +1 @@ + diff --git a/tests/test_xarray.py b/tests/test_xarray.py index 41e65c7e..06d0b32d 100644 --- a/tests/test_xarray.py +++ b/tests/test_xarray.py @@ -1,9 +1,15 @@ import os +# for some reason xr.open_dataset triggers a warning: +# "RuntimeWarning: numpy.ndarray size changed, may indicate binary incompatibility. Expected 16 from C header, got 96 from PyObject" +# We will just filter it out for now... +# https://github.com/pydata/xarray/issues/7259 +import warnings + import numpy as np +import omfilesrspy.omfilesrspy as om +import omfilesrspy.xarray_backend as om_xarray import xarray as xr -from omfilesrspy.omfilesrspy import OmFilePyReader, OmFilePyWriter -from omfilesrspy.xarray_backend import OmBackendArray from xarray.core import indexing from .test_utils import create_test_om_file @@ -27,8 +33,8 @@ def test_om_backend_xarray_dtype(): try: create_test_om_file(temp_file, dtype=dtype) - reader = OmFilePyReader(temp_file) - backend_array = OmBackendArray(reader=reader) + reader = om.OmFilePyReader(temp_file) + backend_array = om_xarray.OmBackendArray(reader=reader) assert isinstance(backend_array.dtype, np.dtype) assert backend_array.dtype == dtype @@ -47,6 +53,7 @@ def test_xarray_backend(): try: create_test_om_file(temp_file) + warnings.filterwarnings("ignore", message="numpy.ndarray size changed", category=RuntimeWarning) ds = xr.open_dataset(temp_file, engine="om") data = ds["data"][:].values del ds @@ -78,7 +85,7 @@ def test_xarray_hierarchical_file(): precipitation_data = np.random.rand(5, 5, 10).astype(np.float32) # Write hierarchical structure - writer = OmFilePyWriter(temp_file) + writer = om.OmFilePyWriter(temp_file) # dimensionality metadata temperature_dimension_var = writer.write_scalar("LATITUDE,LONGITUDE,ALTITUDE,TIME", name="_ARRAY_DIMENSIONS") @@ -125,10 +132,9 @@ def test_xarray_hierarchical_file(): writer.close(root_var) del writer - # Read using xarray backend + warnings.filterwarnings("ignore", message="numpy.ndarray size changed", category=RuntimeWarning) ds = xr.open_dataset(temp_file, engine="om") - # Check temperature data temp = ds["temperature"] np.testing.assert_array_almost_equal(temp.values, temperature_data, decimal=4) From a4a6e97ba2ed8ebb1218d85de6fa148c57e5ef24 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:06:47 +0100 Subject: [PATCH 05/37] python version matrix --- .github/workflows/CI.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 0d240dbc..7c656925 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -22,6 +22,7 @@ jobs: runs-on: ${{ matrix.platform.runner }} strategy: matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] platform: # Linux x86_64 - runner: ubuntu-latest @@ -62,7 +63,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: 3.x + python-version: ${{ matrix.python-version }} - name: Build wheels uses: PyO3/maturin-action@v1 From 4451c4d1380deebb18dcd7954f8fd0e2197dfdc0 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:12:33 +0100 Subject: [PATCH 06/37] separate python version compatibility job --- .github/workflows/CI.yml | 33 +++++++++++++++++++++++++++++++-- 1 file changed, 31 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 7c656925..687a1a07 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -22,7 +22,6 @@ jobs: runs-on: ${{ matrix.platform.runner }} strategy: matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] platform: # Linux x86_64 - runner: ubuntu-latest @@ -63,7 +62,7 @@ jobs: - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.python-version }} + python-version: "3.12" # Use a stable Python version for building - name: Build wheels uses: PyO3/maturin-action@v1 @@ -86,6 +85,36 @@ jobs: name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} path: dist + test-python-versions: + needs: build + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] + + steps: + - uses: actions/checkout@v4 + + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + + - name: Download Linux wheel + uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-latest-x86_64 + path: dist + + - name: Install and test + run: | + python -m pip install --upgrade pip + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" + python -m pip install pytest + pytest tests/ + sdist: runs-on: ubuntu-latest steps: From ab896f0777e6445378ff0f542dfd138f76bcb3cc Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:20:21 +0100 Subject: [PATCH 07/37] further relax dependencies --- .github/actions/run-tests/action.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml index f6284e9e..ea0c5e69 100644 --- a/.github/actions/run-tests/action.yml +++ b/.github/actions/run-tests/action.yml @@ -18,7 +18,7 @@ runs: if: ${{ inputs.skip_tests != 'true' && inputs.is_musl != 'true' }} shell: bash run: | - python -m pip install pytest numpy + python -m pip install pytest WHEEL_PATH=$(ls dist/*.whl) python -m pip install --force-reinstall "$WHEEL_PATH" pytest tests/ diff --git a/pyproject.toml b/pyproject.toml index 349d515d..5d3b5d52 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "numpy>=1.26.0", + "numpy>=1.16.0", "fsspec>=2023.1.0", "s3fs>=2023.1.0", "xarray>=2023.1.0", From 9b0e1d83158cc7c36f427b4ef781ffbff9f6da53 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:26:22 +0100 Subject: [PATCH 08/37] fix types for older python versions --- python/omfilesrspy/types.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/python/omfilesrspy/types.py b/python/omfilesrspy/types.py index a8dbadbd..906cfef9 100644 --- a/python/omfilesrspy/types.py +++ b/python/omfilesrspy/types.py @@ -1,5 +1,9 @@ -from types import EllipsisType +try: + from types import EllipsisType +except ImportError: + EllipsisType = type(Ellipsis) +from typing import Tuple, Union -# This is from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 -BasicSelector = int | slice | EllipsisType -BasicSelection = BasicSelector | tuple[BasicSelector, ...] # also used for BlockIndex +# This is similar to https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 +BasicSelector = Union[int, slice, EllipsisType] +BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] # also used for BlockIndex From d28e60b7adc2a6fc76430f4b5d9dc304d764cc1f Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:40:06 +0100 Subject: [PATCH 09/37] fix type problems with annotations module --- python/omfilesrspy/omfilesrspy.pyi | 2 ++ python/omfilesrspy/types.py | 7 ++++--- python/omfilesrspy/xarray_backend.py | 2 ++ 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/python/omfilesrspy/omfilesrspy.pyi b/python/omfilesrspy/omfilesrspy.pyi index 4759f185..d9546393 100644 --- a/python/omfilesrspy/omfilesrspy.pyi +++ b/python/omfilesrspy/omfilesrspy.pyi @@ -1,3 +1,5 @@ +from __future__ import annotations + from typing import Tuple, Union import numpy as np diff --git a/python/omfilesrspy/types.py b/python/omfilesrspy/types.py index 906cfef9..dcd3b4e1 100644 --- a/python/omfilesrspy/types.py +++ b/python/omfilesrspy/types.py @@ -1,9 +1,10 @@ +from __future__ import annotations + try: from types import EllipsisType except ImportError: EllipsisType = type(Ellipsis) -from typing import Tuple, Union # This is similar to https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 -BasicSelector = Union[int, slice, EllipsisType] -BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] # also used for BlockIndex +BasicSelector = int | slice | EllipsisType +BasicSelection = BasicSelector | tuple[BasicSelector, ...] # also used for BlockIndex diff --git a/python/omfilesrspy/xarray_backend.py b/python/omfilesrspy/xarray_backend.py index 79109721..3c4803b1 100644 --- a/python/omfilesrspy/xarray_backend.py +++ b/python/omfilesrspy/xarray_backend.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import numpy as np from xarray.backends.common import BackendArray, BackendEntrypoint, WritableCFDataStore, _normalize_path from xarray.backends.store import StoreBackendEntrypoint From 6c65d6e927963b228cfc7597e86bdbdb929487dd Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:49:16 +0100 Subject: [PATCH 10/37] revert change in types --- python/omfilesrspy/types.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/python/omfilesrspy/types.py b/python/omfilesrspy/types.py index dcd3b4e1..776d9681 100644 --- a/python/omfilesrspy/types.py +++ b/python/omfilesrspy/types.py @@ -1,10 +1,9 @@ -from __future__ import annotations - try: from types import EllipsisType except ImportError: EllipsisType = type(Ellipsis) +from typing import Tuple, Union -# This is similar to https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 -BasicSelector = int | slice | EllipsisType -BasicSelection = BasicSelector | tuple[BasicSelector, ...] # also used for BlockIndex +# This is from https://github.com/zarr-developers/zarr-python/blob/main/src/zarr/core/indexing.py#L38C1-L40C87 +BasicSelector = Union[int, slice, EllipsisType] +BasicSelection = Union[BasicSelector, Tuple[Union[int, slice, EllipsisType], ...]] # also used for BlockIndex From a81193227a5e08d8802ab646c9637fd8ac76bad4 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 12:57:51 +0100 Subject: [PATCH 11/37] fix types for older python versions --- tests/test_utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/test_utils.py b/tests/test_utils.py index 1e3bbacd..d8107b5e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import numpy as np from omfilesrspy import OmFilePyWriter From fbc3fee7094faabce69dc6c38f9d8596afc00695 Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 13:04:40 +0100 Subject: [PATCH 12/37] update readme --- README.md | 46 ++++++++++++++++++++++++++++++---------------- 1 file changed, 30 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index a115531b..6eeb16ca 100644 --- a/README.md +++ b/README.md @@ -1,24 +1,17 @@ # Omfiles-rs Python bindings -## Development +[![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) +[![Project Status: WIP – Initial development is in progress, but there has not yet been a stable, usable release suitable for the public.](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip) -```bash -# setup python virtual environment with pyenv -python -m venv .venv -source .venv/bin/activate -# To always activate this environment in this directory run `pyenv local pyo3` -pip install maturin +> **Note:** This package is currently under active development and not yet ready for production use. APIs may change without notice until the first stable release. -maturin develop --extras=dev -# if you encounter an error: Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them -unset CONDA_PREFIX -``` +## Features -### Tests - -```bash -cargo test --no-default-features -``` +- Fast reading and writing of multi-dimensional arrays +- Hierarchical data structure support +- Integration with NumPy arrays +- Chunked data access for efficient I/O +- Support for fsspec and xarray ### Basic Reading @@ -100,6 +93,27 @@ root_var = writer.write_scalar( writer.close(root_var) ``` + +## Development + +```bash +# setup python virtual environment with pyenv +python -m venv .venv +source .venv/bin/activate +# To always activate this environment in this directory run `pyenv local pyo3` +pip install maturin + +maturin develop --extras=dev +# if you encounter an error: Both VIRTUAL_ENV and CONDA_PREFIX are set. Please unset one of them +unset CONDA_PREFIX +``` + +### Tests + +```bash +cargo test --no-default-features +``` + ## Benchmarks Before running the benchmarks, make sure to compile the release version of the library: From a2c1fc6e9b10a4c8e658e6c03835b85662efa9dc Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 13:09:42 +0100 Subject: [PATCH 13/37] test with minimum dependencies --- .github/workflows/CI.yml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 687a1a07..211b435e 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -115,6 +115,34 @@ jobs: python -m pip install pytest pytest tests/ + test-min-deps: + needs: build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.8" # Use the oldest supported Python version + + - name: Download Linux wheel + uses: actions/download-artifact@v4 + with: + name: wheels-ubuntu-latest-x86_64 + path: dist + + - name: Test with minimum dependencies + run: | + python -m pip install pytest==6.0 + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" \ + "numpy==1.16.0" \ + "fsspec==2023.1.0" \ + "s3fs==2023.1.0" \ + "xarray==2023.1.0" + # Run the tests with minimum dependencies + pytest tests/ -v + sdist: runs-on: ubuntu-latest steps: From 817852c6a2ee9c6381db003bf0d8aaa3be89124c Mon Sep 17 00:00:00 2001 From: terraputix Date: Tue, 4 Mar 2025 13:15:51 +0100 Subject: [PATCH 14/37] update required numpy version --- .github/workflows/CI.yml | 2 +- pyproject.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 211b435e..c5b5e9b9 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -136,7 +136,7 @@ jobs: python -m pip install pytest==6.0 WHEEL_PATH=$(ls dist/*.whl) python -m pip install --force-reinstall "$WHEEL_PATH" \ - "numpy==1.16.0" \ + "numpy==1.20.0" \ "fsspec==2023.1.0" \ "s3fs==2023.1.0" \ "xarray==2023.1.0" diff --git a/pyproject.toml b/pyproject.toml index 5d3b5d52..bbcf0494 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ classifiers = [ ] dynamic = ["version"] dependencies = [ - "numpy>=1.16.0", + "numpy>=1.20.0", "fsspec>=2023.1.0", "s3fs>=2023.1.0", "xarray>=2023.1.0", From 0eef9cbd46c4b51dc6a62bcd45fc166f96ce8299 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 15:27:54 +0100 Subject: [PATCH 15/37] separate action into multiple workflows --- .github/actions/run-tests/action.yml | 40 ----- .github/workflows/CI.yml | 218 --------------------------- .github/workflows/build.yml | 100 ++++++++++++ .github/workflows/release.yml | 81 ++++++++++ .github/workflows/test.yml | 144 ++++++++++++++++++ 5 files changed, 325 insertions(+), 258 deletions(-) delete mode 100644 .github/actions/run-tests/action.yml delete mode 100644 .github/workflows/CI.yml create mode 100644 .github/workflows/build.yml create mode 100644 .github/workflows/release.yml create mode 100644 .github/workflows/test.yml diff --git a/.github/actions/run-tests/action.yml b/.github/actions/run-tests/action.yml deleted file mode 100644 index ea0c5e69..00000000 --- a/.github/actions/run-tests/action.yml +++ /dev/null @@ -1,40 +0,0 @@ -name: "Run Tests" -description: "Install package and run tests" - -inputs: - skip_tests: - description: "Skip running tests" - required: false - default: "false" - is_musl: - description: "Whether this is a musl build" - required: false - default: "false" - -runs: - using: "composite" - steps: - - name: Run regular tests - if: ${{ inputs.skip_tests != 'true' && inputs.is_musl != 'true' }} - shell: bash - run: | - python -m pip install pytest - WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" - pytest tests/ - - - name: Run musl tests - if: ${{ inputs.skip_tests != 'true' && inputs.is_musl == 'true' }} - uses: addnab/docker-run-action@v3 - with: - image: alpine:latest - options: -v ${{ github.workspace }}:/io -w /io - run: | - apk add python3 py3-pip - python -m venv .venv - source .venv/bin/activate - pip install --upgrade pip - WHEEL_PATH=$(ls dist/*.whl) - pip install --force-reinstall "$WHEEL_PATH" - pip install pytest - pytest tests/ diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml deleted file mode 100644 index c5b5e9b9..00000000 --- a/.github/workflows/CI.yml +++ /dev/null @@ -1,218 +0,0 @@ -# This file is autogenerated by maturin v1.7.4 -# To update, run -# -# maturin generate-ci github -# -name: CI - -on: - push: - branches: - - main - tags: - - "*" - pull_request: - workflow_dispatch: - -permissions: - contents: read - -jobs: - build: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - # Linux x86_64 - - runner: ubuntu-latest - target: x86_64 - before-script: python3 -m ensurepip && cat /etc/os-release && yum install clang -y - manylinux: "2_28" - - # Linux aarch64 - - runner: ubuntu-latest - target: aarch64-unknown-linux-gnu - skip_tests: true - before-script: | - apt-get update && \ - apt-get install --assume-yes --no-install-recommends crossbuild-essential-arm64 - manylinux: "2_28" - - # Musl x86_64 - - runner: ubuntu-22.04 - target: x86_64-unknown-linux-musl - container: docker://messense/rust-musl-cross:x86_64-musl - before-script: cat /etc/os-release && apt install clang -y - manylinux: musllinux_1_2 - is_musl: true - - # Windows - - runner: windows-latest - target: x64 - - # macOS - - runner: macos-13 - target: x86_64 - - runner: macos-14 - target: aarch64 - - container: ${{ matrix.platform.container }} - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" # Use a stable Python version for building - - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: "true" - manylinux: ${{ matrix.platform.manylinux }} - container: ${{ matrix.platform.is_musl && 'off' || '' }} - before-script-linux: ${{ matrix.platform.before-script }} - - - uses: ./.github/actions/run-tests - with: - skip_tests: ${{ matrix.platform.skip_tests }} - is_musl: ${{ matrix.platform.is_musl }} - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} - path: dist - - test-python-versions: - needs: build - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] - - steps: - - uses: actions/checkout@v4 - - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - - name: Download Linux wheel - uses: actions/download-artifact@v4 - with: - name: wheels-ubuntu-latest-x86_64 - path: dist - - - name: Install and test - run: | - python -m pip install --upgrade pip - WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" - python -m pip install pytest - pytest tests/ - - test-min-deps: - needs: build - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.8" # Use the oldest supported Python version - - - name: Download Linux wheel - uses: actions/download-artifact@v4 - with: - name: wheels-ubuntu-latest-x86_64 - path: dist - - - name: Test with minimum dependencies - run: | - python -m pip install pytest==6.0 - WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" \ - "numpy==1.20.0" \ - "fsspec==2023.1.0" \ - "s3fs==2023.1.0" \ - "xarray==2023.1.0" - # Run the tests with minimum dependencies - pytest tests/ -v - - sdist: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist - - name: Upload sdist - uses: actions/upload-artifact@v4 - with: - name: wheels-sdist - path: dist - - release: - name: Release to PyPI - runs-on: ubuntu-latest - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: - - build - - sdist - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@v4 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-path: "wheels-*/*" - - name: Publish to PyPI - if: "startsWith(github.ref, 'refs/tags/')" - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* - - test-pypi-release: - name: Release to TestPyPI - runs-on: ubuntu-latest - # Run on pushes or manual workflow dispatch - if: ${{github.event_name == 'workflow_dispatch' || (github.event_name == 'push') }} - needs: - - build - - sdist - permissions: - # Use to sign the release artifacts - id-token: write - # Used to upload release artifacts - contents: write - # Used to generate artifact attestation - attestations: write - steps: - - uses: actions/download-artifact@v4 - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-path: "wheels-*/*" - - name: Publish to TestPyPI - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - MATURIN_REPOSITORY: testpypi - with: - command: upload - args: --non-interactive --skip-existing wheels-*/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 00000000..ec972c46 --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,100 @@ +name: Build + +on: + push: + branches: + - main + tags: + - "*" + pull_request: + workflow_dispatch: + +permissions: + contents: read + +jobs: + build: + runs-on: ${{ matrix.platform.runner }} + strategy: + matrix: + platform: + - name: linux-x86_64 + runner: ubuntu-latest + target: x86_64 + before-script: python3 -m ensurepip && cat /etc/os-release && yum install clang -y + manylinux: "2_28" + + - name: linux-aarch64 + runner: ubuntu-latest + target: aarch64-unknown-linux-gnu + skip_tests: true + before-script: | + apt-get update && \ + apt-get install --assume-yes --no-install-recommends crossbuild-essential-arm64 + manylinux: "2_28" + + - name: linux-musl-x86_64 + runner: ubuntu-22.04 + target: x86_64-unknown-linux-musl + container: docker://messense/rust-musl-cross:x86_64-musl + before-script: cat /etc/os-release && apt install clang -y + manylinux: musllinux_1_2 + is_musl: true + + - name: windows-x64 + runner: windows-latest + target: x64 + + - name: macos-x86_64 + runner: macos-13 + target: x86_64 + - name: macos-arm64 + runner: macos-14 + target: aarch64 + + container: ${{ matrix.platform.container }} + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" # Use a stable Python version for building + cache: pip + + # Run Rust tests before building + - name: Run cargo tests + if: ${{ !matrix.platform.is_musl }} + run: cargo test --no-default-features + shell: bash + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: "true" + manylinux: ${{ matrix.platform.manylinux }} + container: ${{ matrix.platform.is_musl && 'off' || '' }} + before-script-linux: ${{ matrix.platform.before-script }} + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} + path: dist + + sdist: + name: Build Source Distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 00000000..ee202624 --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,81 @@ +name: Release + +on: + # push: + # tags: + # - "v*" + workflow_dispatch: + inputs: + publish_to_pypi: + description: "Publish to PyPI" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + +permissions: + contents: read + +jobs: + publish: + name: Publish Package + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + attestations: write + actions: read + + steps: + - name: Download wheel artifacts + uses: actions/download-artifact@v4 + with: + pattern: wheels-* + path: dist + merge-multiple: true + + - name: Download sdist + uses: actions/download-artifact@v4 + with: + name: sdist + path: dist + + # List files for debugging + - name: List distribution files + run: ls -la dist/ + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: "dist/*" + + # Publish to PyPI when tagged or requested + - name: Publish to PyPI + if: startsWith(github.ref, 'refs/tags/') || github.event.inputs.publish_to_pypi == 'true' + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --non-interactive --skip-existing dist/* + + # Otherwise publish to TestPyPI + - name: Publish to TestPyPI + if: ${{ !(startsWith(github.ref, 'refs/tags/') || github.event.inputs.publish_to_pypi == 'true') }} + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + MATURIN_REPOSITORY: testpypi + with: + command: upload + args: --non-interactive --skip-existing dist/* + + # # Create a GitHub release when a tag is created + # - name: Create GitHub Release + # if: startsWith(github.ref, 'refs/tags/') + # uses: softprops/action-gh-release@v1 + # with: + # generate_release_notes: true + # files: dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..d25cd3f6 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,144 @@ +name: Test + +on: + workflow_run: + workflows: ["Build"] + types: + - completed + +permissions: + contents: read + actions: read # Needed to download artifacts + +jobs: + test: + name: ${{ matrix.name }} + # Only run if the build workflow was successful + if: ${{ github.event.workflow_run.conclusion == 'success' }} + runs-on: ${{ matrix.platform.runner }} + strategy: + fail-fast: false + matrix: + # Default values for all matrix entries + test_type: [standard] + python-version: ["3.12"] + platform: + - &linux_x86_64 + name: linux-x86_64 + runner: ubuntu-latest + is_musl: false + + # Override with specific combinations + include: + # Platform tests (uses default Python 3.12 and test_type) + - name: "Linux x86_64" + platform: *linux_x86_64 + + - name: "Linux musl" + platform: + name: linux-musl-x86_64 + runner: ubuntu-22.04 + is_musl: true + + - name: "Windows" + platform: + name: windows-x64 + runner: windows-latest + is_musl: false + + - name: "macOS x86_64" + platform: + name: macos-x86_64 + runner: macos-13 + is_musl: false + + - name: "macOS ARM64" + platform: + name: macos-arm64 + runner: macos-14 + is_musl: false + + # Additional Python versions for Linux x86_64 + - name: "Python 3.8" + platform: *linux_x86_64 + python-version: "3.8" + + - name: "Python 3.9" + platform: *linux_x86_64 + python-version: "3.9" + + - name: "Python 3.10" + platform: *linux_x86_64 + python-version: "3.10" + + - name: "Python 3.11" + platform: *linux_x86_64 + python-version: "3.11" + + - name: "Python 3.13" + platform: *linux_x86_64 + python-version: "3.13" + + # Minimum dependencies test + - name: "Min Dependencies" + platform: *linux_x86_64 + python-version: "3.8" + test_type: min_deps + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + cache: "pip" + + - name: Download wheel artifacts + uses: actions/download-artifact@v4 + with: + name: wheels-${{ matrix.platform.name }} + path: dist + repository: ${{ github.repository }} + run_id: ${{ github.event.workflow_run.id }} + github_token: ${{ secrets.GITHUB_TOKEN }} + + # Regular tests (non-musl platforms) + - name: Run regular tests + if: ${{ matrix.test_type == 'standard' && matrix.platform.is_musl != true }} + run: | + python -m pip install pytest + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" + pytest tests/ + shell: bash + + # Musl tests + - name: Run musl tests + if: ${{ matrix.test_type == 'standard' && matrix.platform.is_musl == true }} + uses: addnab/docker-run-action@v3 + with: + image: alpine:latest + options: -v ${{ github.workspace }}:/io -w /io + run: | + apk add python3 py3-pip + python -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + WHEEL_PATH=$(ls dist/*.whl) + pip install --force-reinstall "$WHEEL_PATH" + pip install pytest + pytest tests/ + + # Minimum dependencies test + - name: Run tests with minimum dependencies + if: ${{ matrix.test_type == 'min_deps' }} + run: | + python -m pip install pytest==6.0 + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" \ + "numpy==1.20.0" \ + "fsspec==2023.1.0" \ + "s3fs==2023.1.0" \ + "xarray==2023.1.0" + pytest tests/ -v + shell: bash From f1ea37ca26387c5298222ddba653feae04c9cd85 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 15:38:57 +0100 Subject: [PATCH 16/37] potentially fix cache problem on musl --- .github/workflows/build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index ec972c46..5cfeefc2 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -59,7 +59,7 @@ jobs: - uses: actions/setup-python@v5 with: python-version: "3.12" # Use a stable Python version for building - cache: pip + cache: ${{ matrix.platform.is_musl != true && 'pip' || '' }} # Run Rust tests before building - name: Run cargo tests From 0b56a1b0de84669ed00d9c88661f5eb6af77c7cc Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 15:46:54 +0100 Subject: [PATCH 17/37] install python dependencies with maturin develop and then run cargo test after build --- .github/workflows/build.yml | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5cfeefc2..15a5eb6c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,22 +61,28 @@ jobs: python-version: "3.12" # Use a stable Python version for building cache: ${{ matrix.platform.is_musl != true && 'pip' || '' }} - # Run Rust tests before building - - name: Run cargo tests - if: ${{ !matrix.platform.is_musl }} - run: cargo test --no-default-features - shell: bash - - - name: Build wheels + - name: Build wheels and run development install uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter + command: | + # First build wheels + maturin build --release --out dist --find-interpreter + # Then do a development install if not musl and not skipping tests + if [[ "${{ matrix.platform.is_musl }}" != "true" && "${{ matrix.platform.skip_tests }}" != "true" ]]; then + maturin develop --release + fi sccache: "true" manylinux: ${{ matrix.platform.manylinux }} container: ${{ matrix.platform.is_musl && 'off' || '' }} before-script-linux: ${{ matrix.platform.before-script }} + # Run Rust tests after building! This ensures that the python dependencies are installed + - name: Run cargo tests + if: ${{ !matrix.platform.is_musl }} + run: cargo test --no-default-features + shell: bash + - name: Upload wheels uses: actions/upload-artifact@v4 with: From 9f6e20d46de59dd08b2a683c9cd82589daa6f33e Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 16:14:36 +0100 Subject: [PATCH 18/37] explicitly install python dependencies --- .github/workflows/build.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 15a5eb6c..bec3c87d 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,23 +61,21 @@ jobs: python-version: "3.12" # Use a stable Python version for building cache: ${{ matrix.platform.is_musl != true && 'pip' || '' }} - - name: Build wheels and run development install + - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} - command: | - # First build wheels - maturin build --release --out dist --find-interpreter - # Then do a development install if not musl and not skipping tests - if [[ "${{ matrix.platform.is_musl }}" != "true" && "${{ matrix.platform.skip_tests }}" != "true" ]]; then - maturin develop --release - fi + args: --release --out dist --find-interpreter sccache: "true" manylinux: ${{ matrix.platform.manylinux }} container: ${{ matrix.platform.is_musl && 'off' || '' }} before-script-linux: ${{ matrix.platform.before-script }} - # Run Rust tests after building! This ensures that the python dependencies are installed + # Install python dependencies to run cargo tests + - name: Install required python dependencies + run: pip install numpy fsspec s3fs xarray + shell: bash + - name: Run cargo tests if: ${{ !matrix.platform.is_musl }} run: cargo test --no-default-features From 22f59986f8b28491f51bc839cf8748f0d9d3cd2b Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 16:17:37 +0100 Subject: [PATCH 19/37] fix musl --- .github/workflows/build.yml | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index bec3c87d..212c01e8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,18 +61,9 @@ jobs: python-version: "3.12" # Use a stable Python version for building cache: ${{ matrix.platform.is_musl != true && 'pip' || '' }} - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: "true" - manylinux: ${{ matrix.platform.manylinux }} - container: ${{ matrix.platform.is_musl && 'off' || '' }} - before-script-linux: ${{ matrix.platform.before-script }} - # Install python dependencies to run cargo tests - name: Install required python dependencies + if: ${{ !matrix.platform.is_musl }} run: pip install numpy fsspec s3fs xarray shell: bash @@ -81,6 +72,16 @@ jobs: run: cargo test --no-default-features shell: bash + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ matrix.platform.target }} + args: --release --out dist --find-interpreter + sccache: "true" + manylinux: ${{ matrix.platform.manylinux }} + container: ${{ matrix.platform.is_musl && 'off' || '' }} + before-script-linux: ${{ matrix.platform.before-script }} + - name: Upload wheels uses: actions/upload-artifact@v4 with: From 3cb8cca7d1de5615855179411be7aae39f1d3424 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 16:28:28 +0100 Subject: [PATCH 20/37] cache rust and potentially fix test trigger --- .github/workflows/build.yml | 11 +++++++++++ .github/workflows/test.yml | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 212c01e8..e00b01c8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -56,6 +56,17 @@ jobs: steps: - uses: actions/checkout@v4 + # Cache Rust build artifacts + - name: Cache Rust build + if: ${{ !matrix.platform.is_musl }} # Skip for musl due to container limitations + uses: actions/cache@v4 + with: + path: | + target + key: ${{ runner.os }}-build-${{ matrix.platform.name }}-${{ hashFiles('**/Cargo.lock', 'src/**') }} + restore-keys: | + ${{ runner.os }}-build-${{ matrix.platform.name }}- + - uses: actions/setup-python@v5 with: python-version: "3.12" # Use a stable Python version for building diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d25cd3f6..8fbd51ca 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -2,7 +2,7 @@ name: Test on: workflow_run: - workflows: ["Build"] + workflows: ["build.yml"] types: - completed From 74580e99287948edb5efdacea2ee116039ba46d5 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 17:02:22 +0100 Subject: [PATCH 21/37] master ci file again --- .github/workflows/CI.yml | 45 +++++++++++++++++++++++++++++++++++ .github/workflows/build.yml | 13 ++++------ .github/workflows/release.yml | 34 ++++++++------------------ .github/workflows/test.yml | 13 +++++----- 4 files changed, 66 insertions(+), 39 deletions(-) create mode 100644 .github/workflows/CI.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml new file mode 100644 index 00000000..4b4dab9d --- /dev/null +++ b/.github/workflows/CI.yml @@ -0,0 +1,45 @@ +name: CI + +on: + push: + branches: + - main + tags: + - "v*" + pull_request: + workflow_dispatch: + inputs: + publish_to_pypi: + description: "Publish to PyPI" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + +permissions: + contents: read + +jobs: + # Import the build workflow + build: + uses: ./.github/workflows/build.yml + + # Import the test workflow, dependent on build + test: + needs: build + uses: ./.github/workflows/test.yml + with: + build_run_id: ${{ github.run_id }} + + # Import the release workflow, dependent on test + release: + if: ${{ startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_to_pypi == 'true' }} + needs: test + uses: ./.github/workflows/release.yml + with: + publish_to_pypi: ${{ github.event.inputs.publish_to_pypi == 'true' || startsWith(github.ref, 'refs/tags/v') }} + secrets: + PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e00b01c8..388df07c 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -1,13 +1,7 @@ name: Build on: - push: - branches: - - main - tags: - - "*" - pull_request: - workflow_dispatch: + workflow_call: {} # Empty but required to make this a reusable workflow permissions: contents: read @@ -48,6 +42,7 @@ jobs: - name: macos-x86_64 runner: macos-13 target: x86_64 + - name: macos-arm64 runner: macos-14 target: aarch64 @@ -96,7 +91,7 @@ jobs: - name: Upload wheels uses: actions/upload-artifact@v4 with: - name: wheels-${{ matrix.platform.runner }}-${{ matrix.platform.target }} + name: wheels-${{ matrix.platform.name }} path: dist sdist: @@ -112,5 +107,5 @@ jobs: - name: Upload sdist uses: actions/upload-artifact@v4 with: - name: wheels-sdist + name: sdist path: dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index ee202624..3d37c931 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1,19 +1,17 @@ name: Release on: - # push: - # tags: - # - "v*" - workflow_dispatch: + workflow_call: inputs: publish_to_pypi: - description: "Publish to PyPI" required: true - default: "false" - type: choice - options: - - "false" - - "true" + type: boolean + description: "Whether to publish to PyPI" + secrets: + PYPI_API_TOKEN: + required: true + TEST_PYPI_API_TOKEN: + required: true permissions: contents: read @@ -26,21 +24,10 @@ jobs: id-token: write contents: write attestations: write - actions: read steps: - name: Download wheel artifacts uses: actions/download-artifact@v4 - with: - pattern: wheels-* - path: dist - merge-multiple: true - - - name: Download sdist - uses: actions/download-artifact@v4 - with: - name: sdist - path: dist # List files for debugging - name: List distribution files @@ -53,7 +40,7 @@ jobs: # Publish to PyPI when tagged or requested - name: Publish to PyPI - if: startsWith(github.ref, 'refs/tags/') || github.event.inputs.publish_to_pypi == 'true' + if: ${{ inputs.publish_to_pypi }} uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} @@ -63,7 +50,6 @@ jobs: # Otherwise publish to TestPyPI - name: Publish to TestPyPI - if: ${{ !(startsWith(github.ref, 'refs/tags/') || github.event.inputs.publish_to_pypi == 'true') }} uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} @@ -74,7 +60,7 @@ jobs: # # Create a GitHub release when a tag is created # - name: Create GitHub Release - # if: startsWith(github.ref, 'refs/tags/') + # if: ${{ inputs.publish_to_pypi }} # uses: softprops/action-gh-release@v1 # with: # generate_release_notes: true diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8fbd51ca..43862023 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -1,10 +1,12 @@ name: Test on: - workflow_run: - workflows: ["build.yml"] - types: - - completed + workflow_call: + inputs: + build_run_id: + required: true + type: string + description: "The run ID of the build workflow" permissions: contents: read @@ -98,8 +100,7 @@ jobs: with: name: wheels-${{ matrix.platform.name }} path: dist - repository: ${{ github.repository }} - run_id: ${{ github.event.workflow_run.id }} + run_id: ${{ inputs.build_run_id }} github_token: ${{ secrets.GITHUB_TOKEN }} # Regular tests (non-musl platforms) From 3742908caf210723aeb99b4ae1a469717ba2fe91 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 17:07:40 +0100 Subject: [PATCH 22/37] remove yaml anchor --- .github/workflows/test.yml | 12 +----------- 1 file changed, 1 insertion(+), 11 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 43862023..fd9ba913 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -15,8 +15,6 @@ permissions: jobs: test: name: ${{ matrix.name }} - # Only run if the build workflow was successful - if: ${{ github.event.workflow_run.conclusion == 'success' }} runs-on: ${{ matrix.platform.runner }} strategy: fail-fast: false @@ -25,8 +23,7 @@ jobs: test_type: [standard] python-version: ["3.12"] platform: - - &linux_x86_64 - name: linux-x86_64 + - name: linux-x86_64 runner: ubuntu-latest is_musl: false @@ -34,7 +31,6 @@ jobs: include: # Platform tests (uses default Python 3.12 and test_type) - name: "Linux x86_64" - platform: *linux_x86_64 - name: "Linux musl" platform: @@ -62,28 +58,22 @@ jobs: # Additional Python versions for Linux x86_64 - name: "Python 3.8" - platform: *linux_x86_64 python-version: "3.8" - name: "Python 3.9" - platform: *linux_x86_64 python-version: "3.9" - name: "Python 3.10" - platform: *linux_x86_64 python-version: "3.10" - name: "Python 3.11" - platform: *linux_x86_64 python-version: "3.11" - name: "Python 3.13" - platform: *linux_x86_64 python-version: "3.13" # Minimum dependencies test - name: "Min Dependencies" - platform: *linux_x86_64 python-version: "3.8" test_type: min_deps From fc34f02b4c31dd45bdd6e8e24ba78571bf11c5f5 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 18:31:04 +0100 Subject: [PATCH 23/37] use composite actions instead where it makes sense --- .github/actions/test/action.yml | 75 ++++++++++ .github/workflows/CI.yml | 247 +++++++++++++++++++++++++++++--- .github/workflows/build.yml | 111 -------------- .github/workflows/release.yml | 67 --------- .github/workflows/test.yml | 135 ----------------- 5 files changed, 306 insertions(+), 329 deletions(-) create mode 100644 .github/actions/test/action.yml delete mode 100644 .github/workflows/build.yml delete mode 100644 .github/workflows/release.yml delete mode 100644 .github/workflows/test.yml diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml new file mode 100644 index 00000000..0cd06de5 --- /dev/null +++ b/.github/actions/test/action.yml @@ -0,0 +1,75 @@ +name: "Test Package" +description: "Run tests on built wheels" + +inputs: + python-version: + description: "Python version to use for testing" + required: false + default: "3.12" + is-musl: + description: "Whether this is a musl build" + required: false + default: "false" + test-type: + description: "Type of test to run (standard or min_deps)" + required: false + default: "standard" + platform-name: + description: "Platform name for artifact identification" + required: true + +runs: + using: "composite" + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: "pip" + + - name: Download wheels + uses: actions/download-artifact@v4 + with: + name: wheels-${{ inputs.platform-name }} + path: dist + + # Regular tests (non-musl platforms) + - name: Run regular tests + if: ${{ inputs.test_type == 'standard' && inputs.is-musl != true }} + run: | + python -m pip install pytest + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" + pytest tests/ + shell: bash + + # Musl tests + - name: Run musl tests + if: ${{ inputs.test_type == 'standard' && inputs.is-musl == true }} + uses: addnab/docker-run-action@v3 + with: + image: alpine:latest + options: -v ${{ github.workspace }}:/io -w /io + run: | + apk add python3 py3-pip + python -m venv .venv + source .venv/bin/activate + pip install --upgrade pip + WHEEL_PATH=$(ls dist/*.whl) + pip install --force-reinstall "$WHEEL_PATH" + pip install pytest + pytest tests/ + + # Minimum dependencies test + - name: Run tests with minimum dependencies + if: ${{ inputs.test_type == 'min_deps' }} + run: | + python -m pip install pytest==6.0 + WHEEL_PATH=$(ls dist/*.whl) + python -m pip install --force-reinstall "$WHEEL_PATH" \ + "numpy==1.20.0" \ + "fsspec==2023.1.0" \ + "s3fs==2023.1.0" \ + "xarray==2023.1.0" + pytest tests/ -v + shell: bash diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 4b4dab9d..97f24681 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -5,7 +5,7 @@ on: branches: - main tags: - - "v*" + - "*" pull_request: workflow_dispatch: inputs: @@ -22,24 +22,239 @@ permissions: contents: read jobs: - # Import the build workflow build: - uses: ./.github/workflows/build.yml + name: Build - ${{ matrix.platform.name }} + runs-on: ${{ matrix.platform.runner }} + container: ${{ matrix.platform.container }} + strategy: + matrix: + platform: + - name: linux-x86_64 + runner: ubuntu-latest + target: x86_64 + before-script: python3 -m ensurepip && cat /etc/os-release && yum install clang -y + manylinux: "2_28" + is-musl: false + + - name: linux-aarch64 + runner: ubuntu-latest + target: aarch64-unknown-linux-gnu + skip_tests: true + before-script: | + apt-get update && \ + apt-get install --assume-yes --no-install-recommends crossbuild-essential-arm64 + manylinux: "2_28" + is-musl: false + + - name: linux-musl-x86_64 + runner: ubuntu-22.04 + target: x86_64-unknown-linux-musl + container: docker://messense/rust-musl-cross:x86_64-musl + before-script: cat /etc/os-release && apt install clang -y + manylinux: musllinux_1_2 + is-musl: true + + - name: windows-x64 + runner: windows-latest + target: x64 + is-musl: false + + - name: macos-x86_64 + runner: macos-13 + target: x86_64 + is-musl: false + + - name: macos-arm64 + runner: macos-14 + target: aarch64 + is-musl: false + + steps: + # Use the build composite action + - uses: actions/checkout@v4 + + - name: Build wheels + uses: ./.github/actions/build + with: + python-version: "3.12" + target: ${{ matrix.platform.target }} + manylinux: ${{ matrix.platform.manylinux }} + is-musl: ${{ matrix.platform.is-musl }} + before-script: ${{ matrix.platform.before-script }} + platform-name: ${{ matrix.platform.name }} + + sdist: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Build sdist + uses: PyO3/maturin-action@v1 + with: + command: sdist + args: --out dist + - name: Upload sdist + uses: actions/upload-artifact@v4 + with: + name: wheels-sdist + path: dist - # Import the test workflow, dependent on build test: needs: build - uses: ./.github/workflows/test.yml - with: - build_run_id: ${{ github.run_id }} + name: Test - ${{ matrix.config.name }} + runs-on: ${{ matrix.config.runner }} + strategy: + fail-fast: false + matrix: + config: + # Platform tests with Python 3.12 + - name: Linux x86_64 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.12" + test_type: standard + + - name: Linux musl + runner: ubuntu-22.04 + platform-name: linux-musl-x86_64 + is-musl: true + python_version: "3.12" + test_type: standard + + - name: Windows + runner: windows-latest + platform-name: windows-x64 + is-musl: false + python_version: "3.12" + test_type: standard + + - name: macOS x86_64 + runner: macos-13 + platform-name: macos-x86_64 + is-musl: false + python_version: "3.12" + test_type: standard + + - name: macOS ARM64 + runner: macos-14 + platform-name: macos-arm64 + is-musl: false + python_version: "3.12" + test_type: standard + + # Additional Python versions for Linux x86_64 + - name: Python 3.8 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.8" + test_type: standard - # Import the release workflow, dependent on test + - name: Python 3.9 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.9" + test_type: standard + + - name: Python 3.10 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.10" + test_type: standard + + - name: Python 3.11 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.11" + test_type: standard + + - name: Python 3.13 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.13" + test_type: standard + + # Minimum dependencies test + - name: Min Dependencies + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python_version: "3.8" + test_type: min_deps + steps: + # Use the test composite action + - uses: actions/checkout@v4 + + - name: Run tests + uses: ./.github/actions/test + with: + python-version: ${{ matrix.config.python_version }} + is-musl: ${{ matrix.config.is-musl }} + test-type: ${{ matrix.config.test_type }} + platform-name: ${{ matrix.config.platform-name }} + + # Release workflow, dependent on both test and sdist release: - if: ${{ startsWith(github.ref, 'refs/tags/v') || github.event.inputs.publish_to_pypi == 'true' }} - needs: test - uses: ./.github/workflows/release.yml - with: - publish_to_pypi: ${{ github.event.inputs.publish_to_pypi == 'true' || startsWith(github.ref, 'refs/tags/v') }} - secrets: - PYPI_API_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - TEST_PYPI_API_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: [test, sdist] + name: Release Package + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + attestations: write + steps: + - name: Download all wheel artifacts + uses: actions/download-artifact@v4 + with: + path: artifact-download + + # Create dist directory + - name: Create dist directory + run: mkdir -p dist + shell: bash + + # Move all wheels to dist directory + - name: Collect wheels + run: | + find artifact-download -name "*.whl" -o -name "*.tar.gz" | xargs -I{} cp {} dist/ + echo "Package files in dist directory:" + ls -la dist/ + shell: bash + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: "dist/*" + + # Publish to PyPI when tagged or requested + - name: Publish to PyPI + if: ${{ github.event.inputs.publish_to_pypi }} + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --non-interactive --skip-existing dist/* + + # Otherwise publish to TestPyPI + - name: Publish to TestPyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + MATURIN_REPOSITORY: testpypi + with: + command: upload + args: --non-interactive --skip-existing dist/* + + # # Create a GitHub release when a tag is created + # - name: Create GitHub Release + # if: ${{ inputs.publish_to_pypi }} + # uses: softprops/action-gh-release@v1 + # with: + # generate_release_notes: true + # files: dist/* diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml deleted file mode 100644 index 388df07c..00000000 --- a/.github/workflows/build.yml +++ /dev/null @@ -1,111 +0,0 @@ -name: Build - -on: - workflow_call: {} # Empty but required to make this a reusable workflow - -permissions: - contents: read - -jobs: - build: - runs-on: ${{ matrix.platform.runner }} - strategy: - matrix: - platform: - - name: linux-x86_64 - runner: ubuntu-latest - target: x86_64 - before-script: python3 -m ensurepip && cat /etc/os-release && yum install clang -y - manylinux: "2_28" - - - name: linux-aarch64 - runner: ubuntu-latest - target: aarch64-unknown-linux-gnu - skip_tests: true - before-script: | - apt-get update && \ - apt-get install --assume-yes --no-install-recommends crossbuild-essential-arm64 - manylinux: "2_28" - - - name: linux-musl-x86_64 - runner: ubuntu-22.04 - target: x86_64-unknown-linux-musl - container: docker://messense/rust-musl-cross:x86_64-musl - before-script: cat /etc/os-release && apt install clang -y - manylinux: musllinux_1_2 - is_musl: true - - - name: windows-x64 - runner: windows-latest - target: x64 - - - name: macos-x86_64 - runner: macos-13 - target: x86_64 - - - name: macos-arm64 - runner: macos-14 - target: aarch64 - - container: ${{ matrix.platform.container }} - steps: - - uses: actions/checkout@v4 - - # Cache Rust build artifacts - - name: Cache Rust build - if: ${{ !matrix.platform.is_musl }} # Skip for musl due to container limitations - uses: actions/cache@v4 - with: - path: | - target - key: ${{ runner.os }}-build-${{ matrix.platform.name }}-${{ hashFiles('**/Cargo.lock', 'src/**') }} - restore-keys: | - ${{ runner.os }}-build-${{ matrix.platform.name }}- - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" # Use a stable Python version for building - cache: ${{ matrix.platform.is_musl != true && 'pip' || '' }} - - # Install python dependencies to run cargo tests - - name: Install required python dependencies - if: ${{ !matrix.platform.is_musl }} - run: pip install numpy fsspec s3fs xarray - shell: bash - - - name: Run cargo tests - if: ${{ !matrix.platform.is_musl }} - run: cargo test --no-default-features - shell: bash - - - name: Build wheels - uses: PyO3/maturin-action@v1 - with: - target: ${{ matrix.platform.target }} - args: --release --out dist --find-interpreter - sccache: "true" - manylinux: ${{ matrix.platform.manylinux }} - container: ${{ matrix.platform.is_musl && 'off' || '' }} - before-script-linux: ${{ matrix.platform.before-script }} - - - name: Upload wheels - uses: actions/upload-artifact@v4 - with: - name: wheels-${{ matrix.platform.name }} - path: dist - - sdist: - name: Build Source Distribution - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - name: Build sdist - uses: PyO3/maturin-action@v1 - with: - command: sdist - args: --out dist - - name: Upload sdist - uses: actions/upload-artifact@v4 - with: - name: sdist - path: dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml deleted file mode 100644 index 3d37c931..00000000 --- a/.github/workflows/release.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Release - -on: - workflow_call: - inputs: - publish_to_pypi: - required: true - type: boolean - description: "Whether to publish to PyPI" - secrets: - PYPI_API_TOKEN: - required: true - TEST_PYPI_API_TOKEN: - required: true - -permissions: - contents: read - -jobs: - publish: - name: Publish Package - runs-on: ubuntu-latest - permissions: - id-token: write - contents: write - attestations: write - - steps: - - name: Download wheel artifacts - uses: actions/download-artifact@v4 - - # List files for debugging - - name: List distribution files - run: ls -la dist/ - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-path: "dist/*" - - # Publish to PyPI when tagged or requested - - name: Publish to PyPI - if: ${{ inputs.publish_to_pypi }} - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - with: - command: upload - args: --non-interactive --skip-existing dist/* - - # Otherwise publish to TestPyPI - - name: Publish to TestPyPI - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - MATURIN_REPOSITORY: testpypi - with: - command: upload - args: --non-interactive --skip-existing dist/* - - # # Create a GitHub release when a tag is created - # - name: Create GitHub Release - # if: ${{ inputs.publish_to_pypi }} - # uses: softprops/action-gh-release@v1 - # with: - # generate_release_notes: true - # files: dist/* diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index fd9ba913..00000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,135 +0,0 @@ -name: Test - -on: - workflow_call: - inputs: - build_run_id: - required: true - type: string - description: "The run ID of the build workflow" - -permissions: - contents: read - actions: read # Needed to download artifacts - -jobs: - test: - name: ${{ matrix.name }} - runs-on: ${{ matrix.platform.runner }} - strategy: - fail-fast: false - matrix: - # Default values for all matrix entries - test_type: [standard] - python-version: ["3.12"] - platform: - - name: linux-x86_64 - runner: ubuntu-latest - is_musl: false - - # Override with specific combinations - include: - # Platform tests (uses default Python 3.12 and test_type) - - name: "Linux x86_64" - - - name: "Linux musl" - platform: - name: linux-musl-x86_64 - runner: ubuntu-22.04 - is_musl: true - - - name: "Windows" - platform: - name: windows-x64 - runner: windows-latest - is_musl: false - - - name: "macOS x86_64" - platform: - name: macos-x86_64 - runner: macos-13 - is_musl: false - - - name: "macOS ARM64" - platform: - name: macos-arm64 - runner: macos-14 - is_musl: false - - # Additional Python versions for Linux x86_64 - - name: "Python 3.8" - python-version: "3.8" - - - name: "Python 3.9" - python-version: "3.9" - - - name: "Python 3.10" - python-version: "3.10" - - - name: "Python 3.11" - python-version: "3.11" - - - name: "Python 3.13" - python-version: "3.13" - - # Minimum dependencies test - - name: "Min Dependencies" - python-version: "3.8" - test_type: min_deps - - steps: - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - cache: "pip" - - - name: Download wheel artifacts - uses: actions/download-artifact@v4 - with: - name: wheels-${{ matrix.platform.name }} - path: dist - run_id: ${{ inputs.build_run_id }} - github_token: ${{ secrets.GITHUB_TOKEN }} - - # Regular tests (non-musl platforms) - - name: Run regular tests - if: ${{ matrix.test_type == 'standard' && matrix.platform.is_musl != true }} - run: | - python -m pip install pytest - WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" - pytest tests/ - shell: bash - - # Musl tests - - name: Run musl tests - if: ${{ matrix.test_type == 'standard' && matrix.platform.is_musl == true }} - uses: addnab/docker-run-action@v3 - with: - image: alpine:latest - options: -v ${{ github.workspace }}:/io -w /io - run: | - apk add python3 py3-pip - python -m venv .venv - source .venv/bin/activate - pip install --upgrade pip - WHEEL_PATH=$(ls dist/*.whl) - pip install --force-reinstall "$WHEEL_PATH" - pip install pytest - pytest tests/ - - # Minimum dependencies test - - name: Run tests with minimum dependencies - if: ${{ matrix.test_type == 'min_deps' }} - run: | - python -m pip install pytest==6.0 - WHEEL_PATH=$(ls dist/*.whl) - python -m pip install --force-reinstall "$WHEEL_PATH" \ - "numpy==1.20.0" \ - "fsspec==2023.1.0" \ - "s3fs==2023.1.0" \ - "xarray==2023.1.0" - pytest tests/ -v - shell: bash From 607a9854c68f5350dd4d0c1ea08d1eafef9b68c3 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 18:33:45 +0100 Subject: [PATCH 24/37] rename --- .github/actions/build_wheels/action.yml | 71 +++++++++++++++++++++++++ .github/workflows/CI.yml | 2 +- 2 files changed, 72 insertions(+), 1 deletion(-) create mode 100644 .github/actions/build_wheels/action.yml diff --git a/.github/actions/build_wheels/action.yml b/.github/actions/build_wheels/action.yml new file mode 100644 index 00000000..4d8ff9c9 --- /dev/null +++ b/.github/actions/build_wheels/action.yml @@ -0,0 +1,71 @@ +name: "Build Package" +description: "Build wheels for all platforms" + +inputs: + python-version: + description: "Python version to use for building" + required: false + default: "3.12" + target: + description: "Target architecture" + required: true + manylinux: + description: "Manylinux version" + required: false + default: "" + is-musl: + description: "Whether this is a musl build" + required: false + default: "false" + before-script: + description: "Script to run before building" + required: false + default: "" + platform-name: + description: "Platform name for artifact identification" + required: true + +runs: + using: "composite" + steps: + # Cache Rust build artifacts + - name: Cache Rust build + uses: actions/cache@v4 + with: + path: | + target + key: ${{ runner.os }}-build-${{ inputs.platform-name }}-${{ hashFiles('**/Cargo.lock', 'src/**') }} + restore-keys: | + ${{ runner.os }}-build-${{ inputs.platform-name }}- + + - uses: actions/setup-python@v5 + with: + python-version: ${{ inputs.python-version }} + cache: ${{ inputs.is-musl != 'true' && 'pip' || '' }} + + # Install python dependencies to run cargo tests + - name: Install required python dependencies + if: ${{ inputs.is-musl != 'true' }} + run: pip install numpy fsspec s3fs xarray + shell: bash + + - name: Run cargo tests + if: ${{ inputs.is-musl != 'true' }} + run: cargo test --no-default-features + shell: bash + + - name: Build wheels + uses: PyO3/maturin-action@v1 + with: + target: ${{ inputs.target }} + args: --release --out dist --find-interpreter + sccache: "true" + manylinux: ${{ inputs.manylinux }} + container: ${{ inputs.is-musl == 'true' && 'off' || '' }} + before-script-linux: ${{ inputs.before-script }} + + - name: Upload wheels + uses: actions/upload-artifact@v4 + with: + name: wheels-${{ inputs.platform-name }} + path: dist diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 97f24681..f1639763 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -74,7 +74,7 @@ jobs: - uses: actions/checkout@v4 - name: Build wheels - uses: ./.github/actions/build + uses: ./.github/actions/build_wheels with: python-version: "3.12" target: ${{ matrix.platform.target }} From 20894da7f41e294a7f251cd0cee0b53d4ae83643 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 18:40:32 +0100 Subject: [PATCH 25/37] remove pip cache --- .github/actions/test/action.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 0cd06de5..0e88194f 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -25,7 +25,6 @@ runs: - uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} - cache: "pip" - name: Download wheels uses: actions/download-artifact@v4 From cee69291efcdad59e3fe7b9a3f7ba456c0ea198a Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 18:50:52 +0100 Subject: [PATCH 26/37] cleanup --- .github/actions/test/action.yml | 6 +-- .github/workflows/CI.yml | 68 +++++++++------------------------ 2 files changed, 22 insertions(+), 52 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 0e88194f..051b2052 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -34,7 +34,7 @@ runs: # Regular tests (non-musl platforms) - name: Run regular tests - if: ${{ inputs.test_type == 'standard' && inputs.is-musl != true }} + if: ${{ inputs.test-type == 'standard' && inputs.is-musl != true }} run: | python -m pip install pytest WHEEL_PATH=$(ls dist/*.whl) @@ -44,7 +44,7 @@ runs: # Musl tests - name: Run musl tests - if: ${{ inputs.test_type == 'standard' && inputs.is-musl == true }} + if: ${{ inputs.test-type == 'standard' && inputs.is-musl == true }} uses: addnab/docker-run-action@v3 with: image: alpine:latest @@ -61,7 +61,7 @@ runs: # Minimum dependencies test - name: Run tests with minimum dependencies - if: ${{ inputs.test_type == 'min_deps' }} + if: ${{ inputs.test-type == 'min_deps' }} run: | python -m pip install pytest==6.0 WHEEL_PATH=$(ls dist/*.whl) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index f1639763..2c9ae0bb 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -105,86 +105,56 @@ jobs: strategy: fail-fast: false matrix: - config: + # Define default values + runner: [ubuntu-latest] + platform-name: [linux-x86_64] + is-musl: [false] + python-version: ["3.12"] + test-type: [standard] + + # Define the matrix combinations + include: # Platform tests with Python 3.12 - name: Linux x86_64 - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.12" - test_type: standard + # Using defaults - name: Linux musl runner: ubuntu-22.04 platform-name: linux-musl-x86_64 is-musl: true - python_version: "3.12" - test_type: standard - name: Windows runner: windows-latest platform-name: windows-x64 - is-musl: false - python_version: "3.12" - test_type: standard - name: macOS x86_64 runner: macos-13 platform-name: macos-x86_64 - is-musl: false - python_version: "3.12" - test_type: standard - name: macOS ARM64 runner: macos-14 platform-name: macos-arm64 - is-musl: false - python_version: "3.12" - test_type: standard # Additional Python versions for Linux x86_64 - name: Python 3.8 - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.8" - test_type: standard + python-version: "3.8" - name: Python 3.9 - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.9" - test_type: standard + python-version: "3.9" - name: Python 3.10 - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.10" - test_type: standard + python-version: "3.10" - name: Python 3.11 - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.11" - test_type: standard + python-version: "3.11" - name: Python 3.13 - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.13" - test_type: standard + python-version: "3.13" # Minimum dependencies test - name: Min Dependencies - runner: ubuntu-latest - platform-name: linux-x86_64 - is-musl: false - python_version: "3.8" - test_type: min_deps + python-version: "3.8" + test-type: min_deps steps: # Use the test composite action - uses: actions/checkout@v4 @@ -192,9 +162,9 @@ jobs: - name: Run tests uses: ./.github/actions/test with: - python-version: ${{ matrix.config.python_version }} + python-version: ${{ matrix.config.python-version }} is-musl: ${{ matrix.config.is-musl }} - test-type: ${{ matrix.config.test_type }} + test-type: ${{ matrix.config.test-type }} platform-name: ${{ matrix.config.platform-name }} # Release workflow, dependent on both test and sdist From f6adb2fd7752664c2d468e57d380ed61e043c962 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 18:52:03 +0100 Subject: [PATCH 27/37] version 0.0.1 --- .github/actions/test/action.yml | 4 +- .github/workflows/CI.yml | 66 ++++++++++++++++++++++++--------- Cargo.lock | 2 +- Cargo.toml | 2 +- 4 files changed, 52 insertions(+), 22 deletions(-) diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml index 051b2052..188885ff 100644 --- a/.github/actions/test/action.yml +++ b/.github/actions/test/action.yml @@ -34,7 +34,7 @@ runs: # Regular tests (non-musl platforms) - name: Run regular tests - if: ${{ inputs.test-type == 'standard' && inputs.is-musl != true }} + if: ${{ inputs.test-type == 'standard' && inputs.is-musl != 'true' }} run: | python -m pip install pytest WHEEL_PATH=$(ls dist/*.whl) @@ -44,7 +44,7 @@ runs: # Musl tests - name: Run musl tests - if: ${{ inputs.test-type == 'standard' && inputs.is-musl == true }} + if: ${{ inputs.test-type == 'standard' && inputs.is-musl == 'true' }} uses: addnab/docker-run-action@v3 with: image: alpine:latest diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index 2c9ae0bb..a555ce2b 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -105,54 +105,84 @@ jobs: strategy: fail-fast: false matrix: - # Define default values - runner: [ubuntu-latest] - platform-name: [linux-x86_64] - is-musl: [false] - python-version: ["3.12"] - test-type: [standard] - - # Define the matrix combinations - include: + config: # Platform tests with Python 3.12 - name: Linux x86_64 - # Using defaults + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false + python-version: "3.12" + test-type: standard - name: Linux musl runner: ubuntu-22.04 platform-name: linux-musl-x86_64 is-musl: true + python-version: "3.12" + test-type: standard - name: Windows runner: windows-latest platform-name: windows-x64 + is-musl: false + python-version: "3.12" + test-type: standard - name: macOS x86_64 runner: macos-13 platform-name: macos-x86_64 + is-musl: false + python-version: "3.12" + test-type: standard - name: macOS ARM64 runner: macos-14 platform-name: macos-arm64 + is-musl: false + python-version: "3.12" + test-type: standard # Additional Python versions for Linux x86_64 - name: Python 3.8 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false python-version: "3.8" + test-type: standard - name: Python 3.9 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false python-version: "3.9" + test-type: standard - name: Python 3.10 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false python-version: "3.10" + test-type: standard - name: Python 3.11 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false python-version: "3.11" + test-type: standard - name: Python 3.13 + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false python-version: "3.13" + test-type: standard # Minimum dependencies test - name: Min Dependencies + runner: ubuntu-latest + platform-name: linux-x86_64 + is-musl: false python-version: "3.8" test-type: min_deps steps: @@ -201,22 +231,22 @@ jobs: with: subject-path: "dist/*" - # Publish to PyPI when tagged or requested - - name: Publish to PyPI - if: ${{ github.event.inputs.publish_to_pypi }} + # Publish to TestPyPI + - name: Publish to TestPyPI uses: PyO3/maturin-action@v1 env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + MATURIN_REPOSITORY: testpypi with: command: upload args: --non-interactive --skip-existing dist/* - # Otherwise publish to TestPyPI - - name: Publish to TestPyPI + # Only publish to PyPI if the publish_to_pypi input is true + - name: Publish to PyPI + if: ${{ github.event.inputs.publish_to_pypi }} uses: PyO3/maturin-action@v1 env: - MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - MATURIN_REPOSITORY: testpypi + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} with: command: upload args: --non-interactive --skip-existing dist/* diff --git a/Cargo.lock b/Cargo.lock index d9582018..1a80671a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "omfilesrspy" -version = "0.1.0" +version = "0.0.1" dependencies = [ "delegate", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index 44edec8f..d3cd2952 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omfilesrspy" -version = "0.1.0" +version = "0.0.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 64444b9ca5472447a77641e0d9a76fa3741937b9 Mon Sep 17 00:00:00 2001 From: terraputix Date: Wed, 5 Mar 2025 22:11:22 +0100 Subject: [PATCH 28/37] version 0.0.2 --- .github/workflows/CI.yml | 2 +- Cargo.lock | 2 +- Cargo.toml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/CI.yml b/.github/workflows/CI.yml index a555ce2b..ba812f87 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/CI.yml @@ -243,7 +243,7 @@ jobs: # Only publish to PyPI if the publish_to_pypi input is true - name: Publish to PyPI - if: ${{ github.event.inputs.publish_to_pypi }} + if: ${{ github.event.inputs.publish_to_pypi == 'true' }} uses: PyO3/maturin-action@v1 env: MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} diff --git a/Cargo.lock b/Cargo.lock index 1a80671a..ad8ab70e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "omfilesrspy" -version = "0.0.1" +version = "0.0.2" dependencies = [ "delegate", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index d3cd2952..f005b2b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "omfilesrspy" -version = "0.0.1" +version = "0.0.2" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html From 05f9be6b1aebf3e962613aa6e53bdc5a29759f07 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 12:44:30 +0100 Subject: [PATCH 29/37] rename package --- .gitignore | 3 +++ Cargo.lock | 2 +- Cargo.toml | 4 ++-- README.md | 6 +++--- benchmarks/helpers/args.py | 2 +- benchmarks/helpers/io/readers.py | 4 ++-- benchmarks/helpers/io/writers.py | 2 +- pyproject.toml | 4 ++-- python/{omfilesrspy => pyomfiles}/__init__.py | 2 +- python/{omfilesrspy => pyomfiles}/py.typed | 0 .../omfilesrspy.pyi => pyomfiles/pyomfiles.pyi} | 0 python/{omfilesrspy => pyomfiles}/types.py | 0 python/{omfilesrspy => pyomfiles}/xarray_backend.py | 2 +- src/lib.rs | 2 +- tests/test_fsspec.py | 8 ++++---- tests/test_omfilesrspy.py | 12 ++++++------ tests/test_utils.py | 2 +- tests/test_xarray.py | 4 ++-- 18 files changed, 31 insertions(+), 28 deletions(-) rename python/{omfilesrspy => pyomfiles}/__init__.py (66%) rename python/{omfilesrspy => pyomfiles}/py.typed (100%) rename python/{omfilesrspy/omfilesrspy.pyi => pyomfiles/pyomfiles.pyi} (100%) rename python/{omfilesrspy => pyomfiles}/types.py (100%) rename python/{omfilesrspy => pyomfiles}/xarray_backend.py (99%) diff --git a/.gitignore b/.gitignore index 107d90b0..e809d8d0 100644 --- a/.gitignore +++ b/.gitignore @@ -79,3 +79,6 @@ docs/_build/ # Pyenv .python-version + +# CSpell Spell Checker +cspell.json diff --git a/Cargo.lock b/Cargo.lock index ad8ab70e..1a80671a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -325,7 +325,7 @@ dependencies = [ [[package]] name = "omfilesrspy" -version = "0.0.2" +version = "0.0.1" dependencies = [ "delegate", "num-traits", diff --git a/Cargo.toml b/Cargo.toml index f005b2b9..634102f4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,11 @@ [package] name = "omfilesrspy" -version = "0.0.2" +version = "0.0.1" edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] -name = "omfilesrspy" +name = "pyomfiles" crate-type = ["cdylib"] [dependencies] diff --git a/README.md b/README.md index 6eeb16ca..1f00c353 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ OM files are [structured like a tree of variables](https://github.com/open-meteo/om-file-format?tab=readme-ov-file#data-hierarchy-model). The following example assumes that the file `test_file.om` contains an array variable as a root variable which has a dimensionality greater than 2 and a size of at least 2x100: ```python -from omfilesrspy import OmFilePyReader +from pyomfiles import OmFilePyReader reader = OmFilePyReader("test_file.om") data = reader[0:2, 0:100, ...] @@ -29,7 +29,7 @@ data = reader[0:2, 0:100, ...] #### Simple Array ```python import numpy as np -from omfilesrspy import OmFilePyWriter +from pyomfiles import OmFilePyWriter # Create sample data data = np.random.rand(100, 100).astype(np.float32) @@ -54,7 +54,7 @@ writer.close(variable) #### Hierarchical Structure ```python import numpy as np -from omfilesrspy import OmFilePyWriter +from pyomfiles import OmFilePyWriter # Create sample data features = np.random.rand(1000, 64).astype(np.float32) diff --git a/benchmarks/helpers/args.py b/benchmarks/helpers/args.py index b62b12bb..c2185ce1 100644 --- a/benchmarks/helpers/args.py +++ b/benchmarks/helpers/args.py @@ -1,6 +1,6 @@ import argparse -from omfilesrspy.types import BasicSelection +from pyomfiles.types import BasicSelection def parse_args() -> argparse.Namespace: diff --git a/benchmarks/helpers/io/readers.py b/benchmarks/helpers/io/readers.py index d49b4061..a01a1feb 100644 --- a/benchmarks/helpers/io/readers.py +++ b/benchmarks/helpers/io/readers.py @@ -4,10 +4,10 @@ import h5py import netCDF4 as nc import numpy as np -import omfilesrspy as om +import pyomfiles as om import xarray as xr import zarr -from omfilesrspy.types import BasicSelection +from pyomfiles.types import BasicSelection class BaseReader(ABC): diff --git a/benchmarks/helpers/io/writers.py b/benchmarks/helpers/io/writers.py index 36d47777..bc73f44c 100644 --- a/benchmarks/helpers/io/writers.py +++ b/benchmarks/helpers/io/writers.py @@ -5,7 +5,7 @@ import h5py import netCDF4 as nc import numpy as np -import omfilesrspy as om +import pyomfiles as om import zarr from zarr.core.buffer import NDArrayLike diff --git a/pyproject.toml b/pyproject.toml index bbcf0494..f922a924 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["maturin>=1.7,<2.0"] build-backend = "maturin" [project] -name = "omfilesrspy" +name = "pyomfiles" requires-python = ">=3.8" classifiers = [ "Programming Language :: Rust", @@ -27,4 +27,4 @@ bindings = "pyo3" # pyo3 bindings is actually the default for maturin dev = ["pytest>=6.0", "hidefix", "h5py", "netCDF4", "zarr"] [project.entry-points."xarray.backends"] -om = "omfilesrspy.xarray_backend:OmXarrayEntrypoint" +om = "pyomfiles.xarray_backend:OmXarrayEntrypoint" diff --git a/python/omfilesrspy/__init__.py b/python/pyomfiles/__init__.py similarity index 66% rename from python/omfilesrspy/__init__.py rename to python/pyomfiles/__init__.py index c4cc74a0..23c559ad 100644 --- a/python/omfilesrspy/__init__.py +++ b/python/pyomfiles/__init__.py @@ -1,4 +1,4 @@ from . import types, xarray_backend -from .omfilesrspy import OmFilePyReader, OmFilePyWriter +from .pyomfiles import OmFilePyReader, OmFilePyWriter __all__ = ["OmFilePyReader", "OmFilePyWriter", "xarray_backend", "types"] diff --git a/python/omfilesrspy/py.typed b/python/pyomfiles/py.typed similarity index 100% rename from python/omfilesrspy/py.typed rename to python/pyomfiles/py.typed diff --git a/python/omfilesrspy/omfilesrspy.pyi b/python/pyomfiles/pyomfiles.pyi similarity index 100% rename from python/omfilesrspy/omfilesrspy.pyi rename to python/pyomfiles/pyomfiles.pyi diff --git a/python/omfilesrspy/types.py b/python/pyomfiles/types.py similarity index 100% rename from python/omfilesrspy/types.py rename to python/pyomfiles/types.py diff --git a/python/omfilesrspy/xarray_backend.py b/python/pyomfiles/xarray_backend.py similarity index 99% rename from python/omfilesrspy/xarray_backend.py rename to python/pyomfiles/xarray_backend.py index 3c4803b1..24a79286 100644 --- a/python/omfilesrspy/xarray_backend.py +++ b/python/pyomfiles/xarray_backend.py @@ -7,7 +7,7 @@ from xarray.core.utils import FrozenDict from xarray.core.variable import Variable -from .omfilesrspy import OmFilePyReader, OmVariable +from .pyomfiles import OmFilePyReader, OmVariable # need some special secret attributes to tell us the dimensions DIMENSION_KEY = "_ARRAY_DIMENSIONS" diff --git a/src/lib.rs b/src/lib.rs index c912e3b0..dc0de855 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ mod writer; /// A Python module implemented in Rust. #[pymodule(gil_used = false)] -fn omfilesrspy<'py>(m: &Bound<'py, PyModule>) -> PyResult<()> { +fn pyomfiles<'py>(m: &Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 2ac6b210..306fd672 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -1,11 +1,11 @@ import fsspec import numpy as np -import omfilesrspy +import pyomfiles # def test_fsspec_backend(): # fsspec_object = fsspec.open("test_files/read_test.om", "rb") -# file = omfilesrspy.FsSpecBackend(fsspec_object) +# file = pyomfiles.FsSpecBackend(fsspec_object) # assert file.file_size == 144 @@ -15,7 +15,7 @@ def test_s3_reader(): backend = fs.open(file_path, mode="rb") # Create reader over fs spec backend - reader = omfilesrspy.OmFilePyReader(backend) + reader = pyomfiles.OmFilePyReader(backend) data = reader[57812:57813, 0:100] # Verify the data @@ -29,7 +29,7 @@ def test_s3_reader_with_cache(): backend = fs.open(file_path, mode="rb", cache_type="mmap", block_size=1024, cache_options={"location": "cache"}) # Create reader over fs spec backend - reader = omfilesrspy.OmFilePyReader(backend) + reader = pyomfiles.OmFilePyReader(backend) data = reader[57812:57813, 0:100] # Verify the data diff --git a/tests/test_omfilesrspy.py b/tests/test_omfilesrspy.py index bc197cd6..dd1a5d24 100644 --- a/tests/test_omfilesrspy.py +++ b/tests/test_omfilesrspy.py @@ -1,7 +1,7 @@ import os import numpy as np -import omfilesrspy +import pyomfiles from .test_utils import create_test_om_file @@ -12,7 +12,7 @@ def test_write_om_roundtrip(): try: create_test_om_file(temp_file) - reader = omfilesrspy.OmFilePyReader(temp_file) + reader = pyomfiles.OmFilePyReader(temp_file) data = reader[0:5, 0:5] del reader @@ -54,14 +54,14 @@ def test_round_trip_array_datatypes(): try: # Write data - writer = omfilesrspy.OmFilePyWriter(temp_file) + writer = pyomfiles.OmFilePyWriter(temp_file) variable = writer.write_array(test_data, chunks=chunks, scale_factor=10000.0, add_offset=0.0) writer.close(variable) del writer # Read data back - reader = omfilesrspy.OmFilePyReader(temp_file) + reader = pyomfiles.OmFilePyReader(temp_file) read_data = reader[:] del reader @@ -85,7 +85,7 @@ def test_write_hierarchical_file(): child2_data = np.random.rand(3, 3).astype(np.float32) # Write hierarchical structure - writer = omfilesrspy.OmFilePyWriter(temp_file) + writer = pyomfiles.OmFilePyWriter(temp_file) # Write child2 array child2_var = writer.write_array( @@ -123,7 +123,7 @@ def test_write_hierarchical_file(): del writer # Read and verify the data using OmFilePyReader - reader = omfilesrspy.OmFilePyReader(temp_file) + reader = pyomfiles.OmFilePyReader(temp_file) # Verify root data read_root = reader[:] diff --git a/tests/test_utils.py b/tests/test_utils.py index d8107b5e..fd7e5a34 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ from __future__ import annotations import numpy as np -from omfilesrspy import OmFilePyWriter +from pyomfiles import OmFilePyWriter def create_test_om_file(filename: str = "test_file.om", shape=(5, 5), dtype=np.float32) -> tuple[str, np.ndarray]: diff --git a/tests/test_xarray.py b/tests/test_xarray.py index 06d0b32d..c879c691 100644 --- a/tests/test_xarray.py +++ b/tests/test_xarray.py @@ -7,8 +7,8 @@ import warnings import numpy as np -import omfilesrspy.omfilesrspy as om -import omfilesrspy.xarray_backend as om_xarray +import pyomfiles.pyomfiles as om +import pyomfiles.xarray_backend as om_xarray import xarray as xr from xarray.core import indexing From b643dd824feec5d4cb8066dc0425b0a2fa4f6fbf Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 13:14:25 +0100 Subject: [PATCH 30/37] separate testing and publish workflows --- .github/workflows/{CI.yml => build-test.yml} | 73 +------------ .github/workflows/publish.yml | 102 +++++++++++++++++++ 2 files changed, 103 insertions(+), 72 deletions(-) rename .github/workflows/{CI.yml => build-test.yml} (71%) create mode 100644 .github/workflows/publish.yml diff --git a/.github/workflows/CI.yml b/.github/workflows/build-test.yml similarity index 71% rename from .github/workflows/CI.yml rename to .github/workflows/build-test.yml index ba812f87..49b22508 100644 --- a/.github/workflows/CI.yml +++ b/.github/workflows/build-test.yml @@ -1,4 +1,4 @@ -name: CI +name: Build and Test on: push: @@ -8,15 +8,6 @@ on: - "*" pull_request: workflow_dispatch: - inputs: - publish_to_pypi: - description: "Publish to PyPI" - required: true - default: "false" - type: choice - options: - - "false" - - "true" permissions: contents: read @@ -196,65 +187,3 @@ jobs: is-musl: ${{ matrix.config.is-musl }} test-type: ${{ matrix.config.test-type }} platform-name: ${{ matrix.config.platform-name }} - - # Release workflow, dependent on both test and sdist - release: - if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: [test, sdist] - name: Release Package - runs-on: ubuntu-latest - permissions: - id-token: write - contents: write - attestations: write - steps: - - name: Download all wheel artifacts - uses: actions/download-artifact@v4 - with: - path: artifact-download - - # Create dist directory - - name: Create dist directory - run: mkdir -p dist - shell: bash - - # Move all wheels to dist directory - - name: Collect wheels - run: | - find artifact-download -name "*.whl" -o -name "*.tar.gz" | xargs -I{} cp {} dist/ - echo "Package files in dist directory:" - ls -la dist/ - shell: bash - - - name: Generate artifact attestation - uses: actions/attest-build-provenance@v1 - with: - subject-path: "dist/*" - - # Publish to TestPyPI - - name: Publish to TestPyPI - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} - MATURIN_REPOSITORY: testpypi - with: - command: upload - args: --non-interactive --skip-existing dist/* - - # Only publish to PyPI if the publish_to_pypi input is true - - name: Publish to PyPI - if: ${{ github.event.inputs.publish_to_pypi == 'true' }} - uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} - with: - command: upload - args: --non-interactive --skip-existing dist/* - - # # Create a GitHub release when a tag is created - # - name: Create GitHub Release - # if: ${{ inputs.publish_to_pypi }} - # uses: softprops/action-gh-release@v1 - # with: - # generate_release_notes: true - # files: dist/* diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 00000000..ac5f1fb3 --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,102 @@ +name: Publish + +on: + workflow_dispatch: + inputs: + publish_to_pypi: + description: "Publish to PyPI" + required: true + default: "false" + type: choice + options: + - "false" + - "true" + +jobs: + # First check if CI workflow succeeded + check-ci: + runs-on: ubuntu-latest + steps: + - name: Check CI workflow status + uses: actions/github-script@v7 + id: check-status + with: + script: | + const ref = context.sha; + const workflow_name = 'CI'; + + const workflow_runs = await github.rest.actions.listWorkflowRuns({ + owner: context.repo.owner, + repo: context.repo.repo, + workflow_id: workflow_name, + head_sha: ref, + }); + + const latest_run = workflow_runs.data.workflow_runs[0]; + if (!latest_run || latest_run.conclusion !== 'success') { + core.setFailed('CI workflow has not succeeded for this commit'); + return false; + } + return true; + + publish: + if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} + needs: check-ci + runs-on: ubuntu-latest + permissions: + id-token: write + contents: write + attestations: write + steps: + - name: Download all wheel artifacts + uses: actions/download-artifact@v4 + with: + workflow: build-and-test.yml + commit: ${{ github.sha }} + path: artifact-download + + # Create dist directory + - name: Create dist directory + run: mkdir -p dist + shell: bash + + # Move all wheels to dist directory + - name: Collect wheels + run: | + find artifact-download -name "*.whl" -o -name "*.tar.gz" | xargs -I{} cp {} dist/ + echo "Package files in dist directory:" + ls -la dist/ + shell: bash + + - name: Generate artifact attestation + uses: actions/attest-build-provenance@v1 + with: + subject-path: "dist/*" + + # Publish to TestPyPI + - name: Publish to TestPyPI + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} + MATURIN_REPOSITORY: testpypi + with: + command: upload + args: --non-interactive --skip-existing dist/* + + # Only publish to PyPI if the publish_to_pypi input is true + - name: Publish to PyPI + if: ${{ github.event.inputs.publish_to_pypi == 'true' }} + uses: PyO3/maturin-action@v1 + env: + MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} + with: + command: upload + args: --non-interactive --skip-existing dist/* + + # # Create a GitHub release when a tag is created + # - name: Create GitHub Release + # if: ${{ inputs.publish_to_pypi }} + # uses: softprops/action-gh-release@v1 + # with: + # generate_release_notes: true + # files: dist/* From 826ce2893d212ff1ee389a295e7c1a18e56fed41 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 13:31:38 +0100 Subject: [PATCH 31/37] adapt workflow --- .github/workflows/publish.yml | 48 +++++++++++++++++------------------ 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ac5f1fb3..5315b6a1 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,35 +13,35 @@ on: - "true" jobs: - # First check if CI workflow succeeded - check-ci: - runs-on: ubuntu-latest - steps: - - name: Check CI workflow status - uses: actions/github-script@v7 - id: check-status - with: - script: | - const ref = context.sha; - const workflow_name = 'CI'; + # # First check if CI workflow succeeded + # check-ci: + # runs-on: ubuntu-latest + # steps: + # - name: Check CI workflow status + # uses: actions/github-script@v7 + # id: check-status + # with: + # script: | + # const ref = context.sha; + # const workflow_name = 'Build and Test'; - const workflow_runs = await github.rest.actions.listWorkflowRuns({ - owner: context.repo.owner, - repo: context.repo.repo, - workflow_id: workflow_name, - head_sha: ref, - }); + # const workflow_runs = await github.rest.actions.listWorkflowRuns({ + # owner: context.repo.owner, + # repo: context.repo.repo, + # workflow_id: workflow_name, + # head_sha: ref, + # }); - const latest_run = workflow_runs.data.workflow_runs[0]; - if (!latest_run || latest_run.conclusion !== 'success') { - core.setFailed('CI workflow has not succeeded for this commit'); - return false; - } - return true; + # const latest_run = workflow_runs.data.workflow_runs[0]; + # if (!latest_run || latest_run.conclusion !== 'success') { + # core.setFailed('CI workflow has not succeeded for this commit'); + # return false; + # } + # return true; publish: if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - needs: check-ci + # needs: check-ci runs-on: ubuntu-latest permissions: id-token: write From f63c714688d5b5eb62de3f6c973ec5d051bbb1c9 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 15:31:41 +0100 Subject: [PATCH 32/37] potentially fix workflow --- .github/workflows/publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 5315b6a1..db4c3690 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -48,8 +48,8 @@ jobs: contents: write attestations: write steps: - - name: Download all wheel artifacts - uses: actions/download-artifact@v4 + - name: Download artifacts from CI + uses: dawidd6/action-download-artifact@v3 with: workflow: build-and-test.yml commit: ${{ github.sha }} From 696eabc84ddd6714d5521a015bda8ebabb34cf71 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 15:39:02 +0100 Subject: [PATCH 33/37] more links --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1f00c353..353cbe01 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,9 @@ - Fast reading and writing of multi-dimensional arrays - Hierarchical data structure support -- Integration with NumPy arrays +- Integration with [NumPy](https://github.com/numpy/numpy) arrays - Chunked data access for efficient I/O -- Support for fsspec and xarray +- Support for [fsspec](https://github.com/fsspec/filesystem_spec) and [xarray](https://github.com/pydata/xarray) ### Basic Reading From 49a598179228b238c53c3d6b415abaa9ce9dee9f Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 16:11:20 +0100 Subject: [PATCH 34/37] fix publish --- .github/workflows/publish.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 8299bd9a..ad44c632 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -47,11 +47,14 @@ jobs: id-token: write contents: write attestations: write + actions: read steps: - name: Download artifacts from CI uses: dawidd6/action-download-artifact@v9 with: - workflow: build-and-test.yml + github_token: ${{ secrets.GITHUB_TOKEN }} + workflow: build-test.yml + commit: ${{ github.event.workflow_run.head_sha }} path: artifact-download # Create dist directory From 2299e93c5fb15a24436018f86b7179dd7fa430cd Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 16:51:22 +0100 Subject: [PATCH 35/37] cleanup publish --- .github/workflows/publish.yml | 31 +------------------------------ 1 file changed, 1 insertion(+), 30 deletions(-) diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index ad44c632..79a8a0e9 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -13,35 +13,8 @@ on: - "true" jobs: - # # First check if CI workflow succeeded - # check-ci: - # runs-on: ubuntu-latest - # steps: - # - name: Check CI workflow status - # uses: actions/github-script@v7 - # id: check-status - # with: - # script: | - # const ref = context.sha; - # const workflow_name = 'Build and Test'; - - # const workflow_runs = await github.rest.actions.listWorkflowRuns({ - # owner: context.repo.owner, - # repo: context.repo.repo, - # workflow_id: workflow_name, - # head_sha: ref, - # }); - - # const latest_run = workflow_runs.data.workflow_runs[0]; - # if (!latest_run || latest_run.conclusion !== 'success') { - # core.setFailed('CI workflow has not succeeded for this commit'); - # return false; - # } - # return true; - publish: if: ${{ startsWith(github.ref, 'refs/tags/') || github.event_name == 'workflow_dispatch' }} - # needs: check-ci runs-on: ubuntu-latest permissions: id-token: write @@ -55,6 +28,7 @@ jobs: github_token: ${{ secrets.GITHUB_TOKEN }} workflow: build-test.yml commit: ${{ github.event.workflow_run.head_sha }} + workflow_conclusion: success path: artifact-download # Create dist directory @@ -79,7 +53,6 @@ jobs: - name: Publish to TestPyPI uses: PyO3/maturin-action@v1 env: - MATURIN_PYPI_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} MATURIN_REPOSITORY: testpypi with: command: upload @@ -89,8 +62,6 @@ jobs: - name: Publish to PyPI if: ${{ github.event.inputs.publish_to_pypi == 'true' }} uses: PyO3/maturin-action@v1 - env: - MATURIN_PYPI_TOKEN: ${{ secrets.PYPI_API_TOKEN }} with: command: upload args: --non-interactive --skip-existing dist/* From 7df20a1f3b1911a68bd55c0ad7e26b1a2d9221d0 Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 17:12:31 +0100 Subject: [PATCH 36/37] naming things is hard --- Cargo.toml | 2 +- README.md | 6 +++--- benchmarks/helpers/args.py | 2 +- benchmarks/helpers/io/readers.py | 4 ++-- benchmarks/helpers/io/writers.py | 2 +- pyproject.toml | 4 ++-- python/{pyomfiles => omfiles}/__init__.py | 2 +- python/{pyomfiles => omfiles}/py.typed | 0 python/{pyomfiles => omfiles}/pyomfiles.pyi | 0 python/{pyomfiles => omfiles}/types.py | 0 python/{pyomfiles => omfiles}/xarray_backend.py | 2 +- src/lib.rs | 2 +- tests/test_fsspec.py | 8 ++++---- tests/test_omfilesrspy.py | 12 ++++++------ tests/test_utils.py | 2 +- tests/test_xarray.py | 4 ++-- 16 files changed, 26 insertions(+), 26 deletions(-) rename python/{pyomfiles => omfiles}/__init__.py (67%) rename python/{pyomfiles => omfiles}/py.typed (100%) rename python/{pyomfiles => omfiles}/pyomfiles.pyi (100%) rename python/{pyomfiles => omfiles}/types.py (100%) rename python/{pyomfiles => omfiles}/xarray_backend.py (99%) diff --git a/Cargo.toml b/Cargo.toml index 634102f4..80f17227 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [lib] -name = "pyomfiles" +name = "omfiles" crate-type = ["cdylib"] [dependencies] diff --git a/README.md b/README.md index 353cbe01..d10cf8c1 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ OM files are [structured like a tree of variables](https://github.com/open-meteo/om-file-format?tab=readme-ov-file#data-hierarchy-model). The following example assumes that the file `test_file.om` contains an array variable as a root variable which has a dimensionality greater than 2 and a size of at least 2x100: ```python -from pyomfiles import OmFilePyReader +from omfiles import OmFilePyReader reader = OmFilePyReader("test_file.om") data = reader[0:2, 0:100, ...] @@ -29,7 +29,7 @@ data = reader[0:2, 0:100, ...] #### Simple Array ```python import numpy as np -from pyomfiles import OmFilePyWriter +from omfiles import OmFilePyWriter # Create sample data data = np.random.rand(100, 100).astype(np.float32) @@ -54,7 +54,7 @@ writer.close(variable) #### Hierarchical Structure ```python import numpy as np -from pyomfiles import OmFilePyWriter +from omfiles import OmFilePyWriter # Create sample data features = np.random.rand(1000, 64).astype(np.float32) diff --git a/benchmarks/helpers/args.py b/benchmarks/helpers/args.py index c2185ce1..d755aac7 100644 --- a/benchmarks/helpers/args.py +++ b/benchmarks/helpers/args.py @@ -1,6 +1,6 @@ import argparse -from pyomfiles.types import BasicSelection +from omfiles.types import BasicSelection def parse_args() -> argparse.Namespace: diff --git a/benchmarks/helpers/io/readers.py b/benchmarks/helpers/io/readers.py index a01a1feb..7557cecd 100644 --- a/benchmarks/helpers/io/readers.py +++ b/benchmarks/helpers/io/readers.py @@ -4,10 +4,10 @@ import h5py import netCDF4 as nc import numpy as np -import pyomfiles as om +import omfiles as om import xarray as xr import zarr -from pyomfiles.types import BasicSelection +from omfiles.types import BasicSelection class BaseReader(ABC): diff --git a/benchmarks/helpers/io/writers.py b/benchmarks/helpers/io/writers.py index bc73f44c..b5e7b7af 100644 --- a/benchmarks/helpers/io/writers.py +++ b/benchmarks/helpers/io/writers.py @@ -5,7 +5,7 @@ import h5py import netCDF4 as nc import numpy as np -import pyomfiles as om +import omfiles as om import zarr from zarr.core.buffer import NDArrayLike diff --git a/pyproject.toml b/pyproject.toml index f922a924..4de12732 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ requires = ["maturin>=1.7,<2.0"] build-backend = "maturin" [project] -name = "pyomfiles" +name = "omfiles" requires-python = ">=3.8" classifiers = [ "Programming Language :: Rust", @@ -27,4 +27,4 @@ bindings = "pyo3" # pyo3 bindings is actually the default for maturin dev = ["pytest>=6.0", "hidefix", "h5py", "netCDF4", "zarr"] [project.entry-points."xarray.backends"] -om = "pyomfiles.xarray_backend:OmXarrayEntrypoint" +om = "omfiles.xarray_backend:OmXarrayEntrypoint" diff --git a/python/pyomfiles/__init__.py b/python/omfiles/__init__.py similarity index 67% rename from python/pyomfiles/__init__.py rename to python/omfiles/__init__.py index 23c559ad..9ce9d4c8 100644 --- a/python/pyomfiles/__init__.py +++ b/python/omfiles/__init__.py @@ -1,4 +1,4 @@ from . import types, xarray_backend -from .pyomfiles import OmFilePyReader, OmFilePyWriter +from .omfiles import OmFilePyReader, OmFilePyWriter __all__ = ["OmFilePyReader", "OmFilePyWriter", "xarray_backend", "types"] diff --git a/python/pyomfiles/py.typed b/python/omfiles/py.typed similarity index 100% rename from python/pyomfiles/py.typed rename to python/omfiles/py.typed diff --git a/python/pyomfiles/pyomfiles.pyi b/python/omfiles/pyomfiles.pyi similarity index 100% rename from python/pyomfiles/pyomfiles.pyi rename to python/omfiles/pyomfiles.pyi diff --git a/python/pyomfiles/types.py b/python/omfiles/types.py similarity index 100% rename from python/pyomfiles/types.py rename to python/omfiles/types.py diff --git a/python/pyomfiles/xarray_backend.py b/python/omfiles/xarray_backend.py similarity index 99% rename from python/pyomfiles/xarray_backend.py rename to python/omfiles/xarray_backend.py index 24a79286..11991d72 100644 --- a/python/pyomfiles/xarray_backend.py +++ b/python/omfiles/xarray_backend.py @@ -7,7 +7,7 @@ from xarray.core.utils import FrozenDict from xarray.core.variable import Variable -from .pyomfiles import OmFilePyReader, OmVariable +from .omfiles import OmFilePyReader, OmVariable # need some special secret attributes to tell us the dimensions DIMENSION_KEY = "_ARRAY_DIMENSIONS" diff --git a/src/lib.rs b/src/lib.rs index dc0de855..17b8dd1e 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,7 +11,7 @@ mod writer; /// A Python module implemented in Rust. #[pymodule(gil_used = false)] -fn pyomfiles<'py>(m: &Bound<'py, PyModule>) -> PyResult<()> { +fn omfiles<'py>(m: &Bound<'py, PyModule>) -> PyResult<()> { m.add_class::()?; m.add_class::()?; m.add_class::()?; diff --git a/tests/test_fsspec.py b/tests/test_fsspec.py index 306fd672..71b7556c 100644 --- a/tests/test_fsspec.py +++ b/tests/test_fsspec.py @@ -1,11 +1,11 @@ import fsspec import numpy as np -import pyomfiles +import omfiles # def test_fsspec_backend(): # fsspec_object = fsspec.open("test_files/read_test.om", "rb") -# file = pyomfiles.FsSpecBackend(fsspec_object) +# file = omfiles.FsSpecBackend(fsspec_object) # assert file.file_size == 144 @@ -15,7 +15,7 @@ def test_s3_reader(): backend = fs.open(file_path, mode="rb") # Create reader over fs spec backend - reader = pyomfiles.OmFilePyReader(backend) + reader = omfiles.OmFilePyReader(backend) data = reader[57812:57813, 0:100] # Verify the data @@ -29,7 +29,7 @@ def test_s3_reader_with_cache(): backend = fs.open(file_path, mode="rb", cache_type="mmap", block_size=1024, cache_options={"location": "cache"}) # Create reader over fs spec backend - reader = pyomfiles.OmFilePyReader(backend) + reader = omfiles.OmFilePyReader(backend) data = reader[57812:57813, 0:100] # Verify the data diff --git a/tests/test_omfilesrspy.py b/tests/test_omfilesrspy.py index dd1a5d24..e5b3298f 100644 --- a/tests/test_omfilesrspy.py +++ b/tests/test_omfilesrspy.py @@ -1,7 +1,7 @@ import os import numpy as np -import pyomfiles +import omfiles from .test_utils import create_test_om_file @@ -12,7 +12,7 @@ def test_write_om_roundtrip(): try: create_test_om_file(temp_file) - reader = pyomfiles.OmFilePyReader(temp_file) + reader = omfiles.OmFilePyReader(temp_file) data = reader[0:5, 0:5] del reader @@ -54,14 +54,14 @@ def test_round_trip_array_datatypes(): try: # Write data - writer = pyomfiles.OmFilePyWriter(temp_file) + writer = omfiles.OmFilePyWriter(temp_file) variable = writer.write_array(test_data, chunks=chunks, scale_factor=10000.0, add_offset=0.0) writer.close(variable) del writer # Read data back - reader = pyomfiles.OmFilePyReader(temp_file) + reader = omfiles.OmFilePyReader(temp_file) read_data = reader[:] del reader @@ -85,7 +85,7 @@ def test_write_hierarchical_file(): child2_data = np.random.rand(3, 3).astype(np.float32) # Write hierarchical structure - writer = pyomfiles.OmFilePyWriter(temp_file) + writer = omfiles.OmFilePyWriter(temp_file) # Write child2 array child2_var = writer.write_array( @@ -123,7 +123,7 @@ def test_write_hierarchical_file(): del writer # Read and verify the data using OmFilePyReader - reader = pyomfiles.OmFilePyReader(temp_file) + reader = omfiles.OmFilePyReader(temp_file) # Verify root data read_root = reader[:] diff --git a/tests/test_utils.py b/tests/test_utils.py index fd7e5a34..db90cdcb 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -1,7 +1,7 @@ from __future__ import annotations import numpy as np -from pyomfiles import OmFilePyWriter +from omfiles import OmFilePyWriter def create_test_om_file(filename: str = "test_file.om", shape=(5, 5), dtype=np.float32) -> tuple[str, np.ndarray]: diff --git a/tests/test_xarray.py b/tests/test_xarray.py index c879c691..1bf0a038 100644 --- a/tests/test_xarray.py +++ b/tests/test_xarray.py @@ -7,8 +7,8 @@ import warnings import numpy as np -import pyomfiles.pyomfiles as om -import pyomfiles.xarray_backend as om_xarray +import omfiles.omfiles as om +import omfiles.xarray_backend as om_xarray import xarray as xr from xarray.core import indexing From d2786f179e8dc19cb2d48fd9b059b2c8c1d622fb Mon Sep 17 00:00:00 2001 From: terraputix Date: Fri, 7 Mar 2025 17:24:28 +0100 Subject: [PATCH 37/37] update readme --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d10cf8c1..a3ce083a 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,10 @@ -# Omfiles-rs Python bindings +# Python bindings for Open Meteo file format [![Python 3.8+](https://img.shields.io/badge/python-3.8+-blue.svg)](https://www.python.org/downloads/) +[![Build and Test](https://github.com/terraputix/omfilesrspy/actions/workflows/build-test.yml/badge.svg)](https://github.com/terraputix/omfilesrspy/actions/workflows/build-test.yml) [![Project Status: WIP – Initial development is in progress, but there has not yet been a stable, usable release suitable for the public.](https://www.repostatus.org/badges/latest/wip.svg)](https://www.repostatus.org/#wip) +[![Publish](https://github.com/terraputix/omfilesrspy/actions/workflows/publish.yml/badge.svg)](https://github.com/terraputix/omfilesrspy/actions/workflows/publish.yml) + > **Note:** This package is currently under active development and not yet ready for production use. APIs may change without notice until the first stable release.