feat[python]: support full read/write from object storage

docs/api/python/index.rst

@@ -10,5 +10,6 @@ Python API
    expr
    compress
    io
+   store
    dataset
    type_aliases

docs/api/python/io.rst

@@ -28,3 +28,4 @@ HTTP, S3, Google Cloud Storage, and Azure Blob Storage.
 .. automodule:: vortex.io
     :members:
     :imported-members:
+

docs/api/python/store.rst

@@ -0,0 +1,17 @@
+Object Store support
+====================
+
+Vortex arrays support reading and writing to object storage systems such as, S3, Google Cloud Storage, and
+Azure Blob Storage.
+
+.. autosummary::
+   :nosignatures:
+
+.. raw:: html
+
+   <hr>
+
+.. automodule:: vortex.store
+    :members:
+    :imported-members:
+

pyproject.toml

@@ -68,3 +68,6 @@ known-first-party = [
 log_cli = true
 log_cli_level = "INFO"
 xfail_strict = true
+
+[tool.basedpyright]
+exclude = ["vortex-python/python/vortex/_lib/store/**.pyi"]

vortex-io/src/write.rs

@@ -62,7 +62,7 @@ where
     }
 
     fn shutdown(&mut self) -> impl Future<Output = io::Result<()>> {
-        ready(Ok(()))
+        ready(Write::flush(self))
     }
 }
 

vortex-python/Cargo.toml

@@ -39,6 +39,7 @@ parking_lot = { workspace = true }
 pyo3 = { workspace = true, features = ["abi3", "abi3-py311"] }
 pyo3-bytes = { workspace = true }
 pyo3-log = { workspace = true }
+pyo3-object_store = { version = "0.7" }
 tokio = { workspace = true, features = ["fs", "rt-multi-thread"] }
 # This feature makes the underlying tracing logs to be emitted as `log` events
 tracing = { workspace = true, features = ["std", "log"] }

vortex-python/pyproject.toml

@@ -64,6 +64,9 @@ include = [
     { path = "python/vortex/py.typed", format = "sdist" },
 ]
 
+[tool.basedpyright]
+exclude = ["python/vortex/_lib/store/**.pyi"]
+
 [dependency-groups]
 dev = [
     "basedpyright>=1.31",

vortex-python/python/vortex/__init__.py

@@ -1,7 +1,7 @@
 # SPDX-License-Identifier: Apache-2.0
 # SPDX-FileCopyrightText: Copyright the Vortex contributors
 
-from importlib import metadata as _metadata
+import importlib.metadata
 
 from . import _lib, arrays, dataset, expr, file, io, ray, registry, scan
 from ._lib.arrays import (  # pyright: ignore[reportMissingModuleSource]
@@ -84,11 +84,12 @@
 assert _lib, "Ensure we eagerly import the Vortex native library"
 
 # Resolve the installed distribution version so it is available as vortex.__version__.
+
 __version__ = "unknown"
 try:
     # Try to read the installed distribution version for the Python package name.
-    __version__ = _metadata.version("vortex-data")
-except _metadata.PackageNotFoundError:
+    __version__ = importlib.metadata.version("vortex-data")
+except importlib.metadata.PackageNotFoundError:
     # If the distribution is not installed, keep the unknown fallback.
     pass
 

vortex-python/python/vortex/_lib/__init__.pyi

@@ -1,2 +1,3 @@
 #  SPDX-License-Identifier: Apache-2.0
 #  SPDX-FileCopyrightText: Copyright the Vortex contributors
+

vortex-python/python/vortex/_lib/file.pyi

@@ -14,6 +14,7 @@ from .dtype import DType
 from .expr import Expr
 from .iter import ArrayIterator
 from .scan import RepeatedScan
+from .store import ObjectStore
 
 @final
 class VortexFile:
@@ -47,4 +48,4 @@ class VortexFile:
     def to_polars(self) -> pl.LazyFrame: ...
     def splits(self) -> list[tuple[int, int]]: ...
 
-def open(path: str, *, without_segment_cache: bool = False) -> VortexFile: ...
+def open(path: str, *, store: ObjectStore | None = None, without_segment_cache: bool = False) -> VortexFile: ...

vortex-python/python/vortex/_lib/io.pyi

@@ -4,20 +4,32 @@
 from ..type_aliases import IntoArrayIterator
 from .arrays import Array
 from .expr import Expr
+from .store import ObjectStore
 
 def read_url(
     url: str,
     *,
+    store: ObjectStore | None = None,
     projection: list[str] | list[int] | None = None,
     row_filter: Expr | None = None,
     indices: Array | None = None,
 ) -> Array: ...
-def write(iter: IntoArrayIterator, path: str) -> None: ...
+def write(
+    iter: IntoArrayIterator,
+    path: str,
+    *,
+    store: ObjectStore | None = None,
+) -> None: ...
 
 class VortexWriteOptions:
     @staticmethod
     def default() -> VortexWriteOptions: ...
     @staticmethod
     def compact() -> VortexWriteOptions: ...
     @staticmethod
-    def write_path(iter: IntoArrayIterator, path: str) -> VortexWriteOptions: ...
+    def write(
+        iter: IntoArrayIterator,
+        path: str,
+        *,
+        store: ObjectStore | None = None,
+    ) -> VortexWriteOptions: ...

vortex-python/python/vortex/_lib/store/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2024 Development Seed
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.