Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 8 additions & 0 deletions AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,14 @@ Pre-commit hooks include YAML checks, EOF fixer, `sync-with-uv`, Ruff, and `ty`.
- Logging uses `loguru` in several packages; workflows also supports explicit logger/tracer configuration.
- Tests use `pytest`, with async coverage (`pytest-asyncio`) and property-based testing (`hypothesis`) in multiple packages.

### Import-Time Discipline

- Keep `tilebox.workflows` task-authoring imports light. Release runners create fresh virtual environments, so avoid
importing heavy optional/runtime dependencies (`pandas`, `numpy`, `xarray`, cloud SDKs, `ipywidgets`, OpenTelemetry
SDK/exporters, cache backends) from package `__init__` modules or core `Runner`/`Task` import paths.
- Prefer lazy imports inside the methods that actually need those dependencies. Module-level `__getattr__` aliases are
acceptable for public package aliases and are supported by Python 3.7+.

## Protobuf And Generated Code

Generated files live under paths such as:
Expand Down
14 changes: 13 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0

## [Unreleased]

## [0.55.1] - 2026-07-02

### Changed

- `tilebox-workflows`: Reduced import-time overhead for release runners by lazily loading optional heavy dependencies
such as datasets, pandas, cloud SDKs, notebook widgets, and runtime/observability modules until they are needed.
- `tilebox-datasets`: Reduced import-time overhead by lazily exporting the root and async client APIs and deferring
pandas/xarray imports in time interval parsing until parsing requires them.
- `tilebox-storage`: Reduced startup overhead by lazily exporting sync storage clients and deferring geospatial,
notebook, object-store, cloud SDK, HTTP, and progress-display dependencies until storage operations require them.

## [0.55.0] - 2026-07-01

### Added
Expand Down Expand Up @@ -394,7 +405,8 @@ the first client that does not cache data (since it's already on the local file
- Released under the [MIT](https://opensource.org/license/mit) license.
- Released packages: `tilebox-datasets`, `tilebox-workflows`, `tilebox-storage`, `tilebox-grpc`

[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.55.0...HEAD
[Unreleased]: https://github.com/tilebox/tilebox-python/compare/v0.55.1...HEAD
[0.55.1]: https://github.com/tilebox/tilebox-python/compare/v0.55.0...v0.55.1
[0.55.0]: https://github.com/tilebox/tilebox-python/compare/v0.54.0...v0.55.0
[0.54.0]: https://github.com/tilebox/tilebox-python/compare/v0.53.0...v0.54.0
[0.53.0]: https://github.com/tilebox/tilebox-python/compare/v0.52.0...v0.53.0
Expand Down
38 changes: 34 additions & 4 deletions tilebox-datasets/tilebox/datasets/__init__.py
Original file line number Diff line number Diff line change
@@ -1,16 +1,46 @@
import os
import sys
from typing import TYPE_CHECKING, Any

from loguru import logger

# only here for backwards compatibility, to preserve backwards compatibility with older imports
from tilebox.datasets.aio.timeseries import TimeseriesCollection, TimeseriesDataset
from tilebox.datasets.sync.client import Client
from tilebox.datasets.sync.dataset import CollectionClient, DatasetClient
if TYPE_CHECKING:
from tilebox.datasets.aio.timeseries import TimeseriesCollection, TimeseriesDataset
from tilebox.datasets.sync.client import Client
from tilebox.datasets.sync.dataset import CollectionClient, DatasetClient

__all__ = ["Client", "CollectionClient", "DatasetClient", "TimeseriesCollection", "TimeseriesDataset"]


def __getattr__(name: str) -> Any:
# PEP 562 module __getattr__ is supported since Python 3.7. Keep these aliases lazy so importing a focused
# submodule like tilebox.datasets.query.id_interval does not also import the sync/aio clients and their data-model
# dependencies.
match name:
case "Client":
from tilebox.datasets.sync.client import Client # noqa: PLC0415

return Client
case "CollectionClient":
from tilebox.datasets.sync.dataset import CollectionClient # noqa: PLC0415

return CollectionClient
case "DatasetClient":
from tilebox.datasets.sync.dataset import DatasetClient # noqa: PLC0415

return DatasetClient
case "TimeseriesCollection":
from tilebox.datasets.aio.timeseries import TimeseriesCollection # noqa: PLC0415

return TimeseriesCollection
case "TimeseriesDataset":
from tilebox.datasets.aio.timeseries import TimeseriesDataset # noqa: PLC0415

return TimeseriesDataset
case _:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")


def _init_logging(level: str = "INFO") -> None:
logger.remove()
logger.add(sys.stdout, level=level, format="{message}", catch=True)
Expand Down
37 changes: 33 additions & 4 deletions tilebox-datasets/tilebox/datasets/aio/__init__.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,36 @@
from tilebox.datasets.aio.client import Client
from tilebox.datasets.aio.dataset import CollectionClient, DatasetClient
from typing import TYPE_CHECKING, Any

# only here for backwards compatibility, to preserve backwards compatibility with older imports
from tilebox.datasets.aio.timeseries import TimeseriesCollection, TimeseriesDataset
if TYPE_CHECKING:
from tilebox.datasets.aio.client import Client
from tilebox.datasets.aio.dataset import CollectionClient, DatasetClient
from tilebox.datasets.aio.timeseries import TimeseriesCollection, TimeseriesDataset

__all__ = ["Client", "CollectionClient", "DatasetClient", "TimeseriesCollection", "TimeseriesDataset"]


def __getattr__(name: str) -> Any:
# PEP 562 module __getattr__ is supported since Python 3.7. Keep these aliases lazy so importing
# tilebox.datasets.aio does not also import xarray/pandas-backed dataset clients.
match name:
case "Client":
from tilebox.datasets.aio.client import Client # noqa: PLC0415

return Client
case "CollectionClient":
from tilebox.datasets.aio.dataset import CollectionClient # noqa: PLC0415

return CollectionClient
case "DatasetClient":
from tilebox.datasets.aio.dataset import DatasetClient # noqa: PLC0415

return DatasetClient
case "TimeseriesCollection":
from tilebox.datasets.aio.timeseries import TimeseriesCollection # noqa: PLC0415

return TimeseriesCollection
case "TimeseriesDataset":
from tilebox.datasets.aio.timeseries import TimeseriesDataset # noqa: PLC0415

return TimeseriesDataset
case _:
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
Original file line number Diff line number Diff line change
Expand Up @@ -231,7 +231,7 @@ def resize(self, buffer_size: int) -> None:
elif buffer_size > len(self._data):
# resize the data buffer to the new capacity, by just padding it with zeros at the end
missing = buffer_size - len(self._data)
self._data = np.pad( # ty: ignore[no-matching-overload]
self._data = np.pad(
self._data,
((0, missing), (0, 0)),
constant_values=self._type.fill_value,
Expand Down Expand Up @@ -309,7 +309,7 @@ def _resize(self) -> None:
else: # resize the data buffer to the new capacity, by just padding it with zeros at the end
missing_capacity = self._capacity - self._data.shape[0]
missing_array_dim = self._array_dim - self._data.shape[1]
self._data = np.pad( # ty: ignore[no-matching-overload]
self._data = np.pad(
self._data,
((0, missing_capacity), (0, missing_array_dim), (0, 0)),
constant_values=self._type.fill_value,
Expand Down
71 changes: 41 additions & 30 deletions tilebox-datasets/tilebox/datasets/query/time_interval.py
Original file line number Diff line number Diff line change
@@ -1,21 +1,26 @@
from dataclasses import dataclass
from datetime import datetime, timedelta, timezone
from typing import TypeAlias
from typing import TYPE_CHECKING, Any, TypeAlias

import numpy as np
import xarray as xr
from google.protobuf.duration_pb2 import Duration
from google.protobuf.timestamp_pb2 import Timestamp
from pandas.core.tools.datetimes import DatetimeScalar, to_datetime

from tilebox.datasets.tilebox.v1 import query_pb2

if TYPE_CHECKING:
from pandas.core.tools.datetimes import DatetimeScalar
from xarray import DataArray, Dataset
else:
DataArray = Any
Dataset = Any
DatetimeScalar = Any

_SMALLEST_POSSIBLE_TIMEDELTA = timedelta(microseconds=1)
_EPOCH = datetime(1970, 1, 1, tzinfo=timezone.utc)

# A type alias for the different types that can be used to specify a time interval
TimeIntervalLike: TypeAlias = (
"DatetimeScalar | tuple[DatetimeScalar, DatetimeScalar] | xr.DataArray | xr.Dataset | TimeInterval"
"DatetimeScalar | tuple[DatetimeScalar, DatetimeScalar] | list[DatetimeScalar] | DataArray | Dataset | TimeInterval"
)
# once we require python >= 3.12 we can replace this with a type statement, which doesn't require a string at all
# type TimeIntervalLike = DatetimeScalar | tuple[DatetimeScalar ... | TimeInterval
Expand Down Expand Up @@ -133,30 +138,34 @@ def parse(cls, arg: TimeIntervalLike) -> "TimeInterval":
TimeInterval: The parsed time interval
"""

match arg:
case TimeInterval(_, _, _, _):
return arg
case (start, end):
return TimeInterval(start=_convert_to_datetime(start), end=_convert_to_datetime(end))
case point_in_time if isinstance(point_in_time, DatetimeScalar | int):
dt = _convert_to_datetime(point_in_time)
return TimeInterval(start=dt, end=dt, start_exclusive=False, end_inclusive=True)
case arr if (
isinstance(arr, xr.DataArray)
and arr.ndim == 1
and arr.size > 0
and arr.dtype == np.dtype("datetime64[ns]")
):
start = arr.data[0]
end = arr.data[-1]
return TimeInterval(
start=_convert_to_datetime(start),
end=_convert_to_datetime(end),
start_exclusive=False,
end_inclusive=True,
)
case ds if isinstance(ds, xr.Dataset) and "time" in ds.coords:
return cls.parse(ds.time)
if isinstance(arg, TimeInterval):
return arg

if isinstance(arg, list | tuple) and len(arg) == 2:
start, end = arg
return TimeInterval(start=_convert_to_datetime(start), end=_convert_to_datetime(end))

from pandas.core.tools.datetimes import DatetimeScalar # noqa: PLC0415

if isinstance(arg, DatetimeScalar | int):
dt = _convert_to_datetime(arg)
return TimeInterval(start=dt, end=dt, start_exclusive=False, end_inclusive=True)

import numpy as np # noqa: PLC0415
import xarray as xr # noqa: PLC0415

if isinstance(arg, xr.DataArray) and arg.ndim == 1 and arg.size > 0 and arg.dtype == np.dtype("datetime64[ns]"):
start = arg.data[0]
end = arg.data[-1]
return TimeInterval(
start=_convert_to_datetime(start),
end=_convert_to_datetime(end),
start_exclusive=False,
end_inclusive=True,
)

if isinstance(arg, xr.Dataset) and "time" in arg.coords:
return cls.parse(arg.time)

raise ValueError(f"Failed to convert {arg} ({type(arg)}) to TimeInterval)")

Expand Down Expand Up @@ -192,8 +201,10 @@ def to_message(self) -> query_pb2.TimeInterval:
_EMPTY_TIME_INTERVAL = TimeInterval(_EPOCH, _EPOCH, start_exclusive=True, end_inclusive=False)


def _convert_to_datetime(arg: DatetimeScalar) -> datetime:
def _convert_to_datetime(arg: Any) -> datetime:
"""Convert the given datetime scalar to a datetime object in the UTC timezone"""
from pandas.core.tools.datetimes import to_datetime # noqa: PLC0415

dt: datetime = to_datetime(arg, utc=True).to_pydatetime()
if dt.tzinfo is None:
dt = dt.replace(tzinfo=timezone.utc)
Expand Down
Loading
Loading