Source code for slipstream.checkpointing

"""Slipstream checkpointing."""

import logging
from collections.abc import (
    AsyncIterable,
    AsyncIterator,
    Awaitable,
    Callable,
    Generator,
)
from datetime import datetime, timedelta
from functools import wraps
from typing import Any

from slipstream.core import Conf, Signal
from slipstream.interfaces import ICache
from slipstream.utils import AsyncCallable, awaitable

_logger = logging.getLogger(__name__)


def _validate_state_markers(
    dependencies: list['Dependency'],
    state: Callable[[Any, dict[str, Any]], dict[str, Any]] | None,
    marker: Callable[..., Any] | str | None,
) -> None:
    """Require explicit dependency markers for a state-aware marker."""
    if not state or not callable(marker):
        return
    missing = [
        dependency.name for dependency in dependencies if not dependency.marker
    ]
    if missing:
        err_msg = (
            'Dependencies need their own marker when the checkpoint '
            f'marker uses state: {missing}'
        )
        raise ValueError(err_msg)


[docs] STATE_NAME = 'state'
[docs] STATE_MARKER_NAME = 'state_marker'
[docs] CHECKPOINT_STATE_NAME = 'checkpoint_state'
[docs] CHECKPOINT_MARKER_NAME = 'checkpoint_marker'
[docs] CHECKPOINTS_NAME = 'checkpoints'
[docs] class Downtime(dict): """Per-dependency pulse result. Falsy when empty. A single entry compares equal to that value, so ``downtime == timedelta(...)`` still holds for one leader. """ def __eq__(self, other: object) -> bool: """Match a dict, or the sole value when only one leader is down.""" if isinstance(other, dict): return dict.__eq__(self, other) if len(self) == 1: return next(iter(self.values())) == other return NotImplemented
[docs] class Dependency: """Track the dependent stream state to recover from downtime. The dependency name should not be changed once created, it is used to persist the dependency in the cache. >>> async def emoji(): ... for emoji in '🏆📞🐟👌': ... yield emoji >>> Dependency('emoji', emoji()) {'checkpoint_state': None, 'checkpoint_marker': None} """ def __init__( self, name: str, dependency: AsyncIterable[Any], downtime_threshold: Any = timedelta(minutes=10), downtime_check: AsyncCallable[['Checkpoint', 'Dependency'], Any] | None = None, recovery_check: AsyncCallable[['Checkpoint', 'Dependency'], bool] | None = None, marker: Callable[..., Any] | str | None = None, state: Callable[[Any, dict[str, Any]], dict[str, Any]] | None = None, ) -> None: """Initialize dependency for checkpointing."""
[docs] self.name = name
[docs] self.dependency = dependency
[docs] self.checkpoint_state: Any = None
[docs] self.checkpoint_marker: Any = None
[docs] self.downtime_threshold = downtime_threshold
[docs] self.downtime_check = downtime_check or self._default_downtime_check
[docs] self.recovery_check = recovery_check or self._default_recovery_check
[docs] self.marker = marker
[docs] self.state_extractor = state
[docs] self.is_down = False
[docs] def uses_default_downtime_check(self) -> bool: """Return whether first-pulse event-time seeding applies.""" check = self.downtime_check default = self._default_downtime_check return check is default or getattr(check, '__func__', None) is default
def _checkpoint_key(self, cache_key_prefix: str) -> str: """Cache key prefix for this dependency.""" return f'{cache_key_prefix}{self.name}_'
[docs] def save( self, cache: ICache, cache_key_prefix: str, checkpoint_state: Any, checkpoint_marker: datetime, ) -> None: """Save checkpoint state to cache.""" key = self._checkpoint_key(cache_key_prefix) cache[key + CHECKPOINT_STATE_NAME] = checkpoint_state cache[key + CHECKPOINT_MARKER_NAME] = checkpoint_marker
[docs] def load(self, cache: ICache, cache_key_prefix: str) -> None: """Load checkpoint state from cache.""" key = self._checkpoint_key(cache_key_prefix) self.checkpoint_state = cache[key + CHECKPOINT_STATE_NAME] self.checkpoint_marker = cache[key + CHECKPOINT_MARKER_NAME]
@staticmethod def _default_downtime_check( c: 'Checkpoint', d: 'Dependency', ) -> timedelta | None: """Determine dependency downtime by comparing event timestamps. This behavior can be overridden by passing a callable to `downtime_check` that takes a `Checkpoint` object. """ diff = c.state_marker - d.checkpoint_marker if diff > d.downtime_threshold: return diff return None @staticmethod def _default_recovery_check(c: 'Checkpoint', d: 'Dependency') -> bool: """Determine dependency has caught up by comparing event timestamps. This behavior can be overridden by passing a callable to `recovery_check` that takes a `Checkpoint` object. """ return d.checkpoint_marker > c.state_marker def __iter__(self) -> Generator[tuple[str, Any | None], None, None]: """Get relevant values when dict is called.""" yield from ( { CHECKPOINT_STATE_NAME: self.checkpoint_state, CHECKPOINT_MARKER_NAME: self.checkpoint_marker, }.items() ) def __repr__(self) -> str: """Represent checkpoint.""" return str(dict(self))
[docs] class Checkpoint: """Track one stream against the streams it depends on. >>> async def emoji(): ... for emoji in '🏆📞🐟👌': ... yield emoji >>> dependent, dependency = emoji(), emoji() >>> checkpoint = Checkpoint( ... dependent, ... Dependency('dependency', dependency), ... name='dependent', ... ) Pass a marker and bind the checkpoint to call ``heartbeat`` and ``check_pulse`` automatically: >>> checkpoint = Checkpoint( ... dependent, ... Dependency('dependency', dependency, marker='timestamp'), ... name='dependent', ... marker='timestamp', ... ) >>> from slipstream import handle >>> @handle(checkpoint) ... async def dependent_handler(msg, checkpoint=None): ... yield msg If no cache is provided, the checkpoint lasts only for this process. """ def __init__( self, dependent: AsyncIterable[Any], *leaders: AsyncIterable[Any] | Dependency, dependencies: AsyncIterable[Any] | Dependency | list[AsyncIterable[Any] | Dependency] | None = None, name: str | None = None, on_downtime: Callable[['Checkpoint', Dependency], Any] | None = None, on_recovery: Callable[['Checkpoint', Dependency], Any] | None = None, cache: ICache | None = None, cache_key_prefix: str = '_', pause_dependent: bool = True, downtime_threshold: timedelta | None = None, marker: Callable[..., Any] | str | None = None, state: Callable[[Any, dict[str, Any]], dict[str, Any]] | None = None, ) -> None: """Create a checkpoint for ``dependent`` against its dependencies. :: Checkpoint( activity, Dependency('weather', weather), name='activity', ) """ items = [*leaders] if isinstance(dependencies, list): items.extend(dependencies) elif dependencies is not None: items.append(dependencies) threshold: dict[str, Any] = ( {} if downtime_threshold is None else {'downtime_threshold': downtime_threshold} ) built = [ item if isinstance(item, Dependency) else Dependency(str(id(item)), item, **threshold) for item in items ] names = [dependency.name for dependency in built] if len(names) != len(set(names)): err_msg = 'Dependency names must be unique.' raise ValueError(err_msg)
[docs] self.name = name or getattr(dependent, 'name', 'checkpoint')
[docs] self.dependent = dependent
[docs] self.dependencies = {d.name: d for d in built}
if id(self.dependent) in {id(d.dependency) for d in built}: err_msg = 'Checkpoint cannot depend on its dependent stream.' raise ValueError(err_msg) _validate_state_markers(built, state, marker)
[docs] self.pause_dependent = pause_dependent
[docs] self.marker = marker
[docs] self.state_extractor = state
[docs] self.downtime: Any | None = None
self._cache = cache self._cache_key = f'{cache_key_prefix}_{self.name}_' self._downtime_callback = on_downtime self._recovery_callback = on_recovery self._awaiting_resume = False
[docs] self.state = {}
[docs] self.state_marker: Any = None
if self._cache: self.state = self._cache[f'{self._cache_key}_{STATE_NAME}'] or {} self.state_marker = self._cache[ f'{self._cache_key}_{STATE_MARKER_NAME}' ] for dependency in self.dependencies.values(): dependency.load(self._cache, self._cache_key) def __aiter__(self) -> AsyncIterator[Any]: """Iterate the dependent stream.""" return self.dependent.__aiter__()
[docs] async def heartbeat( self, marker: datetime | Any, dependency_name: str | None = None, checkpoint_state: Any = None, ) -> dict: """Update checkpoint to latest state. Args: marker (datetime | Any): Typically the event timestamp that is compared to the event timestamp of a dependent stream. dependency_name (str, optional): Required when there are multiple dependencies to specify which one the heartbeat is for. checkpoint_state: Complete caller-selected dependency state. """ if dependency_name: if not (dependency := self.dependencies.get(dependency_name)): err_msg = 'Dependency does not exist.' raise KeyError(err_msg) elif len(self.dependencies) == 1: dependency = next(iter(self.dependencies.values())) else: err_msg = ( 'Argument `dependency_name` must be provided ' 'for checkpoint with multiple dependencies.' ) raise ValueError(err_msg) self._save_checkpoint( dependency, self.state if checkpoint_state is None else checkpoint_state, marker, ) if dependency.is_down: if await awaitable(dependency.recovery_check(self, dependency)): dependency.is_down = False await self._resume_if_cleared(dependency) return { 'is_late': dependency.is_down, 'dependent_marker': self.state_marker, 'dependency_marker': dependency.checkpoint_marker, }
[docs] async def check_pulse( self, marker: datetime | Any, checkpoint_state: Any = None, **kwargs: Any, ) -> Any | None: """Update state that can be used as checkpoint. Args: marker (datetime | Any): Typically the event timestamp that is compared to the event timestamp of a dependency stream. checkpoint_state: Complete caller-selected dependent state. kwargs (Any): Any information that can be used for reprocessing any incorrect data that was sent out during downtime of a dependency stream, stored in `state`. Returns: None when every dependency is healthy. Otherwise a :class:`Downtime` map of name → check (or ``True`` if still down but not over threshold). One leader still compares equal to its timedelta / ``True``. """ self._save_state(marker, checkpoint_state, **kwargs) reported = Downtime() for dependency in self.dependencies.values(): down_now = await self._pulse_dependency(dependency) if down_now: reported[dependency.name] = down_now elif dependency.is_down: reported[dependency.name] = True self.downtime = reported or None return self.downtime
async def _pulse_dependency(self, dependency: Dependency) -> Any | None: """Update one dependency from the current pulse. Returns: A truthy downtime value when this dependency is down. """ if ( dependency.checkpoint_marker is None and dependency.uses_default_downtime_check() ): self._save_checkpoint( dependency, self.state, self.state_marker, ) down_now = await awaitable(dependency.downtime_check(self, dependency)) if down_now: if not dependency.is_down: log_msg = ( f'Downtime of dependency "{dependency.name}" detected' ) _logger.info(log_msg) self._signal_dependent(Signal.PAUSE) self._awaiting_resume = True if self._downtime_callback: await awaitable(self._downtime_callback(self, dependency)) dependency.is_down = True return down_now if dependency.is_down and await awaitable( dependency.recovery_check(self, dependency) ): dependency.is_down = False await self._resume_if_cleared(dependency) return None def _signal_dependent(self, signal: Signal) -> None: """Pause or resume the dependent iterable when configured to do so.""" key, c = str(id(self.dependent)), Conf() if self.pause_dependent and key in c.iterables: c.iterables[key].send_signal(signal) async def _resume_if_cleared(self, dependency: Dependency) -> None: """Resume the dependent stream when no dependency is still down.""" if any(_.is_down for _ in self.dependencies.values()): return if not self._awaiting_resume: return self._awaiting_resume = False _logger.debug( f'Dependency "{dependency.name}" downtime resolved', ) self._signal_dependent(Signal.RESUME) if self._recovery_callback: await awaitable(self._recovery_callback(self, dependency)) def _save_state( self, state_marker: datetime | Any, checkpoint_state: dict[str, Any] | None, **kwargs: Any, ) -> None: """Save state of the stream (to cache). Markers only move forward so another partition cannot rewind the checkpoint. """ if checkpoint_state is None: self.state.update(**kwargs) else: self.state = dict(checkpoint_state) self.state_marker = _later_marker(self.state_marker, state_marker) if not self._cache: return self._cache[f'{self._cache_key}_{STATE_NAME}'] = self.state self._cache[f'{self._cache_key}_{STATE_MARKER_NAME}'] = ( self.state_marker ) def _save_checkpoint( self, dependency: Dependency, checkpoint_state: Any, checkpoint_marker: datetime | Any, ) -> None: """Save state of the dependency checkpoint (to cache). Markers only move forward so another partition cannot rewind the checkpoint. """ checkpoint_marker = _later_marker( dependency.checkpoint_marker, checkpoint_marker, ) dependency.checkpoint_state = dict(checkpoint_state) dependency.checkpoint_marker = checkpoint_marker if not self._cache: return dependency.save( self._cache, self._cache_key, dependency.checkpoint_state, checkpoint_marker, ) def __getitem__(self, key: str) -> Dependency: """Get dependency from dependencies.""" return self.dependencies[key] def __repr__(self) -> str: """Represent checkpoint.""" return str( { STATE_NAME: self.state, STATE_MARKER_NAME: self.state_marker, CHECKPOINTS_NAME: { dependency.name: dict(dependency) for dependency in self.dependencies.values() }, }, )
def _later_marker(old: Any, new: Any) -> Any: """Keep the high-water marker.""" return new if old is None else max(old, new) def _marker_value(marker: Callable[[Any], Any] | str, msg: Any) -> Any: """Apply only the marker policy supplied by the caller.""" if callable(marker): return marker(msg) try: return msg[marker] except TypeError: return msg.value[marker] def _observe( msg: Any, marker: Callable[..., Any] | str, extractor: Callable[[Any, dict[str, Any]], dict[str, Any]] | None, state: dict[str, Any] | None, ) -> tuple[dict[str, Any] | None, Any]: """Return the caller-extracted state and marker for ``msg``.""" if not extractor: return None, _marker_value(marker, msg) state = extractor(msg, dict(state or {})) if callable(marker): return state, marker(msg, state) return state, _marker_value(marker, msg)
[docs] def bind_checkpoint( f: Callable[..., Any], handler: Callable[..., Awaitable[Any]], checkpoint: Checkpoint, ) -> Callable[..., Awaitable[Any]]: """Heartbeat dependencies and pulse the dependent around a handler.""" if checkpoint.marker is None: err_msg = 'A marker is required when binding a checkpoint to handle.' raise ValueError(err_msg) checkpoint_marker = checkpoint.marker @wraps(f) async def _pulsed(msg: Any, **kwargs: Any) -> Any: state, marker = _observe( msg, checkpoint_marker, checkpoint.state_extractor, checkpoint.state, ) downtime = await checkpoint.check_pulse( marker, checkpoint_state=state, ) return await handler( msg, downtime=downtime, checkpoint=checkpoint, **kwargs, ) c = Conf() for dependency in checkpoint.dependencies.values(): key = str(id(dependency.dependency)) if key not in c.iterables: c.register_iterable(key, dependency.dependency) async def _heartbeat( msg: Any, _dependency: Dependency = dependency, _marker: Callable[..., Any] | str = ( dependency.marker or checkpoint_marker ), **_kwargs: Any, ) -> None: state, marker = _observe( msg, _marker, _dependency.state_extractor, _dependency.checkpoint_state, ) await checkpoint.heartbeat( marker, _dependency.name, checkpoint_state=state, ) c.register_handler(key, _heartbeat) _pulsed.checkpoint = checkpoint # type: ignore[attr-defined] return _pulsed