Source code for slipstream.core

"""Core module."""

import logging
from asyncio import (
    Event,
    Lock,
    gather,
    get_running_loop,
    sleep,
    wait_for,
)
from asyncio import TimeoutError as AsyncioTimeoutError
from collections.abc import (
    AsyncGenerator,
    AsyncIterable,
    AsyncIterator,
    Awaitable,
    Callable,
    Coroutine,
    Generator,
    Iterable,
)
from inspect import isasyncgenfunction, signature
from re import sub
from time import monotonic
from typing import (
    TYPE_CHECKING,
    Any,
    ClassVar,
    Literal,
    Protocol,
    cast,
)

from slipstream.interfaces import ICache, ICodec
from slipstream.utils import (
    AsyncCallable,
    AsyncSynchronizedGenerator,
    Pipe,
    PubSub,
    Signal,
    Singleton,
    awaitable,
    get_param_names,
)

if TYPE_CHECKING:
    from slipstream.checkpointing import Checkpoint

aiokafka_available = False

try:
    import aiokafka  # noqa: F401

    aiokafka_available = True
except ImportError:  # pragma: no cover
    pass

__all__ = [
    'READ_FROM_END',
    'READ_FROM_START',
    'PausableStream',
    'Topic',
]


[docs] READ_FROM_START = -2
[docs] READ_FROM_END = -1
PRODUCE_PAUSE_REASON = 'produce' PRODUCE_TIMEOUT_SECONDS = 120.0 YIELD_BUDGET_SECONDS = 0.002 _logger = logging.getLogger(__name__) class Handler(Protocol): """Callable returned by ``@handle``.""" checkpoint: 'Checkpoint | None' async def __call__(self, msg: Any, **kwargs: Any) -> Any: ...
[docs] class PausableStream: """Can signal source stream to pause. If `it` is of type `AsyncGenerator`, it will receive the signal through the yield send syntax in order to handle the state change appropriately. Alternatively, the `signal` property can be used directly. For example, the Topic class uses the signal to pause the Consumer. Only ``Signal.PAUSE`` pauses consumption; ``Signal.RESUME`` resumes it when no other pause reasons remain. Pause and resume are tracked per ``reason`` so independent callers compose (checkpoint downtime vs produce retry). Use ``counted=True`` when overlapping pauses share a reason, such as concurrent produces. """ @property
[docs] def iterable(self) -> AsyncIterable[Any]: """Get iterable.""" return self._iterable
def __init__(self, it: AsyncIterable[Any]) -> None: """Create instance that holds iterable and queue to pause it.""" self._iterable: AsyncIterable[Any] = it self._iterator: AsyncIterator[Any] | None = None self._pause_counts: dict[str, int] = {}
[docs] self.signal: Signal | Any = None
[docs] self.running: Event = Event()
self.running.set()
[docs] def send_signal( self, signal: Signal | Any, reason: str = '', *, counted: bool = False, ) -> None: """Send signal to stream. ``PAUSE``/``RESUME`` compose by ``reason``. A counted reason increments on pause and decrements on resume; an uncounted reason latches until resumed. Resume is applied only when no reasons remain. """ if signal is Signal.PAUSE: if counted: self._pause_counts[reason] = ( self._pause_counts.get(reason, 0) + 1 ) else: self._pause_counts[reason] = 1 self.signal = Signal.PAUSE if self.running.is_set(): self.running.clear() return if signal is Signal.RESUME: if counted: remaining = self._pause_counts.get(reason, 0) - 1 if remaining > 0: self._pause_counts[reason] = remaining else: self._pause_counts.pop(reason, None) else: self._pause_counts.pop(reason, None) if self._pause_counts: return self.signal = Signal.RESUME if not self.running.is_set(): self.running.set() return self.signal = signal
def __aiter__(self) -> AsyncIterator[Any]: """Create iterator.""" if self._iterator is None: if hasattr(self._iterable, 'asend'): self._iterator = cast( 'AsyncGenerator[Any, Signal | Any]', self._iterable, ) else: self._iterator = aiter(self._iterable) return self async def __anext__(self) -> Any: """Consume iterator while it's not paused.""" it = self._iterator or self.__aiter__() if hasattr(it, 'asend'): async_gen = cast('AsyncGenerator[Any, Signal | Any]', it) while True: # The generator gets a chance to handle the signal msg = await async_gen.asend(self.signal) # When the stream is paused and the generator handles # the signal, it should yield SENTINEL if msg is not Signal.SENTINEL: # Otherwise we assume that the generator does not # handle the pause, so we pause here if not self.running.is_set(): await self.running.wait() return msg else: msg = await anext(it) if not self.running.is_set(): await self.running.wait() return msg
[docs] class Conf(metaclass=Singleton): """The application configuration singleton. Register iterables (sources) and handlers (sinks): >>> from slipstream import handle >>> async def messages(): ... for emoji in '🏆📞🐟👌': ... yield emoji >>> @handle(messages(), sink=[print]) ... def handle_message(msg): ... yield f'Hello {msg}!' Set application kafka configuration (optional): >>> Conf({'bootstrap_servers': 'localhost:29091'}) {'bootstrap_servers': 'localhost:29091'} Provide exit hooks: >>> async def exit_hook(): ... print('Shutting down application.') >>> c = Conf() >>> c.register_exit_hook(exit_hook) """ pubsub = PubSub() iterables: ClassVar[dict[str, PausableStream]] = {} pipes: ClassVar[dict[AsyncCallable, tuple[str, tuple[Pipe, ...]]]] = {} exit_hooks: ClassVar[set[AsyncCallable]] = set() def __init__(self, conf: dict[str, Any] | None = None) -> None: """Define init behavior.""" self.conf: dict[str, Any] = {} if conf: self.__update__(conf)
[docs] def register_iterable(self, key: str, it: AsyncIterable[Any]) -> None: """Add iterable to global Conf.""" self.iterables[key] = PausableStream(it)
[docs] def register_handler( self, key: str, handler: AsyncCallable, *pipe: Pipe, ) -> None: """Add handler to global Conf.""" if pipe: self.pipes[handler] = (key, pipe) else: self.pubsub.subscribe(key, handler)
[docs] def register_exit_hook(self, exit_hook: AsyncCallable) -> None: """Add exit hook that's called on shutdown.""" self.exit_hooks.add(exit_hook)
[docs] def signal_iterables( self, signal: Signal, reason: str = '', *, counted: bool = False, ) -> None: """Send ``signal`` to every registered source stream.""" for stream in self.iterables.values(): stream.send_signal(signal, reason, counted=counted)
[docs] async def start(self, **kwargs: Any) -> None: """Start processing registered iterables.""" try: await gather( *[ self._distribute_messages(key, pausable_stream, kwargs) for key, pausable_stream in self.iterables.items() ], ) except KeyboardInterrupt: pass except Exception as e: _logger.critical(e) raise finally: await self._shutdown()
async def _shutdown(self) -> None: """Call exit hooks.""" # When the program immediately crashes give chance for objects # to be fully initialized before shutting them down await sleep(0.05) for hook in self.exit_hooks: await hook() async def _distribute_messages( self, key: str, pausable_stream: PausableStream, kwargs: Any, ) -> None: """Publish messages while giving other ready sources time to run. Synchronous sources and handlers may not suspend while publishing. Check elapsed monotonic time after each message and yield to the event loop after using a 2 millisecond execution budget. """ async def _distribute(stream: AsyncIterator[Any], kwargs: Any) -> None: loop = get_running_loop() yield_deadline = loop.time() + YIELD_BUDGET_SECONDS async for msg in stream: await self.pubsub.apublish(key, msg, **kwargs) if loop.time() >= yield_deadline: await sleep(0) yield_deadline = loop.time() + YIELD_BUDGET_SECONDS if piped_handlers := [ (handler, v[1]) for handler, v in self.pipes.items() if v[0] == key ]: s = AsyncSynchronizedGenerator(pausable_stream) await gather( _distribute(s, kwargs), *[ self._pipe(s.copy(), handler, *funcs, **kwargs) for handler, funcs in piped_handlers ], ) else: await _distribute(pausable_stream, kwargs) async def _pipe( self, stream: AsyncIterable, handler: AsyncCallable, *funcs: Callable[..., AsyncIterable[Any]], **kwargs: Any, ) -> None: """Push stream through pipe before feeding it to the handler.""" for func in funcs: stream = func(stream) async for msg in stream: await handler(msg, **kwargs) def __update__(self, conf: dict[str, Any] | None = None) -> None: """Set default app configuration.""" if not conf: return self.conf = {**self.conf, **conf} for key, value in conf.items(): key = sub('[^0-9a-zA-Z]+', '_', key) setattr(self, key, value) def __getattr__(self, name: str) -> Any: if name in self.conf: return self.conf[name] err_msg = ( f'"{self.__class__.__name__}" object has no attribute "{name}"' ) raise AttributeError(err_msg) def __repr__(self) -> str: """Represent config.""" return str(self.conf)
if aiokafka_available: from aiokafka import ( AIOKafkaClient, AIOKafkaConsumer, AIOKafkaProducer, ConsumerRecord, TopicPartition, ) from aiokafka.errors import KafkaError, KafkaTimeoutError from aiokafka.helpers import create_ssl_context def is_retriable_produce_error(exc: BaseException) -> bool: """Return whether a produce failure is safe to retry. Trust aiokafka's ``retriable`` flag so the set cannot drift. ``KafkaTimeoutError`` is not flagged retriable but is raised when ``send_and_wait`` cannot enqueue before ``request_timeout_ms``. """ return bool(getattr(exc, 'retriable', False)) or isinstance( exc, KafkaTimeoutError ) class Topic: """Act as a consumer and producer. >>> topic = Topic( ... 'emoji', ... { ... 'bootstrap_servers': 'localhost:29091', ... 'auto_offset_reset': 'earliest', ... 'group_id': 'demo', ... }, ... ) Loop over topic (iterable) to consume from it: >>> async for msg in topic: # doctest: +SKIP ... print(msg.value) Call topic (callable) with data to produce to it: >>> await topic({'msg': 'Hello World!'}) # doctest: +SKIP Produce retries default to fail-fast (``produce_retries=0``). Set ``produce_retries`` (and optional ``produce_retry_backoff`` seconds) on ``Conf`` or the topic ``conf`` to retry retriable broker errors. Sources are paused via ``Conf.signal_iterables`` while retrying. A produce attempt that is not acknowledged within ``produce_timeout`` seconds (default 120, ``None`` or ``0`` disables) is not retried and raises a ``RuntimeError``. An idempotent producer otherwise retries forever, and the stream stalls until the consumer is kicked out of its group. The message may still be delivered later, so do not retry it. With a ``group_id``, offsets are committed every ``auto_commit_interval_ms`` once a message is handled, which is when the stream asks for the next one, and not when it is read. A crash or rebalance can replay up to one interval of messages, so handling is at least once. This only applies to topics bound with `handle`; reading a topic directly keeps the consumer auto commit, which can skip the message that was in flight on shutdown. Set ``enable_auto_commit`` to ``False`` to commit yourself with `commit`. """ def __init__( self, name: str, conf: dict[str, Any] | None = None, offset: int | dict[int, int] | None = None, codec: ICodec | None = None, dry: bool = False, ) -> None: """Create topic instance to produce and consume messages.""" c = Conf() c.register_exit_hook(self.exit_hook) self.name = name merged = {**c.conf, **(conf or {})} self.produce_retries = int(merged.pop('produce_retries', 0)) self.produce_retry_backoff = float( merged.pop('produce_retry_backoff', 1.0) ) timeout = merged.pop('produce_timeout', PRODUCE_TIMEOUT_SECONDS) self.produce_timeout: float | None = ( float(timeout) if timeout else None ) self.conf = merged self.starting_offset = offset self.codec = codec self.dry = dry self.consumer: AIOKafkaConsumer | None = None self.producer: AIOKafkaProducer | None = None self.commit_interval: float | None = None self.commit_handled = False self._handled: dict[TopicPartition, int] = {} self._committed: dict[TopicPartition, int] = {} self._committed_at = monotonic() self._producer_lock = Lock() self._generator: ( AsyncGenerator[ Literal[Signal.SENTINEL] | ConsumerRecord[Any, Any], bool | None, ] | None ) = None if diff := set(self.conf).difference( { *get_param_names(AIOKafkaConsumer), *get_param_names(AIOKafkaProducer), *get_param_names(AIOKafkaClient), }, ): log_msg = ( f'Unexpected Topic {self.name} ' f'conf entries: {",".join(diff)}' ) _logger.warning(log_msg) if self.conf.get('security_protocol') in ( 'SSL', 'SASL_SSL', ) and not self.conf.get('ssl_context'): self.conf['ssl_context'] = create_ssl_context() def _maybe_encode(self, val: Any, serializer: str) -> Any: """Encode ``str`` when no serializer is configured.""" if isinstance(val, str) and not self.conf.get(serializer): return val.encode() return val def _maybe_decode(self, val: Any, deserializer: str) -> Any: """Decode ``bytes`` when no deserializer is configured.""" if isinstance(val, bytes) and not self.conf.get(deserializer): return val.decode() return val @property async def admin(self) -> AIOKafkaClient: """Get started instance of Kafka admin client.""" params = get_param_names(AIOKafkaClient) return AIOKafkaClient( **{k: v for k, v in self.conf.items() if k in params}, ) async def seek( self, offset: int | dict[int, int], consumer: AIOKafkaConsumer | None = None, timeout: float = 30.0, ) -> None: """Seek to offset.""" c = consumer or self.consumer if c is None: err_msg = 'No consumer provided' raise RuntimeError(err_msg) if isinstance(offset, int) and offset < READ_FROM_START: err_msg = 'Offset must be bigger than -3' raise ValueError(err_msg) # Wait until all partitions are assigned partitions = c.partitions_for_topic(self.name) or set() ready_partitions = set() max_attempts = int(timeout / 0.1) for i in range(max_attempts): assignment = c.assignment() ready_partitions = {_.partition for _ in assignment} if partitions.issubset(ready_partitions): break if i % 100 == 0: log_msg = ( f'Waiting for partitions ' f'{partitions - ready_partitions}' ) _logger.info(log_msg) await sleep(0.1) else: err_msg = ( f'Failed to assign {partitions} after {timeout}s, ' f'got: {ready_partitions}' ) raise RuntimeError(err_msg) # The desired offset per partition offsets = ( {TopicPartition(self.name, p): offset for p in partitions} if isinstance(offset, int) else { TopicPartition(self.name, p): o for p, o in offset.items() } ) # Perform seek if offset == READ_FROM_START: await c.seek_to_beginning(*assignment) elif offset == READ_FROM_END: await c.seek_to_end(*assignment) else: for p, o in offsets.items(): c.seek(p, o) async def get_consumer( self, commit_handled: bool = False ) -> AIOKafkaConsumer: """Get started instance of Kafka consumer. Args: commit_handled: Replace the consumer auto commit by committing messages the stream has handled. Only useful when the consumer is read through the stream. """ params = get_param_names(AIOKafkaConsumer) if self.codec: self.conf['value_deserializer'] = self.codec.decode conf = {k: v for k, v in self.conf.items() if k in params} if ( commit_handled and conf.get('group_id') and conf.get('enable_auto_commit', True) ): conf['enable_auto_commit'] = False self.commit_interval = ( conf.get('auto_commit_interval_ms', 5000) / 1000 ) consumer = AIOKafkaConsumer(self.name, **conf) await consumer.start() if self.starting_offset is not None: try: await self.seek(self.starting_offset, consumer) except Exception: await consumer.stop() raise return consumer async def get_producer(self) -> AIOKafkaProducer: """Get started instance of Kafka producer.""" params = get_param_names(AIOKafkaProducer) if self.codec: self.conf['value_serializer'] = self.codec.encode producer = AIOKafkaProducer( **{k: v for k, v in self.conf.items() if k in params}, ) await producer.start() return producer async def __call__( self, key: Any, value: Any, headers: dict[str, str] | None = None, **kwargs: Any, ) -> None: """Produce message to topic.""" key = self._maybe_encode(key, 'key_serializer') value = self._maybe_encode(value, 'value_serializer') headers_list = ( [(k, v.encode()) for k, v in headers.items()] if headers else None ) if self.dry: err_msg = f'Skipped sending message to {self.name} [dry=True]' _logger.warning(err_msg) return if not self.producer: async with self._producer_lock: if not self.producer: self.producer = await self.get_producer() await self._send_with_retry(key, value, headers_list, **kwargs) def on_handle(self) -> None: """Commit handled messages instead of the consumer auto commit.""" self.commit_handled = True async def commit( self, offsets: dict[TopicPartition, int] | None = None ) -> None: """Commit given offsets, or the handled ones, and raise on error. Args: offsets: Next offset to read per partition. Defaults to the offsets of messages the stream has handled. """ if not self.consumer: err_msg = f'Topic {self.name} has no consumer to commit' raise RuntimeError(err_msg) offsets = offsets if offsets is not None else self._handled if offsets: await self.consumer.commit(offsets) self._committed = {**self._committed, **offsets} async def _commit_handled(self) -> None: """Commit offsets of handled messages we still own.""" if self.consumer: owned = self.consumer.assignment() # shortcut: a partition revoked and reassigned between # commits keeps its old offset, which only replays messages. self._handled = { tp: o for tp, o in self._handled.items() if tp in owned } if self._handled and self._handled != self._committed: try: await self.consumer.commit(self._handled) self._committed = dict(self._handled) except KafkaError as e: log_msg = ( f'Failed to commit offsets of {self.name}: {e!r}' ) _logger.warning(log_msg) self._committed_at = monotonic() async def _send_with_retry( self, key: Any, value: Any, headers_list: list[tuple[str, bytes]] | None, **kwargs: Any, ) -> None: """Send once by default; retry only if ``produce_retries`` > 0.""" producer = self.producer if producer is None: err_msg = 'Producer not started' raise RuntimeError(err_msg) paused = False attempts = self.produce_retries + 1 try: for attempt in range(attempts): try: await wait_for( producer.send_and_wait( self.name, key=key, value=value, headers=headers_list, **kwargs, ), self.produce_timeout, ) except AsyncioTimeoutError as e: # noqa: PERF203 err_msg = ( f'Error while producing to Topic {self.name}: ' f'not acknowledged within {self.produce_timeout}s' ) _logger.exception(err_msg) raise RuntimeError(err_msg) from e except Exception as e: retriable = ( attempt + 1 < attempts and is_retriable_produce_error(e) ) if not retriable: err_msg = ( f'Error while producing to Topic {self.name}: ' f'{e.args[0] if e.args else ""}' ) _logger.exception(err_msg) raise RuntimeError(err_msg) from e if not paused: Conf().signal_iterables( Signal.PAUSE, PRODUCE_PAUSE_REASON, counted=True, ) paused = True _logger.warning( 'Retrying produce to Topic %s (%s/%s) after %s', self.name, attempt + 1, self.produce_retries, e, ) if self.produce_retry_backoff > 0: await sleep(self.produce_retry_backoff) else: return finally: if paused: Conf().signal_iterables( Signal.RESUME, PRODUCE_PAUSE_REASON, counted=True, ) async def _read( self, messages: AsyncIterator[ConsumerRecord[Any, Any]] ) -> ConsumerRecord[Any, Any]: """Read next message, committing handled offsets while idle.""" while True: try: return await wait_for( anext(messages), self.commit_interval or None ) except AsyncioTimeoutError: # noqa: PERF203 await self._commit_handled() async def _mark_handled(self, msg: ConsumerRecord[Any, Any]) -> None: """Remember handled offset and commit once the interval passed.""" if self.commit_interval is None: return self._handled[TopicPartition(msg.topic, msg.partition)] = ( msg.offset + 1 ) if monotonic() - self._committed_at >= self.commit_interval: await self._commit_handled() async def _get_generator( self, consumer: AIOKafkaConsumer, ) -> AsyncGenerator[ Literal[Signal.SENTINEL] | ConsumerRecord[Any, Any], bool | None, ]: """Return generator that iterates over messages from topic.""" signal = None try: msg: ConsumerRecord[Any, Any] messages = aiter(consumer) while True: try: msg = await self._read(messages) except StopAsyncIteration: break msg.key = self._maybe_decode(msg.key, 'key_deserializer') msg.value = self._maybe_decode( msg.value, 'value_deserializer' ) signal = yield msg await self._mark_handled(msg) if signal is Signal.PAUSE: consumer.pause(*consumer.assignment()) _logger.info(f'{self.name} paused') while True: signal = yield Signal.SENTINEL if signal is Signal.RESUME: _logger.info(f'{self.name} reactivated') consumer.resume(*consumer.assignment()) break try: await consumer.getmany(timeout_ms=3000) except KafkaError as e: if not getattr(e, 'retriable', False): raise _logger.warning( '%s pause poll failed; retrying', self.name, exc_info=True, ) await sleep(3) except Exception as e: err_msg = ( f'Error while consuming from Topic {self.name}: ' f'{e.args[0] if e.args else ""}' ) _logger.exception(err_msg) raise RuntimeError(err_msg) from e async def init_generator( self, ) -> AsyncGenerator[ Literal[Signal.SENTINEL] | ConsumerRecord[Any, Any], bool | None, ]: """Initialize generator.""" if not self.consumer: self.consumer = await self.get_consumer( commit_handled=self.commit_handled ) if not self._generator: return self._get_generator(self.consumer) return self._generator async def __aiter__(self) -> AsyncIterator[ConsumerRecord[Any, Any]]: """Iterate over messages from topic.""" if not self._generator: self._generator = await self.init_generator() async for msg in self._generator: if msg is not Signal.SENTINEL: yield msg async def asend(self, value: Any) -> ConsumerRecord[Any, Any]: """Send data to generator.""" if not self._generator: self._generator = await self.init_generator() generator = cast( 'AsyncGenerator[ConsumerRecord[Any, Any], Signal | None]', self._generator, ) return await generator.asend(value) async def __anext__(self) -> ConsumerRecord[Any, Any]: """Get the next message from topic.""" if not self._generator: self._generator = await self.init_generator() while (msg := await anext(self._generator)) is Signal.SENTINEL: continue return msg async def exit_hook(self) -> None: """Cleanup and finalization.""" if self.commit_interval is not None: try: await wait_for(self._commit_handled(), timeout=10) except Exception as e: # noqa: BLE001 log_msg = ( f'Offsets of topic "{self.name}" failed ' f'to commit on shutdown: {e!r}' ) _logger.critical(log_msg) for client in (self.consumer, self.producer): if not client: continue try: await wait_for(client.stop(), timeout=10) except TimeoutError: log_msg = ( f'Client for topic "{self.name}" failed ' f'to shut down in time {client}' ) _logger.critical(log_msg) except Exception as e: # noqa: BLE001 log_msg = ( f'Client for topic "{self.name}" failed ' f'to shut down gracefully {client}: {e}' ) _logger.critical(log_msg) async def _sink_output( f: Callable[..., Any], s: AsyncCallable, output: Any, ) -> None: """Sink output depending on sink type.""" known_sinks = (Topic, ICache) if aiokafka_available else (ICache,) if not isinstance(s, known_sinks): await awaitable(s(output)) return if not isinstance(output, tuple): err_msg = f'Sink expects: (key, val) in {f.__name__}, got :{output}' raise TypeError(err_msg) await s(*output) def _get_processor( f: AsyncCallable, is_asyncgen: bool, sink: Iterable[Callable | AsyncCallable], ) -> AsyncCallable: """Process output depending on output type.""" async def _process_output(output: Any) -> None: """Process and route output to sinks.""" if is_asyncgen: async for val in cast('AsyncIterator[Any]', output): for s in sink: await _sink_output(f, s, val) elif isinstance(output, Generator): for val in cast('Generator[Any, Any, Any]', output): for s in sink: await _sink_output(f, s, val) else: for s in sink: await _sink_output(f, s, output) return _process_output def _get_handler( f: AsyncCallable, sink: Iterable[Callable | AsyncCallable] ) -> Callable[..., Awaitable[Any]]: """Get handler wrapper depending on handler signature. Forward all keyword arguments to ``**kwargs``, otherwise only the ``downtime`` and ``checkpoint`` parameters the handler names. """ params = signature(f).parameters.values() forward_all = any(p.kind == p.VAR_KEYWORD for p in params) forwarded = {'downtime', 'checkpoint'} & { p.name for p in params if p.kind in (p.POSITIONAL_OR_KEYWORD, p.KEYWORD_ONLY) } _processor = _get_processor(f, isasyncgenfunction(f), sink) async def _handler(msg: Any, **kwargs: Any) -> None: """Execute function and handle its output.""" if not params: output = f() elif forward_all: output = f(msg, **kwargs) else: output = f( msg, **{k: kwargs[k] for k in forwarded & kwargs.keys()} ) await _processor(await awaitable(output)) return _handler
[docs] def handle( *iterable: AsyncIterable[Any], pipe: Iterable[Pipe] = [], sink: Iterable[Callable | AsyncCallable] = [], ) -> Callable[[AsyncCallable], Handler]: """Bind sources and sinks to the handler function. Ex: >>> topic = Topic('demo') # doctest: +SKIP >>> cache = Cache('state/demo') # doctest: +SKIP >>> @handle(topic, sink=[print, cache]) # doctest: +SKIP ... def handler(msg, **kwargs): ... return msg.key, msg.value """ c = Conf() def _deco(f: AsyncCallable) -> Handler: from slipstream.checkpointing import Checkpoint, bind_checkpoint plain: Any = _get_handler(f, sink) plain.checkpoint = None handler = plain for item in iterable: source, source_handler = item, plain if isinstance(item, Checkpoint): source = item.dependent source_handler = handler = bind_checkpoint(f, plain, item) if on_handle := getattr(source, 'on_handle', None): on_handle() iterable_key = str(id(source)) c.register_iterable(iterable_key, source) c.register_handler(iterable_key, source_handler, *pipe) return cast('Handler', handler) return _deco
[docs] def stream(**kwargs: Any) -> Coroutine[None, None, None]: """Start processing iterables bound by `handle` function. Ex: >>> from asyncio import run >>> kwargs = { ... 'env': 'DEV', ... } >>> run(stream(**kwargs)) # doctest: +SKIP """ return Conf().start(**kwargs)