Dirk Riemann
132f110463
Thin wrapper around ix.provenance.apply_reliability_flags. Validate skips entirely when include_provenance is off OR when no provenance data was built (text-only request, etc.). Process reads context.texts + context.use_case_response and lets the verifier mutate the FieldProvenance entries + fill quality_metrics counters in place. 11 unit tests in tests/unit/test_reliability_step.py cover: validate skips on flag off / missing provenance, runs otherwise; per-type flag behaviour (string verified + text_agreement, Literal -> None, None value -> None, short numeric -> text_agreement None, date with both sides parsed, IBAN whitespace-insensitive, disagreement -> False); quality_metrics verified_fields / text_agreement_fields counters. Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
56 lines
1.9 KiB
Python
56 lines
1.9 KiB
Python
"""ReliabilityStep — writes provenance_verified + text_agreement (spec §6).
|
|
|
|
Runs after :class:`~ix.pipeline.genai_step.GenAIStep`. Skips entirely
|
|
when provenance is off OR when no provenance data was built (OCR-skipped
|
|
text-only request, for example). Otherwise delegates to
|
|
:func:`~ix.provenance.apply_reliability_flags`, which mutates each
|
|
:class:`~ix.contracts.FieldProvenance` in place and fills the two
|
|
summary counters (``verified_fields``, ``text_agreement_fields``) on
|
|
``quality_metrics``.
|
|
|
|
No own dispatch logic — everything interesting lives in the normalisers
|
|
+ verifier modules and is unit-tested there.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from typing import cast
|
|
|
|
from pydantic import BaseModel
|
|
|
|
from ix.contracts import RequestIX, ResponseIX
|
|
from ix.pipeline.step import Step
|
|
from ix.provenance import apply_reliability_flags
|
|
|
|
|
|
class ReliabilityStep(Step):
|
|
"""Fills per-field reliability flags on ``response.provenance``."""
|
|
|
|
async def validate(self, request_ix: RequestIX, response_ix: ResponseIX) -> bool:
|
|
if not request_ix.options.provenance.include_provenance:
|
|
return False
|
|
return response_ix.provenance is not None
|
|
|
|
async def process(
|
|
self, request_ix: RequestIX, response_ix: ResponseIX
|
|
) -> ResponseIX:
|
|
assert response_ix.provenance is not None # validate() guarantees
|
|
|
|
ctx = response_ix.context
|
|
texts: list[str] = (
|
|
list(getattr(ctx, "texts", []) or []) if ctx is not None else []
|
|
)
|
|
use_case_response_cls = cast(
|
|
type[BaseModel],
|
|
getattr(ctx, "use_case_response", None) if ctx is not None else None,
|
|
)
|
|
|
|
apply_reliability_flags(
|
|
provenance_data=response_ix.provenance,
|
|
use_case_response=use_case_response_cls,
|
|
texts=texts,
|
|
)
|
|
return response_ix
|
|
|
|
|
|
__all__ = ["ReliabilityStep"]
|