Base

MetricWithPrepareEntryAsSet

Bases: Metric

Base class for metrics that require preparing entries as sets.

Parameters:

Name	Type	Description	Default
field	str | None	Optional; If provided, the field to extract from a dict entry.	None
flatten_dicts	bool	bool; Whether to flatten dict entries before processing.	False
ignore_subfields	dict[str, list] | None	Optional; A dict mapping field names to lists of subfield names to ignore when converting dicts to tuples.	None

Source code in src/kibad_llm/metrics/base.py

class MetricWithPrepareEntryAsSet(Metric):
    """Base class for metrics that require preparing entries as sets.

    Args:
        field: Optional; If provided, the field to extract from a dict entry.
        flatten_dicts: bool; Whether to flatten dict entries before processing.
        ignore_subfields: Optional; A dict mapping field names to lists of subfield names to ignore
            when converting dicts to tuples.
    """

    def __init__(
        self,
        field: str | None = None,
        flatten_dicts: bool = False,
        ignore_subfields: dict[str, list] | None = None,
    ) -> None:
        self.field = field
        self.ignore_subfields = []
        self.flatten_dicts = flatten_dicts
        if ignore_subfields is not None and self.field is not None:
            self.ignore_subfields = ignore_subfields.get(self.field, [])
        super().__init__()

    def _prepare_entry_as_set(self, entry: Any) -> set:
        """Helper method to convert any prediction or reference value into a set of values.

        Uses the provided field to retrieve the correct values from a given dict if necessary.
        Returns empty set when there is no value.
        Wraps any found values into a set whilst keeping the unique values unaltered.
        Removes None values.

        Args:
            entry: Any kind of data structure to maybe extract from and eventually wrap in a set.
        Returns: A set of whatever relevant value was put in.
        """
        if entry is not None and isinstance(entry, dict) and self.flatten_dicts:
            entry = flatten_dict_simple(entry)

        if self.field is not None and entry is not None:
            if not isinstance(entry, dict):
                raise ValueError(
                    f"Expected entry to be a dict when field is set, but got {type(entry)}"
                )
            entry = entry.get(self.field, None)
        if entry is None:
            result = set()
        elif isinstance(entry, (list, set)):
            # convert list entries to tuples (sort each by key to ensure consistent ordering)
            # to make dicts hashable for the set
            maybe_tuples = (
                (
                    _convert_dict_to_tuple(e, ignore_keys=self.ignore_subfields)
                    if isinstance(e, dict)
                    else e
                )
                for e in entry
                if e is not None
            )
            result = set(maybe_tuples)
        elif isinstance(entry, dict):
            result = {_convert_dict_to_tuple(entry, ignore_keys=self.ignore_subfields)}
        else:
            result = {entry}

        return result