Skip to content

Metric

Metric

Simple base class / interface definition for metrics.

Source code in src/kibad_llm/metric.py 
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class Metric:
    """Simple base class / interface definition for metrics."""

    def reset(self) -> None:
        """Reset all internal states."""
        raise NotImplementedError("Subclasses should implement this method.")

    def _update(self, prediction: Any, reference: Any, record_id: Hashable | None = None) -> None:
        """Update internal states with new data."""
        raise NotImplementedError("Subclasses should implement this method.")

    def update(self, prediction: Any, reference: Any, record_id: Hashable | None = None) -> None:
        self._update(prediction=prediction, reference=reference, record_id=record_id)

    def _compute(self, *args, **kwargs) -> dict[str, Any]:
        """Compute and return the metric results."""
        raise NotImplementedError("Subclasses should implement this method.")

    def compute(self, *args, reset: bool = True, **kwargs) -> dict[str, Any]:
        result = self._compute(*args, **kwargs)
        if reset:
            self.reset()
        return result

    def _format_result(self, result: dict[str, Any]) -> str:
        """Utility method to format the metric result as a pretty-printed JSON string."""
        return json.dumps(result, indent=2)

    def show_result(self, result: dict[str, Any] | None = None, reset: bool = True) -> None:
        """Utility method to print the metric result in a readable format."""
        if result is None:
            result = self.compute(reset=reset)

        logger.info(f"Evaluation results:\n{self._format_result(result)}")

reset()

Reset all internal states.

Source code in src/kibad_llm/metric.py 
12
13
14
def reset(self) -> None:
    """Reset all internal states."""
    raise NotImplementedError("Subclasses should implement this method.")

show_result(result=None, reset=True)

Utility method to print the metric result in a readable format.

Source code in src/kibad_llm/metric.py 
37
38
39
40
41
42
def show_result(self, result: dict[str, Any] | None = None, reset: bool = True) -> None:
    """Utility method to print the metric result in a readable format."""
    if result is None:
        result = self.compute(reset=reset)

    logger.info(f"Evaluation results:\n{self._format_result(result)}")