Add statistics attribute to DataRecord (#302)

* add Statistic class
* introduce frame factory to handle passing of statistics attribute
* add xg, psxg and obv for statstbomb
* add xg and psxg for opta
* add xg and psxg for wyscout

Author: DriesDeprest

kloppy/domain/models/common.py

@@ -928,6 +928,85 @@ class DatasetFlag(Flag):
     BALL_STATE = 2
 
 
+@dataclass
+class Statistic(ABC):
+    name: str = field(init=False)
+
+
+@dataclass
+class ScalarStatistic(Statistic):
+    value: float
+
+
+@dataclass
+class ExpectedGoals(ScalarStatistic):
+    """Expected goals"""
+
+    def __post_init__(self):
+        self.name = "xG"
+
+
+@dataclass
+class PostShotExpectedGoals(ScalarStatistic):
+    """Post-shot expected goals"""
+
+    def __post_init__(self):
+        self.name = "PSxG"
+
+
+@dataclass
+class ActionValue(Statistic):
+    """Action value"""
+
+    name: str
+    action_value_scoring_before: Optional[float] = field(default=None)
+    action_value_scoring_after: Optional[float] = field(default=None)
+    action_value_conceding_before: Optional[float] = field(default=None)
+    action_value_conceding_after: Optional[float] = field(default=None)
+
+    @property
+    def offensive_value(self) -> Optional[float]:
+        return (
+            None
+            if None
+            in (
+                self.action_value_scoring_before,
+                self.action_value_scoring_after,
+            )
+            else self.action_value_scoring_after
+            - self.action_value_scoring_before
+        )
+
+    @property
+    def defensive_value(self) -> Optional[float]:
+        return (
+            None
+            if None
+            in (
+                self.action_value_conceding_before,
+                self.action_value_conceding_after,
+            )
+            else self.action_value_conceding_after
+            - self.action_value_conceding_before
+        )
+
+    @property
+    def value(self) -> Optional[float]:
+        if None in (
+            self.action_value_scoring_before,
+            self.action_value_scoring_after,
+            self.action_value_conceding_before,
+            self.action_value_conceding_after,
+        ):
+            return None
+        return (
+            self.action_value_scoring_after - self.action_value_scoring_before
+        ) - (
+            self.action_value_conceding_after
+            - self.action_value_conceding_before
+        )
+
+
 @dataclass
 class DataRecord(ABC):
     """
@@ -945,6 +1024,7 @@ class DataRecord(ABC):
     next_record: Optional["DataRecord"] = field(init=False)
     period: Period
     timestamp: timedelta
+    statistics: List[Statistic]
     ball_owning_team: Optional[Team]
     ball_state: Optional[BallState]
 

kloppy/domain/services/event_factory.py

@@ -48,6 +48,9 @@ def create_event(event_cls: Type[T], **kwargs) -> T:
     if "freeze_frame" not in kwargs:
         kwargs["freeze_frame"] = None
 
+    if "statistics" not in kwargs:
+        kwargs["statistics"] = []
+
     all_kwargs = dict(**kwargs, **extra_kwargs)
 
     relevant_kwargs = {
@@ -66,7 +69,9 @@ def create_event(event_cls: Type[T], **kwargs) -> T:
             f"The following arguments were skipped: {skipped_kwargs}"
         )
 
-    return event_cls(**relevant_kwargs)
+    event = event_cls(**relevant_kwargs)
+
+    return event
 
 
 class EventFactory:

kloppy/domain/services/frame_factory.py

@@ -0,0 +1,36 @@
+import dataclasses
+import warnings
+from dataclasses import fields
+
+from kloppy.domain import Frame
+
+
+def create_frame(**kwargs) -> Frame:
+    """
+    Do the actual construction of a frame.
+
+    This method does a couple of things:
+    1. Fill in some arguments when not passed
+    2. Pass only arguments that are accepted by the Frame class.
+    """
+    if "statistics" not in kwargs:
+        kwargs["statistics"] = []
+
+    relevant_kwargs = {
+        field.name: kwargs.get(field.name, field.default)
+        for field in fields(Frame)
+        if field.init
+        and not (
+            field.default == dataclasses.MISSING and field.name not in kwargs
+        )
+    }
+
+    if len(relevant_kwargs) < len(kwargs):
+        skipped_kwargs = set(kwargs.keys()) - set(relevant_kwargs.keys())
+        warnings.warn(
+            f"The following arguments were skipped: {skipped_kwargs}"
+        )
+
+    frame = Frame(**relevant_kwargs)
+
+    return frame

kloppy/domain/services/transformers/dataset.py

@@ -220,6 +220,7 @@ def __change_frame_coordinate_system(self, frame: Frame):
                 for key, player_data in frame.players_data.items()
             },
             other_data=frame.other_data,
+            statistics=frame.statistics,
         )
 
     def __change_frame_dimensions(self, frame: Frame):
@@ -246,6 +247,7 @@ def __change_frame_dimensions(self, frame: Frame):
                 for key, player_data in frame.players_data.items()
             },
             other_data=frame.other_data,
+            statistics=frame.statistics,
         )
 
     def __change_point_coordinate_system(
@@ -303,6 +305,7 @@ def __flip_frame(self, frame: Frame):
             ball_coordinates=self.flip_point(frame.ball_coordinates),
             players_data=players_data,
             other_data=frame.other_data,
+            statistics=frame.statistics,
         )
 
     def transform_event(self, event: Event) -> Event:

kloppy/infra/serializers/code/sportscode.py

@@ -68,6 +68,7 @@ def deserialize(self, inputs: SportsCodeInputs) -> CodeDataset:
                 labels=parse_labels(instance),
                 ball_state=None,
                 ball_owning_team=None,
+                statistics=[],
             )
             period.end_timestamp = end_timestamp
             codes.append(code)

kloppy/infra/serializers/event/statsbomb/helpers.py

@@ -11,7 +11,10 @@
     Player,
     PlayerData,
     PositionType,
+    ActionValue,
+    Provider,
 )
+from kloppy.domain.services.frame_factory import create_frame
 from kloppy.exceptions import DeserializationError
 
 
@@ -21,6 +24,25 @@ def parse_str_ts(timestamp: str) -> float:
     return timedelta(seconds=int(h) * 3600 + int(m) * 60 + float(s))
 
 
+def parse_obv_values(raw_event: dict) -> Optional[ActionValue]:
+    game_state_values_data = {}
+    obv_mapping = {
+        "obv_for_before": "action_value_scoring_before",
+        "obv_against_before": "action_value_conceding_before",
+        "obv_for_after": "action_value_scoring_after",
+        "obv_against_after": "action_value_conceding_after",
+    }
+    for sb_name, kloppy_name in obv_mapping.items():
+        obv_value = raw_event.get(sb_name)
+        if obv_value is not None:
+            game_state_values_data[kloppy_name] = obv_value
+
+    if game_state_values_data:
+        game_state_value = ActionValue(name="OBV", **game_state_values_data)
+
+        return game_state_value
+
+
 def get_team_by_id(team_id: int, teams: List[Team]) -> Team:
     """Get a team by its id."""
     if str(team_id) == teams[0].team_id:
@@ -129,7 +151,7 @@ def get_player_from_freeze_frame(player_data, team, i):
         + event.timestamp.total_seconds() * FREEZE_FRAME_FPS
     )
 
-    return Frame(
+    frame = create_frame(
         frame_id=frame_id,
         ball_coordinates=Point3D(
             x=event.coordinates.x, y=event.coordinates.y, z=0
@@ -141,3 +163,5 @@ def get_player_from_freeze_frame(player_data, team, i):
         ball_owning_team=event.ball_owning_team,
         other_data={"visible_area": visible_area},
     )
+
+    return frame

kloppy/infra/serializers/event/statsbomb/specification.py

@@ -27,13 +27,16 @@
     FormationType,
     PositionType,
     CounterAttackQualifier,
+    ExpectedGoals,
+    PostShotExpectedGoals,
 )
 from kloppy.exceptions import DeserializationError
 from kloppy.infra.serializers.event.statsbomb.helpers import (
     parse_str_ts,
     get_team_by_id,
     get_period_by_id,
     parse_coordinates,
+    parse_obv_values,
 )
 
 
@@ -293,6 +296,7 @@ def deserialize(self, event_factory: EventFactory) -> List[Event]:
         return events
 
     def _parse_generic_kwargs(self) -> Dict:
+        game_state_value = parse_obv_values(self.raw_event)
         return {
             "period": self.period,
             "timestamp": parse_str_ts(self.raw_event["timestamp"]),
@@ -311,6 +315,7 @@ def _parse_generic_kwargs(self) -> Dict:
             ),
             "related_event_ids": self.raw_event.get("related_events", []),
             "raw_event": self.raw_event,
+            "statistics": [game_state_value] if game_state_value else [],
         }
 
     def _create_aerial_won_event(
@@ -548,6 +553,16 @@ def _create_events(
             EVENT_TYPE.SHOT, shot_dict
         ) + _get_body_part_qualifiers(shot_dict)
 
+        for statistic_cls, prop_name in {
+            ExpectedGoals: "statsbomb_xg",
+            PostShotExpectedGoals: "shot_execution_xg",
+        }.items():
+            value = shot_dict.get(prop_name, None)
+            if value is not None:
+                generic_event_kwargs["statistics"].append(
+                    statistic_cls(value=value)
+                )
+
         shot_event = event_factory.build_shot(
             result=result,
             qualifiers=qualifiers,

kloppy/infra/serializers/event/statsperform/deserializer.py

@@ -35,6 +35,8 @@
     GoalkeeperActionType,
     CounterAttackQualifier,
     PositionType,
+    ExpectedGoals,
+    PostShotExpectedGoals,
 )
 from kloppy.exceptions import DeserializationError
 from kloppy.infra.serializers.event.deserializer import EventDataDeserializer
@@ -152,6 +154,9 @@
 EVENT_QUALIFIER_FORMATION_PLAYER_IDS = 30
 EVENT_QUALIFIER_FORMATION_PLAYER_POSITIONS = 131
 
+EVENT_QUALIFIER_XG = 321
+EVENT_QUALIFIER_POST_SHOT_XG = 322
+
 event_type_names = {
     1: "pass",
     2: "offside pass",
@@ -403,13 +408,27 @@ def _parse_shot(raw_event: OptaEvent) -> Dict:
                 y=100 - result_coordinates.y,
             )
 
-    return dict(
+    event_info = dict(
         coordinates=coordinates,
         result=result,
         result_coordinates=result_coordinates,
         qualifiers=qualifiers,
     )
 
+    statistics = []
+    for event_qualifier, statistic in zip(
+        [EVENT_QUALIFIER_XG, EVENT_QUALIFIER_POST_SHOT_XG],
+        [ExpectedGoals, PostShotExpectedGoals],
+    ):
+        xg_value = raw_event.qualifiers.get(event_qualifier)
+        if xg_value:
+            statistics.append(statistic(value=float(xg_value)))
+
+    if statistics:
+        event_info["statistics"] = statistics
+
+    return event_info
+
 
 def _parse_goalkeeper_events(raw_event: OptaEvent) -> Dict:
     qualifiers = _get_event_qualifiers(raw_event.qualifiers)

kloppy/infra/serializers/event/wyscout/deserializer_v3.py

@@ -37,6 +37,10 @@
     ShotResult,
     TakeOnResult,
     Team,
+    FormationType,
+    CarryResult,
+    ExpectedGoals,
+    PostShotExpectedGoals,
 )
 from kloppy.exceptions import DeserializationError, DeserializationWarning
 from kloppy.utils import performance_logging
@@ -269,10 +273,20 @@ def _parse_shot(raw_event: Dict) -> Dict:
     elif raw_event["shot"]["bodyPart"] == "right_foot":
         qualifiers.append(BodyPartQualifier(value=BodyPart.RIGHT_FOOT))
 
+    statistics = []
+    for statistic_cls, prop_name in {
+        ExpectedGoals: "xg",
+        PostShotExpectedGoals: "postShotXg",
+    }.items():
+        value = raw_event["shot"].get(prop_name, None)
+        if value is not None:
+            statistics.append(statistic_cls(value=value))
+
     return {
         "result": result,
         "result_coordinates": _create_shot_result_coordinates(raw_event),
         "qualifiers": qualifiers,
+        "statistics": statistics,
     }
 
 

kloppy/infra/serializers/tracking/metrica_csv.py

@@ -20,6 +20,7 @@
     Player,
     PlayerData,
 )
+from kloppy.domain.services.frame_factory import create_frame
 from kloppy.infra.serializers.tracking.deserializer import (
     TrackingDataDeserializer,
 )
@@ -188,7 +189,7 @@ def deserialize(
                     **away_partial_frame.players_data,
                 }
 
-                frame = Frame(
+                frame = create_frame(
                     frame_id=frame_id,
                     timestamp=timedelta(seconds=frame_id / frame_rate)
                     - period.start_timestamp,