"""
Backtesting for evaluating prediction accuracy against historical data.
Runs the betting model on past matches (simulating pre-match knowledge),
compares predictions to actual outcomes, and reports win / double-chance
accuracy metrics via ``BacktestResults`` and ``format_backtest_report``.
"""
from dataclasses import dataclass, field
from datetime import datetime, timedelta
from pathlib import Path
from slickbet.api import (
HistoricalMatch,
LivescoreAPIError,
LivescoreClient,
Match,
)
from slickbet.model import BetOutcome, BetPrediction, BettingModel
[docs]
@dataclass
class PredictionResult:
"""Result of a single prediction compared to actual outcome."""
match: HistoricalMatch
prediction: BetPrediction
actual_outcome: str # "H", "A", or "D"
predicted_outcome: str # "H" or "A" (we don't predict draws)
is_correct: bool
probability: float
confidence: float
# Double chance results
home_or_draw_correct: bool = False # 1X bet would have won
away_or_draw_correct: bool = False # X2 bet would have won
no_draw_correct: bool = False # 12 bet would have won
@property
def predicted_team(self) -> str:
"""Name of the team we predicted to win."""
if self.predicted_outcome == "H":
return self.match.home_team.name
return self.match.away_team.name
@property
def actual_winner(self) -> str:
"""Name of the actual winner (or 'Draw')."""
if self.actual_outcome == "H":
return self.match.home_team.name
elif self.actual_outcome == "A":
return self.match.away_team.name
return "Draw"
@property
def best_double_chance_correct(self) -> bool:
"""Whether the recommended double chance bet would have won."""
if self.prediction.double_chance is None:
return False
dc = self.prediction.double_chance
best_type, _ = dc.best_double_chance
if "Home or Draw" in best_type:
return self.home_or_draw_correct
elif "Away or Draw" in best_type:
return self.away_or_draw_correct
else:
return self.no_draw_correct
[docs]
@dataclass
class BacktestResults:
"""Aggregated results from a backtest run."""
results: list[PredictionResult] = field(default_factory=list)
competition: str = ""
from_date: datetime | None = None
to_date: datetime | None = None
@property
def total_predictions(self) -> int:
"""Total number of predictions made."""
return len(self.results)
@property
def correct_predictions(self) -> int:
"""Number of correct predictions."""
return sum(1 for r in self.results if r.is_correct)
@property
def accuracy(self) -> float:
"""Overall accuracy (0-1)."""
if not self.results:
return 0.0
return self.correct_predictions / self.total_predictions
@property
def home_predictions(self) -> list[PredictionResult]:
"""Predictions where we bet on home team."""
return [r for r in self.results if r.predicted_outcome == "H"]
@property
def away_predictions(self) -> list[PredictionResult]:
"""Predictions where we bet on away team."""
return [r for r in self.results if r.predicted_outcome == "A"]
@property
def home_accuracy(self) -> float:
"""Accuracy for home win predictions."""
home = self.home_predictions
if not home:
return 0.0
return sum(1 for r in home if r.is_correct) / len(home)
@property
def away_accuracy(self) -> float:
"""Accuracy for away win predictions."""
away = self.away_predictions
if not away:
return 0.0
return sum(1 for r in away if r.is_correct) / len(away)
@property
def high_confidence_results(self) -> list[PredictionResult]:
"""Predictions with confidence > 0.3."""
return [r for r in self.results if r.confidence > 0.3]
@property
def high_confidence_accuracy(self) -> float:
"""Accuracy for high confidence predictions."""
high_conf = self.high_confidence_results
if not high_conf:
return 0.0
return sum(1 for r in high_conf if r.is_correct) / len(high_conf)
@property
def high_confidence_accuracy_excl_draws(self) -> float:
"""Accuracy for high confidence predictions, excluding draws."""
high_conf = [r for r in self.results if r.confidence > 0.3 and r.actual_outcome != "D"]
if not high_conf:
return 0.0
return sum(1 for r in high_conf if r.is_correct) / len(high_conf)
@property
def low_draw_risk_results(self) -> list[PredictionResult]:
"""Predictions where draw risk < 30%."""
return [r for r in self.results if r.prediction.draw_risk < 0.30]
@property
def low_draw_risk_accuracy(self) -> float:
"""Accuracy for low draw risk predictions."""
low_dr = self.low_draw_risk_results
if not low_dr:
return 0.0
return sum(1 for r in low_dr if r.is_correct) / len(low_dr)
@property
def high_probability_results(self) -> list[PredictionResult]:
"""Predictions with probability > 60%."""
return [r for r in self.results if r.probability > 0.60]
@property
def high_probability_accuracy(self) -> float:
"""Accuracy for high probability predictions (>60%)."""
high_prob = self.high_probability_results
if not high_prob:
return 0.0
return sum(1 for r in high_prob if r.is_correct) / len(high_prob)
@property
def high_probability_accuracy_excl_draws(self) -> float:
"""Accuracy for high probability predictions (>60%), excluding draws."""
high_prob = [r for r in self.results if r.probability > 0.60 and r.actual_outcome != "D"]
if not high_prob:
return 0.0
return sum(1 for r in high_prob if r.is_correct) / len(high_prob)
@property
def draws_encountered(self) -> int:
"""Number of matches that ended in a draw."""
return sum(1 for r in self.results if r.actual_outcome == "D")
@property
def accuracy_excluding_draws(self) -> float:
"""Accuracy excluding matches that ended in draws."""
non_draws = [r for r in self.results if r.actual_outcome != "D"]
if not non_draws:
return 0.0
return sum(1 for r in non_draws if r.is_correct) / len(non_draws)
# Double Chance statistics
@property
def home_or_draw_accuracy(self) -> float:
"""Accuracy for 1X (Home or Draw) double chance bets."""
if not self.results:
return 0.0
return sum(1 for r in self.results if r.home_or_draw_correct) / len(self.results)
@property
def away_or_draw_accuracy(self) -> float:
"""Accuracy for X2 (Away or Draw) double chance bets."""
if not self.results:
return 0.0
return sum(1 for r in self.results if r.away_or_draw_correct) / len(self.results)
@property
def no_draw_accuracy(self) -> float:
"""Accuracy for 12 (No Draw) double chance bets."""
if not self.results:
return 0.0
return sum(1 for r in self.results if r.no_draw_correct) / len(self.results)
@property
def best_double_chance_accuracy(self) -> float:
"""Accuracy when following the recommended double chance bet."""
if not self.results:
return 0.0
return sum(1 for r in self.results if r.best_double_chance_correct) / len(self.results)
[docs]
def by_probability_threshold(self, threshold: float) -> "BacktestResults":
"""
Filter results by minimum probability threshold.
Parameters
----------
threshold : float
Minimum prediction probability (inclusive).
Returns
-------
BacktestResults
New results object with filtered predictions.
"""
filtered = BacktestResults(
results=[r for r in self.results if r.probability >= threshold],
competition=self.competition,
from_date=self.from_date,
to_date=self.to_date,
)
return filtered
[docs]
class Backtester:
"""
Backtester for evaluating betting model against historical data.
Examples
--------
>>> backtester = Backtester()
>>> results = backtester.run(
... competition_id="1", # Premier League
... weeks=4,
... )
>>> print(f"Accuracy: {results.accuracy:.1%}")
"""
# Known competition IDs (from Livescore API)
BUNDESLIGA = "1" # Germany
PREMIER_LEAGUE = "2" # England
LA_LIGA = "3" # Spain
SERIE_A = "4" # Italy
LIGUE_1 = "5" # France
def __init__(
self,
client: LivescoreClient | None = None,
model: BettingModel | None = None,
cache_dir: str | Path | None = None,
cache_only: bool = False,
):
"""
Initialize the backtester.
Parameters
----------
client : LivescoreClient or None, optional
Livescore API client (if None, one is created with cache_dir/cache_only)
model : BettingModel or None, optional
Betting model to evaluate
cache_dir : str or Path or None, optional
If set, API responses are cached here for fast reruns and tuning.
cache_only : bool, optional
If True and cache_dir set, use only cache (no API calls). For offline runs.
"""
if client is not None:
self.client = client
else:
self.client = LivescoreClient(
cache_dir=cache_dir,
cache_only=cache_only,
)
self.model = model or BettingModel()
[docs]
def run(
self,
competition_id: str = "1",
weeks: int = 4,
from_date: datetime | None = None,
to_date: datetime | None = None,
min_probability: float = 0.0,
verbose: bool = True,
debug: bool = False,
) -> BacktestResults:
"""
Run backtest on historical data.
Parameters
----------
competition_id : str, optional
Competition ID to backtest
weeks : int, optional
Number of weeks of history (if from_date not specified)
from_date : datetime or None, optional
Start date for historical data
to_date : datetime or None, optional
End date for historical data (defaults to today)
min_probability : float, optional
Minimum probability threshold for predictions
verbose : bool, optional
Print progress information
debug : bool, optional
Print detailed match-by-match predictions and results
Returns
-------
BacktestResults
BacktestResults with all predictions and accuracy metrics
"""
# Calculate date range: current date minus n weeks
if to_date is None:
to_date = datetime.now()
if from_date is None:
from_date = to_date - timedelta(weeks=weeks)
if verbose:
print("๐ Fetching historical data...")
print(f" Competition ID: {competition_id}")
print(
f" Date range: {from_date.strftime('%Y-%m-%d')} to {to_date.strftime('%Y-%m-%d')}"
)
# Fetch historical matches
matches = self._fetch_all_history(
competition_id=competition_id,
from_date=from_date,
to_date=to_date,
verbose=verbose,
)
if verbose:
print(f" Found {len(matches)} completed matches")
# Filter to only finished matches with clear outcomes
valid_matches = [
m for m in matches if m.status == "FINISHED" and m.outcomes.full_time in ("1", "X", "2")
]
if verbose:
print(f" Valid matches with outcomes: {len(valid_matches)}")
print()
print("๐ฏ Running predictions...")
# Run predictions
results = BacktestResults(
competition=competition_id,
from_date=from_date,
to_date=to_date,
)
for i, hist_match in enumerate(valid_matches):
try:
# Convert HistoricalMatch to Match for prediction
# (simulating not knowing the outcome)
match = self._historical_to_match(hist_match)
# Enrich with stats (this simulates what we'd do pre-match)
match = self._enrich_for_backtest(match, hist_match.date, verbose=debug)
# Generate prediction
prediction = self.model.predict(match)
# Skip if below probability threshold
if prediction.probability < min_probability:
continue
# Determine predicted outcome
if prediction.recommended_outcome == BetOutcome.HOME_WIN:
predicted = "H"
else:
predicted = "A"
# Get actual outcome
actual = hist_match.result # "H", "A", or "D"
# Check if correct
is_correct = predicted == actual
# Calculate double chance results
# 1X (Home or Draw): correct if actual is H or D
home_or_draw_correct = actual in ("H", "D")
# X2 (Away or Draw): correct if actual is A or D
away_or_draw_correct = actual in ("A", "D")
# 12 (No Draw): correct if actual is H or A
no_draw_correct = actual in ("H", "A")
result = PredictionResult(
match=hist_match,
prediction=prediction,
actual_outcome=actual or "D",
predicted_outcome=predicted,
is_correct=is_correct,
probability=prediction.probability,
confidence=prediction.confidence,
home_or_draw_correct=home_or_draw_correct,
away_or_draw_correct=away_or_draw_correct,
no_draw_correct=no_draw_correct,
)
results.results.append(result)
# Debug output: show match details, prediction, and actual result
if debug:
self._print_debug_match(result)
if verbose and (i + 1) % 10 == 0:
print(f" Processed {i + 1}/{len(valid_matches)} matches...")
except Exception as e:
if verbose:
print(f" โ Failed to process match: {e}")
continue
if verbose:
print(f" Completed {len(results.results)} predictions")
# Count how many matches had statistics available
matches_with_stats = 0
matches_without_stats = 0
for result in results.results:
pred = result.prediction
if pred.match_stats_used:
matches_with_stats += 1
else:
matches_without_stats += 1
if matches_with_stats > 0 or matches_without_stats > 0:
total = matches_with_stats + matches_without_stats
stats_pct = (matches_with_stats / total * 100) if total > 0 else 0
print(
f" ๐ Match statistics: {matches_with_stats}/{total} "
f"({stats_pct:.1f}%) had stats available"
)
if matches_without_stats > 0:
print(
f" โ ๏ธ Note: {matches_without_stats} matches had no stats "
f"(common for older matches or certain leagues)"
)
print()
return results
def _fetch_all_history(
self,
competition_id: str,
from_date: datetime,
to_date: datetime,
verbose: bool = True,
) -> list[HistoricalMatch]:
"""
Fetch all historical matches, handling pagination.
Parameters
----------
competition_id : str
Competition ID to fetch.
from_date : datetime
Start of date range.
to_date : datetime
End of date range.
verbose : bool, optional
Print page progress.
Returns
-------
list[HistoricalMatch]
Matches in the date range for the competition.
"""
all_matches = []
page = 1
max_pages = 20 # Safety limit
while page <= max_pages:
try:
matches = self.client.get_history(
competition_id=competition_id,
from_date=from_date,
to_date=to_date,
page=page,
)
if not matches:
break
# Filter to date range AND competition (API might return unexpected data)
matches = [
m
for m in matches
if from_date <= m.date <= to_date and m.competition_id == competition_id
]
all_matches.extend(matches)
page += 1
if verbose:
print(f" Fetched page {page - 1}: {len(matches)} matches")
except LivescoreAPIError as e:
if verbose:
print(f" โ API error on page {page}: {e}")
break
return all_matches
def _historical_to_match(self, hist: HistoricalMatch) -> Match:
"""
Convert a HistoricalMatch to a Match for prediction.
Parameters
----------
hist : HistoricalMatch
Completed historical match.
Returns
-------
Match
Pre-match style Match (status ``NS``, no scores).
"""
return Match(
id=hist.id,
home_team=hist.home_team,
away_team=hist.away_team,
competition=hist.competition,
competition_id=hist.competition_id,
country=hist.country or "Unknown",
kickoff_time=hist.date,
status="NS", # Pretend it hasn't started
pre_odds=hist.pre_odds,
)
def _enrich_for_backtest(
self, match: Match, match_date: datetime, verbose: bool = False
) -> Match:
"""
Enrich match with stats for backtesting.
Note: For a true backtest, we should only use data available
BEFORE the match date. This is a simplified version.
Parameters
----------
match : Match
Match to enrich
match_date : datetime
Date of the match
verbose : bool, optional
Print debug information about statistics availability
Returns
-------
Match
Enriched match (form, standings, H2H, performance when available).
"""
home_id = match.home_team.id if match.home_team else ""
away_id = match.away_team.id if match.away_team else ""
comp_id = match.competition_id or ""
# Skip enrichment if we don't have valid IDs
if not home_id or not away_id:
return match
try:
# Get team forms (recent matches before this match)
match.home_form = self.client.get_team_form(home_id)
match.away_form = self.client.get_team_form(away_id)
except Exception:
# Catch all exceptions during form retrieval
pass
try:
# Get standings
if comp_id:
standings = self.client.get_team_standings(comp_id)
if isinstance(standings, dict):
match.home_position = standings.get(home_id)
match.away_position = standings.get(away_id)
except Exception:
# Catch all exceptions during standings retrieval
pass
try:
match.head_to_head = self.client.get_head_to_head(home_id, away_id)
except Exception:
# Catch all exceptions during H2H retrieval
pass
try:
# Get detailed performance stats (goals, clean sheets, etc.)
# This includes match statistics (xG, shots, possession, etc.)
match.home_performance = self.client.get_team_performance_stats(
home_id, num_matches=10, verbose=verbose
)
match.away_performance = self.client.get_team_performance_stats(
away_id, num_matches=10, verbose=verbose
)
# Debug: Check if match statistics are available
if verbose:
home_stats = (
match.home_performance.matches_with_stats if match.home_performance else 0
)
away_stats = (
match.away_performance.matches_with_stats if match.away_performance else 0
)
if home_stats == 0 or away_stats == 0:
print(
f" โ ๏ธ Match stats: Home {home_stats}, Away {away_stats} "
f"(stats may not be available for historical matches)"
)
except Exception:
# Catch all exceptions during performance stats retrieval
pass
return match
def _get_country_flag(self, country: str) -> str:
"""
Get country flag emoji based on country name.
Parameters
----------
country : str
Country name
Returns
-------
str
Flag emoji or empty string if not found
"""
if not country:
return ""
country_lower = country.lower().strip()
# Country to flag mapping (with variations)
country_flags = {
"germany": "๐ฉ๐ช",
"deutschland": "๐ฉ๐ช",
"england": "๐ฌ๐ง",
"united kingdom": "๐ฌ๐ง",
"uk": "๐ฌ๐ง",
"spain": "๐ช๐ธ",
"espana": "๐ช๐ธ",
"italy": "๐ฎ๐น",
"italia": "๐ฎ๐น",
"france": "๐ซ๐ท",
"belgium": "๐ง๐ช",
"belgie": "๐ง๐ช",
"portugal": "๐ต๐น",
"turkey": "๐น๐ท",
"turkiye": "๐น๐ท",
"netherlands": "๐ณ๐ฑ",
"holland": "๐ณ๐ฑ",
"croatia": "๐ญ๐ท",
"hrvatska": "๐ญ๐ท",
"poland": "๐ต๐ฑ",
"polska": "๐ต๐ฑ",
"scotland": "๐ด๓ ง๓ ข๓ ณ๓ ฃ๓ ด๓ ฟ",
"greece": "๐ฌ๐ท",
"hellas": "๐ฌ๐ท",
"saudi arabia": "๐ธ๐ฆ",
"saudi": "๐ธ๐ฆ",
"australia": "๐ฆ๐บ",
"argentina": "๐ฆ๐ท",
"brazil": "๐ง๐ท",
"brasil": "๐ง๐ท",
"mexico": "๐ฒ๐ฝ",
"mรฉxico": "๐ฒ๐ฝ",
}
# Try exact match first
if country_lower in country_flags:
return country_flags[country_lower]
# Try partial match (country name contains key or vice versa)
for country_name, flag in country_flags.items():
if country_name in country_lower or country_lower in country_name:
return flag
return ""
def _print_debug_match(self, result: PredictionResult) -> None:
"""
Print detailed debug information for a single match prediction.
Parameters
----------
result : PredictionResult
The prediction result to display
"""
match = result.match
pred = result.prediction
# Get country flag
country_flag = self._get_country_flag(match.country) if match.country else ""
competition_display = (
f"{country_flag} {match.competition}" if country_flag else match.competition
)
# Match header
print()
print("=" * 80)
print(f"๐
{match.date.strftime('%Y-%m-%d')} | {competition_display}")
print(f"๐ {match.home_team.name} vs โ๏ธ {match.away_team.name}")
print("-" * 80)
# Actual result
home_score = match.home_score or 0
away_score = match.away_score or 0
actual_winner = result.actual_winner
print(f"โฝ ACTUAL RESULT: {home_score} - {away_score} ({actual_winner})")
# Our prediction
predicted_team = result.predicted_team
outcome_symbol = "โ
" if result.is_correct else "โ"
print(
f"๐ฏ OUR PREDICTION: {outcome_symbol} {predicted_team} "
f"(Prob: {pred.probability:.1%}, Conf: {pred.confidence:.1%})"
)
# Prediction details
if pred.recommended_outcome == BetOutcome.HOME_WIN:
print(f" โ Bet on: {match.home_team.name} (Home Win)")
else:
print(f" โ Bet on: {match.away_team.name} (Away Win)")
# Double chance info
if pred.double_chance:
dc = pred.double_chance
best_type, best_prob = dc.best_double_chance
print(f" โ Best Double Chance: {best_type} ({best_prob:.1%})")
# Key reasoning (first 3 reasons)
if pred.reasoning:
print(" ๐ Key Factors:")
for reason in pred.reasoning[:3]:
print(f" โข {reason}")
print("=" * 80)