#!/usr/bin/env python3
"""
bt_engine.py v3 — Vectorized walk-forward backtesting with PIT fundamentals

Performance
───────────
v1 computed price signals in a per-ticker loop: ~6 min per backtest.
v2 vectorizes all signal computations using matrix operations:
  • Load all weekly close prices into one panel DataFrame (Date × Ticker)
  • Compute drawdown, 52w return, beta, relative return for ALL tickers at
    once using numpy matrix multiply — ~10–15 s for the full 7,200-ticker
    universe
  • Signals panel is computed ONCE and reused across all optimizer trials

v4 adds realism controls:
  • Transaction costs: --tc_bps (default 20 bps one-way) applied to quarterly
    turnover; results report both gross and net returns + turnover drag
  • --strict_pit: tickers without PIT quarterly coverage are excluded instead
    of silently using today's fundamentals for historical dates
  • Known limitation (NOT fixed): the universe is today's listings, so
    delisted stocks are absent → survivorship bias inflates absolute returns.
    Treat results as config comparisons, not achievable performance.

v3 adds Point-In-Time (PIT) fundamentals:
  • Loads cached quarterly statements from data/quarterly/{TICKER}.json
  • Reconstructs P/B, margins, ROE, revenue growth etc. at each rebalance date
    using only data that was publicly available at that date (45-day lag)
  • build_fundamentals_panel() precomputes all ticker × date rows once (~2 min)
  • Eliminates look-ahead bias from using today's valuations for past dates

Methodology
───────────
At each quarterly rebalance date (Jan / Apr / Jul / Oct):
  1. Inject historically-accurate price signals (drawdown_52w, beta,
     relative_52w) from the precomputed signals panel — NO look-ahead
  2. Inject PIT fundamental metrics reconstructed from quarterly statements
     available at that date (with earnings-release lag) — NO look-ahead
  3. Re-run build_scores() so cross-sectional ranks are correct at that date
  4. Select top-N stocks equal-weight, hold one quarter, measure return
  5. Compare against SPY (S&P 500 ETF)

Usage
─────
  python bt_engine.py                                  # defaults
  python bt_engine.py --top_n 20 --train_end 2023-12-31 --eval_start 2024-01-01
  python bt_engine.py --weights "value=40,health=25,sentiment=20,quality=15"
"""

from __future__ import annotations

import argparse
import json
import math
import sys
import time
import warnings
from pathlib import Path
from typing import Any, Dict, List, Optional, Tuple

import numpy as np
import pandas as pd

warnings.filterwarnings("ignore", category=FutureWarning)
warnings.filterwarnings("ignore", category=RuntimeWarning)  # expected NaN in all-NaN tickers

try:
    from rk_tracker import build_scores
except ImportError:
    sys.exit("Cannot import rk_tracker. Run from the project root directory.")


# ── Constants ─────────────────────────────────────────────────────────────────
BENCHMARK        = "SPY"
REBALANCE_MONTHS = [1, 4, 7, 10]
MIN_WEEKS        = 52
DEFAULT_TOP_N    = 20
DEFAULT_WEIGHTS  = {"value": 35, "health": 25, "sentiment": 25, "quality": 15}
ANNUAL_RF        = 0.0
DEFAULT_TC_BPS   = 20.0   # one-way transaction cost in basis points of traded notional
                          # (covers commission + half bid-ask spread + impact; small/mid-caps
                          #  in a contrarian screen often trade wider — tune with --tc_bps)

DEFAULT_GUARDRAILS = {
    "distress_cap": 60,
    "dilution_penalty_threshold": 0.30,
    "dilution_penalty_points": 15,
}


# ── Data loading ──────────────────────────────────────────────────────────────

def load_scores(scores_csv: Path) -> pd.DataFrame:
    """Load rk_tracker scores.csv indexed by ticker.
    Automatically strips special securities (preferred stocks, warrants,
    rights, units) identified by the exchange suffix pattern -(P|W|R|U)[A-Z]{0,2}.
    Dual-class shares like BRK-B and BF-A are retained.
    """
    df = pd.read_csv(scores_csv)
    df["ticker"] = df["ticker"].astype(str).str.upper().str.strip()
    df = df[df["ticker"].str.len() > 0].drop_duplicates("ticker", keep="last")
    # Remove preferred stocks (-PA, -PB…), warrants (-WA, -WS…), rights (-R*), units (-U*)
    special = df["ticker"].str.contains(r"-(P|W|R|U)[A-Z]{0,2}$", regex=True)
    n_removed = special.sum()
    if n_removed:
        import logging
        logging.getLogger(__name__).debug("load_scores: dropped %d special securities", n_removed)
    df = df[~special]
    return df.set_index("ticker")


def load_prices_panel(weekly_dir: Path, tickers: List[str]) -> pd.DataFrame:
    """
    Load all weekly Close prices into a single panel DataFrame.
    Index: Date (sorted ascending)
    Columns: Ticker
    Much faster for vectorized computation than a dict of DataFrames.
    """
    series: Dict[str, pd.Series] = {}
    for t in tickers:
        path = weekly_dir / f"{t}.csv"
        if not path.exists():
            continue
        try:
            df = pd.read_csv(path, parse_dates=["Date"], index_col="Date")
            if "Close" in df.columns and not df.empty:
                series[t] = df["Close"].dropna().sort_index()
        except Exception:
            continue
    panel = pd.DataFrame(series).sort_index()
    return panel


# Backward-compat alias used by bt_optimizer.py
def load_weekly_prices(weekly_dir: Path, tickers: List[str]) -> pd.DataFrame:
    return load_prices_panel(weekly_dir, tickers)


# ── Point-in-time fundamentals ────────────────────────────────────────────────
# Eliminates look-ahead bias: reconstruct P/B, margins, ROE etc. at each
# rebalance date using only quarterly statements released BEFORE that date.

EARNINGS_LAG = pd.Timedelta(days=50)   # ~50 days from quarter-end to release

# Fundamental metrics we reconstruct from quarterly statements.
# Anything not reconstructed falls back to the static value from scores.csv.
PIT_METRICS = [
    "gross_margin", "fcf_margin", "debt_to_equity", "current_ratio",
    "return_on_equity", "revenue_growth", "price_to_book",
    "price_to_sales", "ev_to_ebitda", "net_cash_to_market_cap",
]

# yfinance field name alternatives (tried in order; first hit wins)
_BS_EQUITY   = ["Stockholders Equity", "Total Equity Gross Minority Interest",
                 "Common Stock Equity"]
_BS_DEBT     = ["Total Debt", "Long Term Debt And Capital Lease Obligation"]
_BS_CURR_DEBT = ["Current Debt And Capital Lease Obligation", "Current Debt"]
_BS_CASH     = ["Cash And Cash Equivalents", "Cash Cash Equivalents And Short Term Investments",
                "Cash And Short Term Investments"]
_BS_CUR_ASSETS = ["Current Assets"]
_BS_CUR_LIAB   = ["Current Liabilities"]
_BS_SHARES   = ["Ordinary Shares Number", "Share Issued", "Common Stock Shares Outstanding"]
_INC_REVENUE = ["Total Revenue", "Operating Revenue"]
_INC_GROSS   = ["Gross Profit"]
_INC_NET     = ["Net Income", "Net Income Common Stockholders",
                "Net Income Including Noncontrolling Interests"]
_INC_EBITDA  = ["EBITDA", "Normalized EBITDA", "Reconciled EBITDA"]
_CF_FCF      = ["Free Cash Flow"]
_CF_OCF      = ["Operating Cash Flow"]
_CF_CAPEX    = ["Capital Expenditure"]


def _pick(d: dict, keys: List[str]) -> Optional[float]:
    """Return the first non-None numeric value found among keys in d."""
    for k in keys:
        v = d.get(k)
        if v is not None:
            try:
                f = float(v)
                if f == f:      # not NaN
                    return f
            except (TypeError, ValueError):
                pass
    return None


def load_quarterly_cache(quarterly_dir: Path, tickers: List[str]) -> Dict[str, dict]:
    """
    Load all data/quarterly/{TICKER}.json files into memory.
    Returns {ticker: data_dict}.  Missing / errored files are silently skipped.
    """
    cache: Dict[str, dict] = {}
    qdir = Path(quarterly_dir)
    for t in tickers:
        path = qdir / f"{t}.json"
        if not path.exists():
            continue
        try:
            data = json.loads(path.read_text(encoding="utf-8"))
            if "error" not in data:
                cache[t] = data
        except Exception:
            pass
    return cache


def _latest_q_before(dates: List[str], as_of: pd.Timestamp) -> Optional[str]:
    """
    Return the most recent quarter-end date (from `dates`) whose earnings would
    have been released by `as_of`, applying the EARNINGS_LAG.
    """
    cutoff   = as_of - EARNINGS_LAG
    eligible = [d for d in dates if pd.Timestamp(d) <= cutoff]
    return max(eligible) if eligible else None


def _ttm(stmt: dict, latest_q: str, keys: List[str], n: int = 4) -> Optional[float]:
    """
    Sum the given metric over the last `n` quarters ending at latest_q.
    Tries each key in `keys` until one has data.
    If fewer than n quarters have data, annualises from what's available
    (e.g. 1 quarter × 4, 2 quarters × 2).  This handles yfinance's pattern
    of providing incomplete data for the oldest quarter in the cache.
    Returns None only if no quarters have data.
    """
    all_dates = sorted(stmt.keys(), reverse=True)
    if latest_q not in all_dates:
        return None
    idx  = all_dates.index(latest_q)
    qtrs = all_dates[idx: idx + n]

    for key in keys:
        vals = [stmt[q].get(key) for q in qtrs if q in stmt]
        vals = [float(v) for v in vals if v is not None and float(v) == float(v)]
        if len(vals) >= 1:
            # Annualise when fewer than n quarters are available
            return sum(vals) * (n / len(vals)) if len(vals) < n else sum(vals)
    return None


def get_pit_row(
    ticker:      str,
    q_data:      dict,
    close_panel: pd.DataFrame,
    as_of:       pd.Timestamp,
    static_row:  Optional[Dict[str, Any]] = None,
) -> Dict[str, Any]:
    """
    Reconstruct PIT_METRICS for `ticker` at `as_of` using cached quarterly statements.

    Uses historical price from close_panel for market-cap-dependent ratios.
    Falls back to values in `static_row` (from scores.csv) for any metric
    that can't be reconstructed.

    Returns a dict of {metric_name: value}.
    """
    bs_raw  = q_data.get("balance_sheet",  {})
    inc_raw = q_data.get("income_stmt",    {})
    cf_raw  = q_data.get("cashflow",       {})

    # Find the most recently reported quarter
    all_dates = sorted(set(bs_raw) | set(inc_raw) | set(cf_raw))
    latest_q  = _latest_q_before(all_dates, as_of)

    row: Dict[str, Any] = dict(static_row) if static_row else {}
    row["has_pit"] = latest_q is not None

    if latest_q is None:
        return row   # no historical data available → use static (or exclude in strict mode)

    bs  = bs_raw.get(latest_q,  {})
    inc = inc_raw.get(latest_q, {})

    # ── Balance sheet point values ────────────────────────────────────────────
    equity      = _pick(bs, _BS_EQUITY)
    long_debt   = _pick(bs, _BS_DEBT)
    curr_debt   = _pick(bs, _BS_CURR_DEBT) or 0.0
    total_debt  = (long_debt or 0.0) + curr_debt if long_debt is not None else curr_debt
    cash        = _pick(bs, _BS_CASH)
    cur_assets  = _pick(bs, _BS_CUR_ASSETS)
    cur_liab    = _pick(bs, _BS_CUR_LIAB)
    shares      = _pick(bs, _BS_SHARES)

    if equity and equity > 0:
        row["debt_to_equity"] = total_debt / equity
    if cur_assets and cur_liab and cur_liab > 0:
        row["current_ratio"] = cur_assets / cur_liab

    # ── TTM income metrics ────────────────────────────────────────────────────
    rev_ttm        = _ttm(inc_raw, latest_q, _INC_REVENUE)
    gross_ttm      = _ttm(inc_raw, latest_q, _INC_GROSS)
    net_inc_ttm    = _ttm(inc_raw, latest_q, _INC_NET)
    ebitda_ttm     = _ttm(inc_raw, latest_q, _INC_EBITDA)

    if rev_ttm and rev_ttm > 0:
        if gross_ttm is not None:
            row["gross_margin"] = gross_ttm / rev_ttm
        if net_inc_ttm is not None and equity and equity > 0:
            row["return_on_equity"] = net_inc_ttm / equity

    # ── TTM cashflow ──────────────────────────────────────────────────────────
    fcf_ttm = _ttm(cf_raw, latest_q, _CF_FCF)
    if fcf_ttm is None:
        # derive from operating CF - capex
        ocf   = _ttm(cf_raw, latest_q, _CF_OCF)
        capex = _ttm(cf_raw, latest_q, _CF_CAPEX)
        if ocf is not None and capex is not None:
            fcf_ttm = ocf + capex   # capex is usually negative in yfinance

    if fcf_ttm is not None and rev_ttm and rev_ttm > 0:
        row["fcf_margin"] = fcf_ttm / rev_ttm

    # ── Revenue growth (TTM YoY) ──────────────────────────────────────────────
    prior_q = _latest_q_before(all_dates, as_of - pd.DateOffset(years=1))
    if prior_q:
        prior_rev = _ttm(inc_raw, prior_q, _INC_REVENUE)
        if prior_rev and abs(prior_rev) > 0 and rev_ttm is not None:
            row["revenue_growth"] = (rev_ttm - prior_rev) / abs(prior_rev)

    # ── Price-based ratios (need historical price) ────────────────────────────
    hist_price: Optional[float] = None
    if ticker in close_panel.columns:
        col       = close_panel[ticker].dropna()
        available = col[col.index <= as_of]
        if not available.empty:
            hist_price = float(available.iloc[-1])

    market_cap: Optional[float] = None
    if hist_price and shares and shares > 0:
        market_cap = hist_price * shares

    if market_cap and market_cap > 0:
        if equity and equity > 0:
            row["price_to_book"] = market_cap / equity
        if rev_ttm and rev_ttm > 0:
            row["price_to_sales"] = market_cap / rev_ttm
        if cash is not None:
            net_cash = cash - total_debt
            row["net_cash_to_market_cap"] = net_cash / market_cap
            if ebitda_ttm and ebitda_ttm > 0:
                ev = market_cap + total_debt - (cash or 0.0)
                row["ev_to_ebitda"] = ev / ebitda_ttm

    return row


def build_fundamentals_panel(
    tickers:          List[str],
    quarterly_cache:  Dict[str, dict],
    close_panel:      pd.DataFrame,
    rebalance_dates:  List[pd.Timestamp],
    static_features:  pd.DataFrame,
) -> pd.DataFrame:
    """
    Precompute PIT fundamental metrics for all tickers × all rebalance dates.

    For each (date, ticker) pair:
      - Uses the most recent quarterly statements available BEFORE that date
      - Falls back to static_features row for missing/uncovered tickers

    Returns a DataFrame with MultiIndex (date, ticker) containing PIT_METRICS
    columns.  Build this once and pass into run_backtest / optimizer trials.

    Expected time: ~2–4 minutes for 7,200 tickers × 46 dates.
    """
    rows = []
    n_pit   = 0
    n_static = 0

    for date in rebalance_dates:
        for t in tickers:
            static_r = static_features.loc[t].to_dict() if t in static_features.index else {}

            if t in quarterly_cache:
                pit = get_pit_row(t, quarterly_cache[t], close_panel, date, static_r)
                n_pit += 1
            else:
                pit = dict(static_r)
                pit["has_pit"] = False
                n_static += 1

            pit["_date"]   = date
            pit["_ticker"] = t
            rows.append(pit)

    df = pd.DataFrame(rows)
    df = df.set_index(["_date", "_ticker"])
    df.index.names = ["date", "ticker"]
    return df


# ── Vectorized signal computation ─────────────────────────────────────────────

def build_signals_panel(
    close_panel: pd.DataFrame,
    rebalance_dates: List[pd.Timestamp],
    min_weeks: int = MIN_WEEKS,
) -> pd.DataFrame:
    """
    Compute price-based signals for ALL tickers at ALL rebalance dates
    using vectorized numpy / pandas operations.

    Signals computed at each date (no look-ahead — only history ≤ date used):
      drawdown_52w  : (current_close / 52w_high) - 1
      relative_52w  : ticker 52w return minus SPY 52w return
      beta          : rolling 52w beta vs SPY via matrix multiply
      eligible      : True if ticker has ≥ min_weeks of history at this date

    Returns a DataFrame with MultiIndex (date, ticker).
    """
    spy = BENCHMARK
    tickers_out = [t for t in close_panel.columns if t != spy]

    all_frames: List[pd.DataFrame] = []

    for date in rebalance_dates:
        hist = close_panel[close_panel.index <= date]
        if len(hist) < min_weeks:
            continue

        last52 = hist.iloc[-min_weeks:]

        # ── drawdown_52w ─────────────────────────────────────────────────────
        current = last52.iloc[-1]
        peak    = last52.max()
        drawdown = (current / peak - 1.0).where(peak > 0, np.nan)

        # ── 52-week return ───────────────────────────────────────────────────
        entry = hist.iloc[-min_weeks]
        ret52 = (current / entry - 1.0).where(entry > 0, np.nan)

        # ── relative_52w vs SPY ──────────────────────────────────────────────
        spy_ret52: Optional[float] = None
        if spy in ret52.index and pd.notna(ret52.get(spy)):
            spy_ret52 = float(ret52[spy])

        relative = (ret52 - spy_ret52) if spy_ret52 is not None else pd.Series(np.nan, index=ret52.index)

        # ── beta (matrix multiply — O(T · N) single matmul, not O(N) loops) ─
        beta_arr = np.full(len(close_panel.columns), np.nan)

        if spy in close_panel.columns:
            rets_hist = hist.pct_change().iloc[-min_weeks:].dropna(how="all")
            if spy in rets_hist.columns:
                spy_rets = rets_hist[spy].dropna()
                if len(spy_rets) >= 20:
                    R = rets_hist.reindex(spy_rets.index)   # (n_dates, n_tickers)
                    r_spy = spy_rets.values                  # (n_dates,)

                    r_spy_dm = r_spy - r_spy.mean()
                    var_spy  = float(r_spy_dm @ r_spy_dm)

                    if var_spy > 0:
                        R_arr = R.values.astype(float)
                        # Demean columns — guard against all-NaN columns
                        with np.errstate(all="ignore"):
                            col_means = np.where(
                                np.all(np.isnan(R_arr), axis=0),
                                0.0,
                                np.nanmean(R_arr, axis=0),
                            )
                        R_dm = R_arr - col_means[np.newaxis, :]

                        # Columns with any NaN → mark and zero out
                        nan_cols = np.isnan(R_dm).any(axis=0)
                        R_dm_safe = np.where(np.isnan(R_dm), 0.0, R_dm)

                        # Single matrix-vector multiply: shape (n_tickers,)
                        betas = (r_spy_dm @ R_dm_safe) / var_spy
                        betas[nan_cols] = np.nan

                        # Map back to full column order
                        col_idx = {c: i for i, c in enumerate(close_panel.columns)}
                        for i, c in enumerate(R.columns):
                            if c in col_idx:
                                beta_arr[col_idx[c]] = betas[i]

        beta_series = pd.Series(beta_arr, index=close_panel.columns)

        # ── eligible flag ────────────────────────────────────────────────────
        # Require ≥ min_weeks of TOTAL non-NaN history up to this date
        # (not just in the last 52 rows — that slice is always exactly 52 rows)
        eligible = hist.notna().sum() >= min_weeks

        # ── Assemble for this date ───────────────────────────────────────────
        date_df = pd.DataFrame({
            "drawdown_52w": drawdown,
            "relative_52w": relative,
            "beta":         beta_series,
            "eligible":     eligible,
        }, index=close_panel.columns)

        date_df = date_df.loc[tickers_out].copy()
        date_df.index.name = "ticker"
        date_df["date"] = date
        all_frames.append(date_df)

    if not all_frames:
        return pd.DataFrame(columns=["drawdown_52w", "relative_52w", "beta", "eligible"])

    result = pd.concat(all_frames, axis=0).reset_index()
    result = result.set_index(["date", "ticker"])
    return result


# ── Scoring at a historical date ──────────────────────────────────────────────

def score_universe_at_date(
    date: pd.Timestamp,
    raw_features: pd.DataFrame,
    signals_panel: pd.DataFrame,
    weights: Dict[str, float],
    guardrails: Dict[str, Any],
    fundamentals_panel: Optional[pd.DataFrame] = None,
    strict_pit: bool = False,
) -> pd.DataFrame:
    """
    Compute composite scores for all tickers at `date`.

    1. Starts from raw_features (scores.csv — today's static data as fallback)
    2. If fundamentals_panel is provided, overlays PIT fundamental metrics
       (P/B, margins, ROE etc. reconstructed from quarterly statements available
       at `date` — no look-ahead)
    3. Overlays price signals (drawdown, beta, relative 52w) from signals_panel
    4. Re-runs build_scores() so cross-sectional percentile ranks are correct

    strict_pit — if True, tickers WITHOUT reconstructed PIT fundamentals at
    `date` are marked ineligible instead of silently falling back to today's
    static values (which is look-ahead). Recommended for honest backtests.
    """
    df = raw_features.copy()
    pit_coverage: Optional[pd.Series] = None

    # ── Inject PIT fundamentals (v3: eliminates look-ahead on valuations) ─────
    if fundamentals_panel is not None:
        date_idx = fundamentals_panel.index.get_level_values("date")
        if date in date_idx:
            pit = fundamentals_panel.loc[date]   # DataFrame indexed by ticker
            for col in PIT_METRICS:
                if col in pit.columns:
                    valid  = pit[col].dropna()
                    common = valid.index.intersection(df.index)
                    if len(common):
                        df.loc[common, col] = valid.loc[common]
            if "has_pit" in pit.columns:
                pit_coverage = pit["has_pit"].reindex(df.index).fillna(False).astype(bool)

    # ── Inject historical price signals ──────────────────────────────────────
    date_idx = signals_panel.index.get_level_values("date")
    if date in date_idx:
        signals = signals_panel.loc[date]   # DataFrame indexed by ticker

        for col in ["drawdown_52w", "relative_52w", "beta"]:
            if col in signals.columns:
                valid  = signals[col].dropna()
                common = valid.index.intersection(df.index)
                if len(common):
                    df.loc[common, col] = valid.loc[common]

        if "eligible" in signals.columns:
            df["eligible"] = signals["eligible"].reindex(df.index).fillna(False)
        else:
            df["eligible"] = False
    else:
        df["eligible"] = False

    # ── Strict PIT: exclude tickers with no point-in-time fundamentals ────────
    if strict_pit and pit_coverage is not None:
        df["eligible"] = df["eligible"].astype(bool) & pit_coverage

    # ── Re-run scoring with updated features ──────────────────────────────────
    bw = {
        "value":              weights.get("value",     35),
        "financial_health":   weights.get("health",    25),
        "sentiment_crowding": weights.get("sentiment", 25),
        "quality_momentum":   weights.get("quality",   15),
    }
    scored = build_scores(df.reset_index(), weights=bw, guardrails=guardrails)
    if "ticker" in scored.columns:
        scored = scored.set_index("ticker")

    return scored


# ── Portfolio return ──────────────────────────────────────────────────────────

def portfolio_return(
    tickers: List[str],
    close_panel: pd.DataFrame,
    start: pd.Timestamp,
    end: pd.Timestamp,
) -> Tuple[float, Dict[str, float]]:
    """Equal-weight portfolio return from start to end using the close panel."""
    per_ticker: Dict[str, float] = {}

    for t in tickers:
        if t not in close_panel.columns:
            per_ticker[t] = np.nan
            continue
        col = close_panel[t].dropna()
        after  = col[col.index >= start]
        before = col[col.index <= end]
        if after.empty or before.empty:
            per_ticker[t] = np.nan
            continue
        entry = float(after.iloc[0])
        exit_ = float(before.iloc[-1])
        per_ticker[t] = (exit_ / entry - 1.0) if entry > 0 else np.nan

    valid = [v for v in per_ticker.values() if not np.isnan(v)]
    return (float(np.mean(valid)) if valid else np.nan), per_ticker


# ── Main backtest loop ────────────────────────────────────────────────────────

def _build_rebalance_dates(start: str, end: str) -> List[pd.Timestamp]:
    dates: List[pd.Timestamp] = []
    cur = pd.Timestamp(start).replace(day=1)
    end_ts = pd.Timestamp(end)
    while cur <= end_ts:
        if cur.month in REBALANCE_MONTHS:
            dates.append(cur)
        cur += pd.DateOffset(months=1)
    return sorted(set(dates))


def run_backtest(
    raw_features: pd.DataFrame,
    prices_dict: pd.DataFrame,                        # close panel (Date × Ticker)
    spy_prices: Optional[pd.DataFrame] = None,        # ignored — SPY taken from panel
    start_date: str = "2020-01-01",
    end_date:   str = "2023-12-31",
    top_n:      int = DEFAULT_TOP_N,
    weights:    Optional[Dict[str, float]] = None,
    guardrails: Optional[Dict[str, Any]]  = None,
    label:      str = "",
    signals_panel:      Optional[pd.DataFrame] = None,  # reused across optimizer trials
    fundamentals_panel: Optional[pd.DataFrame] = None,  # PIT features (v3); None = static
    tc_bps:     float = DEFAULT_TC_BPS,                 # one-way cost, bps of traded notional
    strict_pit: bool  = False,                          # exclude tickers w/o PIT coverage
) -> pd.DataFrame:
    """
    Walk-forward quarterly backtest.

    prices_dict    — close panel DataFrame (Date × Ticker)
    signals_panel  — precomputed price signals; built here if not provided
    fundamentals_panel — precomputed PIT fundamentals; if None uses static
                         scores.csv values (mild look-ahead on fundamentals)
    tc_bps         — one-way transaction cost in bps of traded notional.
                     Cost per quarter = tc_bps/1e4 × (buys + sells) as a
                     fraction of portfolio value. First quarter buys 100%.
                     Set 0 to reproduce old frictionless results.
    strict_pit     — if True (and fundamentals_panel given), tickers without
                     PIT quarterly coverage at a rebalance date are ineligible
                     rather than falling back to today's fundamentals.

    Caveat: universe = today's listings (survivorship bias). Delisted tickers
    are absent, so absolute returns are likely overstated; treat results as
    relative comparisons between configs, not achievable performance.

    Returns a DataFrame with per-quarter results (gross + net of costs).
    """
    if weights    is None: weights    = DEFAULT_WEIGHTS.copy()
    if guardrails is None: guardrails = DEFAULT_GUARDRAILS.copy()

    close_panel = prices_dict  # panel DataFrame

    dates = _build_rebalance_dates(start_date, end_date)
    if len(dates) < 2:
        raise ValueError(f"Need ≥ 2 rebalance dates in [{start_date}, {end_date}]")

    # Build signals panel once (or use provided one)
    if signals_panel is None:
        t0 = time.time()
        print(f"  Building signals panel for {len(dates)-1} quarters × "
              f"{len(close_panel.columns):,} tickers...", end=" ", flush=True)
        signals_panel = build_signals_panel(close_panel, dates)
        print(f"done ({time.time()-t0:.1f}s)")

    pit_mode = fundamentals_panel is not None
    tag = f"[{label}] " if label else ""
    print(f"\n{tag}Backtest  {start_date} → {end_date}  "
          f"({len(dates)-1} quarters, top-{top_n})"
          f"  {'[PIT fundamentals]' if pit_mode else '[static fundamentals]'}"
          f"{'  [strict PIT]' if (pit_mode and strict_pit) else ''}"
          f"  [tc {tc_bps:.0f}bps]")
    print(f"  Weights: {weights}")

    rows = []
    tc_rate = tc_bps / 1e4
    prev_portfolio: Optional[set] = None

    for i, rb_date in enumerate(dates[:-1]):
        next_date = dates[i + 1]

        scored = score_universe_at_date(
            date=rb_date,
            raw_features=raw_features,
            signals_panel=signals_panel,
            weights=weights,
            guardrails=guardrails,
            fundamentals_panel=fundamentals_panel,
            strict_pit=strict_pit,
        )

        eligible = scored[scored.get("eligible", pd.Series(True, index=scored.index)) == True]
        if len(eligible) < top_n:
            print(f"  {rb_date.date()} → {next_date.date()} | "
                  f"only {len(eligible)} eligible — skipping")
            continue

        portfolio = eligible.nlargest(top_n, "total_score").index.tolist()
        port_set  = set(portfolio)

        # ── Turnover & transaction cost (equal-weight approximation) ─────────
        # One-way turnover as fraction of portfolio notional traded:
        #   first rebalance → buy 100% (turnover 1.0)
        #   thereafter      → sells + buys of replaced names (2 × fraction swapped)
        if prev_portfolio is None:
            turnover = 1.0
        else:
            n_new    = len(port_set - prev_portfolio)
            turnover = 2.0 * n_new / top_n
        tc_cost = tc_rate * turnover
        prev_portfolio = port_set

        p_gross, ticker_rets = portfolio_return(portfolio, close_panel, rb_date, next_date)
        if np.isnan(p_gross):
            p_ret = np.nan
        elif tc_cost == 0.0:
            p_ret = p_gross
        else:
            p_ret = (1.0 + p_gross) * (1.0 - tc_cost) - 1.0

        # SPY benchmark return (frictionless — buy-and-hold proxy)
        if BENCHMARK in close_panel.columns:
            b_ret, _ = portfolio_return([BENCHMARK], close_panel, rb_date, next_date)
        else:
            b_ret = np.nan

        alpha   = (p_ret - b_ret) if not (np.isnan(p_ret) or np.isnan(b_ret)) else np.nan
        n_valid = sum(1 for v in ticker_rets.values() if not np.isnan(v))

        sign = "+" if (not np.isnan(alpha) and alpha >= 0) else ""
        print(f"  {rb_date.date()} → {next_date.date()} | "
              f"net {p_ret:>+7.1%} (gross {p_gross:>+7.1%}, tc {tc_cost:.2%})  "
              f"SPY {b_ret:>+7.1%}  alpha {sign}{alpha:.1%}  ({n_valid}/{top_n})")

        rows.append({
            "rebalance_date":         rb_date,
            "next_date":              next_date,
            "portfolio_return":       p_ret,      # net of transaction costs
            "portfolio_return_gross": p_gross,
            "turnover":               turnover,
            "tc_cost":                tc_cost,
            "spy_return":             b_ret,
            "outperformance":         alpha,
            "portfolio_tickers":      ",".join(portfolio),
            "n_valid":                n_valid,
        })

    return pd.DataFrame(rows)


# ── Performance metrics ───────────────────────────────────────────────────────

def compute_metrics(results: pd.DataFrame, label: str = "") -> Dict[str, Any]:
    r = results.dropna(subset=["portfolio_return", "spy_return"])
    if r.empty:
        return {"label": label, "n_quarters": 0}

    p = r["portfolio_return"].values
    b = r["spy_return"].values
    n_q     = len(r)
    n_years = n_q / 4.0

    p_cum  = float(np.prod(1 + p))
    b_cum  = float(np.prod(1 + b))
    p_cagr = p_cum ** (1 / n_years) - 1 if n_years > 0 else np.nan
    b_cagr = b_cum ** (1 / n_years) - 1 if n_years > 0 else np.nan

    ann_ret = float(np.mean(p)) * 4
    ann_std = float(np.std(p, ddof=1)) * math.sqrt(4)
    sharpe  = (ann_ret - ANNUAL_RF) / ann_std if ann_std > 0 else np.nan

    excess  = p - b
    ir_std  = float(np.std(excess, ddof=1)) * math.sqrt(4)
    ir      = (float(np.mean(excess)) * 4) / ir_std if ir_std > 0 else np.nan

    nav        = np.cumprod(1 + p)
    running_hi = np.maximum.accumulate(nav)
    max_dd     = float(((nav - running_hi) / running_hi).min())

    out = {
        "label":           label,
        "n_quarters":      n_q,
        "n_years":         round(n_years, 2),
        "port_total_ret":  p_cum - 1,
        "spy_total_ret":   b_cum - 1,
        "port_cagr":       p_cagr,        # net of transaction costs
        "spy_cagr":        b_cagr,
        "alpha_ann":       float(np.mean(excess)) * 4,
        "sharpe":          sharpe,
        "info_ratio":      ir,
        "max_drawdown":    max_dd,
        "win_rate_vs_spy": float(np.mean(p > b)),
    }

    # Gross-of-cost CAGR + turnover diagnostics (present since tc model added)
    if "portfolio_return_gross" in r.columns:
        g = r["portfolio_return_gross"].values
        g_cum = float(np.prod(1 + g))
        out["port_cagr_gross"]  = g_cum ** (1 / n_years) - 1 if n_years > 0 else np.nan
        out["tc_drag_ann"]      = out["port_cagr_gross"] - p_cagr
    if "turnover" in r.columns:
        out["avg_turnover_q"]   = float(r["turnover"].mean())

    return out


def print_metrics(m: Dict[str, Any]) -> None:
    label = m.get("label", "")
    bar   = "─" * 54
    print(f"\n{bar}")
    print(f"  {label}" if label else "  Results")
    print(bar)

    def _f(k: str, fmt: str = ".1%") -> str:
        v = m.get(k, float("nan"))
        return f"{v:{fmt}}" if isinstance(v, float) else str(v)

    print(f"  Period          : {m.get('n_years','?')} yrs  ({m.get('n_quarters','?')} quarters)")
    print(f"  Portfolio CAGR  : {_f('port_cagr'):>10}  (net of costs)")
    if "port_cagr_gross" in m:
        print(f"  Gross CAGR      : {_f('port_cagr_gross'):>10}  (tc drag {_f('tc_drag_ann')}/yr)")
    if "avg_turnover_q" in m:
        print(f"  Avg turnover    : {_f('avg_turnover_q'):>10}  per quarter (one-way)")
    print(f"  S&P 500 CAGR    : {_f('spy_cagr'):>10}")
    print(f"  Alpha (ann.)    : {_f('alpha_ann'):>10}")
    print(f"  Sharpe Ratio    : {m.get('sharpe', float('nan')):>10.2f}")
    print(f"  Info Ratio      : {m.get('info_ratio', float('nan')):>10.2f}")
    print(f"  Max Drawdown    : {_f('max_drawdown'):>10}")
    print(f"  Win Rate vs SPY : {_f('win_rate_vs_spy'):>10}")
    print(f"  Total Return    : {_f('port_total_ret'):>10}")
    print(f"  SPY Total Ret   : {_f('spy_total_ret'):>10}")
    print(bar)


# ── CLI entry ─────────────────────────────────────────────────────────────────

def _parse_weights(s: str) -> Dict[str, float]:
    out = {}
    for part in s.split(","):
        k, _, v = part.strip().partition("=")
        out[k.strip()] = float(v.strip())
    return out


def main() -> None:
    ap = argparse.ArgumentParser(description="Walk-forward backtest — RoaringKittyTracker")
    ap.add_argument("--scores",         default="data/outputs/scores.csv")
    ap.add_argument("--weekly_dir",     default="data/weekly")
    ap.add_argument("--quarterly_dir",  default="data/quarterly",
                    help="Directory of quarterly JSON files; if present enables PIT mode")
    ap.add_argument("--out_dir",        default="data/backtest")
    ap.add_argument("--train_start",    default="2015-01-01")
    ap.add_argument("--train_end",      default="2021-12-31")
    ap.add_argument("--eval_start",     default="2022-01-01")
    ap.add_argument("--eval_end",       default=None)
    ap.add_argument("--top_n",          type=int, default=DEFAULT_TOP_N)
    ap.add_argument("--weights",        type=str, default=None,
                    help="e.g. 'value=40,health=25,sentiment=20,quality=15'")
    ap.add_argument("--no_pit",         action="store_true",
                    help="Disable PIT fundamentals even if quarterly_dir exists")
    ap.add_argument("--tc_bps",         type=float, default=DEFAULT_TC_BPS,
                    help=f"One-way transaction cost in bps (default {DEFAULT_TC_BPS:.0f}; 0 = frictionless)")
    ap.add_argument("--strict_pit",     action="store_true",
                    help="Exclude tickers without PIT quarterly coverage instead of "
                         "falling back to today's fundamentals (recommended)")
    args = ap.parse_args()

    out_dir  = Path(args.out_dir)
    out_dir.mkdir(parents=True, exist_ok=True)
    weights  = _parse_weights(args.weights) if args.weights else DEFAULT_WEIGHTS.copy()
    eval_end = args.eval_end or pd.Timestamp.today().strftime("%Y-%m-%d")

    # ── Load scores ───────────────────────────────────────────────────────────
    scores_path = Path(args.scores)
    if not scores_path.exists():
        sys.exit(f"Scores file not found: {scores_path}")

    print(f"\nLoading scores  ({scores_path})...")
    raw = load_scores(scores_path)
    print(f"  {len(raw):,} tickers")

    # ── Load weekly prices ────────────────────────────────────────────────────
    weekly_dir = Path(args.weekly_dir)
    if not weekly_dir.exists():
        sys.exit(f"Weekly price dir not found: {weekly_dir}")

    print(f"Loading weekly prices ({weekly_dir}/)...")
    t0 = time.time()
    close_panel = load_prices_panel(weekly_dir, list(raw.index))
    spy_path = weekly_dir / f"{BENCHMARK}.csv"
    if BENCHMARK not in close_panel.columns and spy_path.exists():
        spy_df = pd.read_csv(spy_path, parse_dates=["Date"], index_col="Date")
        close_panel[BENCHMARK] = spy_df["Close"].dropna()
        close_panel = close_panel.sort_index()
    print(f"  {close_panel.shape[1]:,} tickers  {close_panel.shape[0]:,} weeks  "
          f"({time.time()-t0:.1f}s)")

    # ── Build signals panel (price signals, vectorized) ───────────────────────
    all_dates = _build_rebalance_dates(args.train_start, eval_end)
    print(f"\nPrecomputing signals for {len(all_dates)} rebalance dates × "
          f"{close_panel.shape[1]:,} tickers...")
    t0 = time.time()
    signals = build_signals_panel(close_panel, all_dates)
    print(f"  Done in {time.time()-t0:.1f}s  —  {len(signals):,} signal rows")

    # ── Build fundamentals panel (PIT mode if quarterly data exists) ──────────
    fund_panel: Optional[pd.DataFrame] = None
    quarterly_dir = Path(args.quarterly_dir)
    if not args.no_pit and quarterly_dir.exists():
        n_json = len(list(quarterly_dir.glob("*.json")))
        if n_json > 0:
            print(f"\nLoading quarterly cache ({n_json:,} tickers from {quarterly_dir}/)...")
            t0 = time.time()
            q_cache = load_quarterly_cache(quarterly_dir, list(raw.index))
            print(f"  {len(q_cache):,} tickers loaded ({time.time()-t0:.1f}s)")

            print(f"Building PIT fundamentals panel for {len(all_dates)} dates × "
                  f"{len(raw):,} tickers  (this takes 2–4 min)...")
            t0 = time.time()
            fund_panel = build_fundamentals_panel(
                tickers=list(raw.index),
                quarterly_cache=q_cache,
                close_panel=close_panel,
                rebalance_dates=all_dates,
                static_features=raw,
            )
            print(f"  Done in {time.time()-t0:.1f}s  —  "
                  f"{len(fund_panel):,} (date, ticker) fundamental rows  [PIT mode ON]")
        else:
            print(f"\nNo quarterly JSON files found in {quarterly_dir}/ — using static fundamentals.")
            print(f"  Run: python download_quarterly_financials.py")
    else:
        if args.no_pit:
            print("\nPIT mode disabled (--no_pit).")
        else:
            print(f"\nNo quarterly data dir ({quarterly_dir}/) — using static fundamentals.")
            print(f"  Run: python download_quarterly_financials.py")

    # ── Train backtest ────────────────────────────────────────────────────────
    train_res = run_backtest(
        raw_features=raw,
        prices_dict=close_panel,
        start_date=args.train_start,
        end_date=args.train_end,
        top_n=args.top_n,
        weights=weights,
        guardrails=DEFAULT_GUARDRAILS,
        label="IN-SAMPLE",
        signals_panel=signals,
        fundamentals_panel=fund_panel,
        tc_bps=args.tc_bps,
        strict_pit=args.strict_pit,
    )
    train_m = compute_metrics(train_res,
                              label=f"In-sample  ({args.train_start} – {args.train_end})")
    print_metrics(train_m)

    # ── Eval backtest ─────────────────────────────────────────────────────────
    eval_res = run_backtest(
        raw_features=raw,
        prices_dict=close_panel,
        start_date=args.eval_start,
        end_date=eval_end,
        top_n=args.top_n,
        weights=weights,
        guardrails=DEFAULT_GUARDRAILS,
        label="OUT-OF-SAMPLE",
        signals_panel=signals,
        fundamentals_panel=fund_panel,
        tc_bps=args.tc_bps,
        strict_pit=args.strict_pit,
    )
    eval_m = compute_metrics(eval_res,
                             label=f"Out-of-sample  ({args.eval_start} – {eval_end})")
    print_metrics(eval_m)

    # ── Save outputs ──────────────────────────────────────────────────────────
    all_res = pd.concat(
        [train_res.assign(split="train"), eval_res.assign(split="test")],
        ignore_index=True,
    )
    all_res.to_csv(out_dir / "results_quarterly.csv", index=False)

    holdings = []
    for _, row in all_res.iterrows():
        for t in str(row["portfolio_tickers"]).split(","):
            holdings.append({"split": row["split"],
                             "rebalance_date": row["rebalance_date"],
                             "ticker": t})
    pd.DataFrame(holdings).to_csv(out_dir / "results_portfolio.csv", index=False)

    pd.DataFrame([train_m, eval_m]).to_csv(out_dir / "metrics_summary.csv", index=False)

    print(f"\n  Results → {out_dir}/")
    print(f"    results_quarterly.csv  — per-quarter returns")
    print(f"    results_portfolio.csv  — tickers held each quarter")
    print(f"    metrics_summary.csv    — performance summary")
    print(f"\n  Next: python bt_optimizer.py\n")


if __name__ == "__main__":
    main()