from __future__ import annotations

import csv
import importlib.util
import json
import math
import statistics
from datetime import datetime, timezone
from pathlib import Path


ROOT = Path(__file__).resolve().parent.parent
WORK = ROOT / "work"
OUT = ROOT / "outputs" / "market_intelligence"
SOURCE_OUT = ROOT / "outputs" / "vptr3_rebuilt_market_patterns"
ENGINE_PATH = WORK / "rebuild_vptr3_market_patterns_from_candles.py"
REGISTRY = OUT / "asset_registry.json"
CSV_OUT = OUT / "trade_market_features_v2.csv"
SUMMARY = OUT / "market_features_v2_summary.json"

TF_MINUTES = {"1H": 60, "2H": 120, "3H": 180, "4H": 240}
NUMERIC_FEATURES = (
    "rsi14", "rsi14_dir_score", "adx14", "plus_di14", "minus_di14", "di_spread_dir",
    "ema20_slope_5_pct", "ema50_slope_5_pct", "close_vs_ema20_dir_pct", "close_vs_ema50_dir_pct",
    "trend_efficiency_10", "realized_vol_20_pct", "range_pct", "volume_zscore_20",
    "bb_width_percentile_100", "atr_percentile_100", "htf_return_3_dir_pct",
    "htf_close_vs_ema20_dir_pct", "htf_adx14", "btc_return_3_dir_pct", "btc_return_10_dir_pct",
    "btc_rsi14_dir_score", "btc_adx14", "mtf_alignment", "entry_hour_utc",
)
TEXT_FEATURES = ("weekday_utc", "market_regime", "volatility_regime")


def load_engine():
    spec = importlib.util.spec_from_file_location("market_feature_engine", ENGINE_PATH)
    if spec is None or spec.loader is None:
        raise RuntimeError("Fingerprint engine cannot be loaded")
    module = importlib.util.module_from_spec(spec)
    spec.loader.exec_module(module)
    return module


def finite(value) -> float | None:
    try:
        number = float(value)
    except (TypeError, ValueError):
        return None
    return number if math.isfinite(number) else None


def ema(values: list[float], period: int) -> list[float | None]:
    result: list[float | None] = [None] * len(values)
    alpha = 2 / (period + 1)
    current = None
    for index, value in enumerate(values):
        current = value if current is None else alpha * value + (1 - alpha) * current
        result[index] = current
    return result


def wilder(values: list[float | None], period: int) -> list[float | None]:
    result: list[float | None] = [None] * len(values)
    window: list[float] = []
    current = None
    for index, value in enumerate(values):
        if value is None:
            continue
        if current is None:
            window.append(value)
            if len(window) == period:
                current = sum(window) / period
                result[index] = current
        else:
            current = ((period - 1) * current + value) / period
            result[index] = current
    return result


def percentile_rank(values: list[float], current: float) -> float | None:
    if not values:
        return None
    return 100 * sum(value <= current for value in values) / len(values)


def add_advanced_features(group: list[dict]) -> None:
    group.sort(key=lambda row: row["time"])
    closes = [float(row["close"]) for row in group]
    highs = [float(row["high"]) for row in group]
    lows = [float(row["low"]) for row in group]
    volumes = [float(row.get("volume") or 0) for row in group]
    ema20, ema50 = ema(closes, 20), ema(closes, 50)

    gains: list[float | None] = [None]
    losses: list[float | None] = [None]
    tr: list[float | None] = [None]
    plus_dm: list[float | None] = [None]
    minus_dm: list[float | None] = [None]
    returns: list[float | None] = [None]
    for index in range(1, len(group)):
        change = closes[index] - closes[index - 1]
        gains.append(max(change, 0))
        losses.append(max(-change, 0))
        up_move = highs[index] - highs[index - 1]
        down_move = lows[index - 1] - lows[index]
        plus_dm.append(up_move if up_move > down_move and up_move > 0 else 0.0)
        minus_dm.append(down_move if down_move > up_move and down_move > 0 else 0.0)
        tr.append(max(highs[index] - lows[index], abs(highs[index] - closes[index - 1]), abs(lows[index] - closes[index - 1])))
        returns.append(change / closes[index - 1] * 100 if closes[index - 1] else None)

    avg_gain, avg_loss = wilder(gains, 14), wilder(losses, 14)
    atr, plus_smoothed, minus_smoothed = wilder(tr, 14), wilder(plus_dm, 14), wilder(minus_dm, 14)
    dx: list[float | None] = [None] * len(group)
    plus_di: list[float | None] = [None] * len(group)
    minus_di: list[float | None] = [None] * len(group)
    rsi: list[float | None] = [None] * len(group)
    for index in range(len(group)):
        if avg_gain[index] is not None and avg_loss[index] is not None:
            if avg_loss[index] == 0:
                rsi[index] = 100.0
            else:
                rs = avg_gain[index] / avg_loss[index]
                rsi[index] = 100 - 100 / (1 + rs)
        if atr[index] and plus_smoothed[index] is not None and minus_smoothed[index] is not None:
            plus_di[index] = 100 * plus_smoothed[index] / atr[index]
            minus_di[index] = 100 * minus_smoothed[index] / atr[index]
            total = plus_di[index] + minus_di[index]
            dx[index] = 100 * abs(plus_di[index] - minus_di[index]) / total if total else 0.0
    adx = wilder(dx, 14)

    atr_pct_history: list[float] = []
    bb_width_history: list[float] = []
    for index, row in enumerate(group):
        close = closes[index]
        row["rsi14"] = rsi[index]
        row["adx14"] = adx[index]
        row["plus_di14"] = plus_di[index]
        row["minus_di14"] = minus_di[index]
        row["ema20"] = ema20[index]
        row["ema50"] = ema50[index]
        row["range_pct"] = (highs[index] - lows[index]) / close * 100 if close else None
        if index >= 5 and ema20[index - 5] and ema50[index - 5]:
            row["ema20_slope_5_pct"] = (ema20[index] - ema20[index - 5]) / ema20[index - 5] * 100
            row["ema50_slope_5_pct"] = (ema50[index] - ema50[index - 5]) / ema50[index - 5] * 100
        if index >= 10:
            path = sum(abs(closes[pos] - closes[pos - 1]) for pos in range(index - 9, index + 1))
            row["trend_efficiency_10"] = abs(closes[index] - closes[index - 10]) / path if path else 0.0
        if index >= 20:
            valid_returns = [value for value in returns[index - 19:index + 1] if value is not None]
            row["realized_vol_20_pct"] = statistics.pstdev(valid_returns) if len(valid_returns) >= 2 else None
            volume_window = volumes[index - 19:index + 1]
            volume_mean = statistics.mean(volume_window)
            volume_sd = statistics.pstdev(volume_window)
            row["volume_zscore_20"] = (volumes[index] - volume_mean) / volume_sd if volume_sd else 0.0

        atr_pct = atr[index] / close * 100 if atr[index] and close else None
        if atr_pct is not None:
            atr_pct_history.append(atr_pct)
            row["atr_percentile_100"] = percentile_rank(atr_pct_history[-100:], atr_pct)
        if index >= 19:
            window = closes[index - 19:index + 1]
            basis = statistics.mean(window)
            width = 4 * statistics.pstdev(window) / basis * 100 if basis else None
            if width is not None:
                bb_width_history.append(width)
                row["bb_width_percentile_100"] = percentile_rank(bb_width_history[-100:], width)


def aggregate(group: list[dict], base_minutes: int, factor: int = 2) -> tuple[list[dict], int]:
    target_minutes = base_minutes * factor
    buckets: dict[int, list[dict]] = {}
    for row in group:
        stamp = int(row["time"].timestamp() // (target_minutes * 60) * target_minutes * 60)
        buckets.setdefault(stamp, []).append(row)
    result = []
    for stamp, bucket in sorted(buckets.items()):
        bucket.sort(key=lambda row: row["time"])
        if len(bucket) < factor:
            continue
        result.append({
            "symbol": bucket[0]["symbol"], "tf": f"{target_minutes}M", "time": datetime.fromtimestamp(stamp, timezone.utc),
            "open": bucket[0]["open"], "high": max(row["high"] for row in bucket),
            "low": min(row["low"] for row in bucket), "close": bucket[-1]["close"],
            "volume": sum(row.get("volume") or 0 for row in bucket),
        })
    add_advanced_features(result)
    return result, target_minutes


def last_closed(group: list[dict], entry_time: datetime, minutes: int) -> dict | None:
    eligible = [row for row in group if row["time"].timestamp() + minutes * 60 <= entry_time.timestamp()]
    return eligible[-1] if eligible else None


def load_trade_rows() -> list[dict]:
    result = []
    sources = (
        (SOURCE_OUT / "trade_fingerprints_rebuilt.csv", "1"),
        (SOURCE_OUT / "account2_vptr3_historical_fingerprints.csv", "2"),
    )
    for path, account in sources:
        with path.open(newline="", encoding="utf-8-sig") as fh:
            for row in csv.DictReader(fh):
                row["account_id"] = account
                result.append(row)
    return result


def main() -> None:
    if not REGISTRY.exists():
        raise RuntimeError(f"Missing asset registry: {REGISTRY}")
    engine = load_engine()
    candles = engine.resample_1h_to_3h(engine.load_all_candles())
    deduped = {(row["symbol"], row["tf"], row["time"]): row for row in candles}
    by_series: dict[tuple[str, str], list[dict]] = {}
    for row in deduped.values():
        by_series.setdefault((row["symbol"], row["tf"]), []).append(row)
    for group in by_series.values():
        add_advanced_features(group)

    higher: dict[tuple[str, str], tuple[list[dict], int]] = {}
    for key, group in by_series.items():
        minutes = TF_MINUTES.get(key[1])
        if minutes:
            higher[key] = aggregate(group, minutes)

    btc = by_series.get(("BTCUSDT", "4H"), [])
    output = []
    for trade in load_trade_rows():
        symbol = str(trade.get("symbol") or "").upper()
        tf = str(trade.get("tf") or engine.SYMBOL_TF.get(symbol) or "").upper()
        entry_time = engine.parse_dt(trade["entry_time"])
        side = str(trade.get("side") or "").upper()
        direction = 1 if side == "LONG" else -1
        group = by_series.get((symbol, tf), [])
        minutes = TF_MINUTES.get(tf)
        candle = last_closed(group, entry_time, minutes) if group and minutes else None
        htf_group, htf_minutes = higher.get((symbol, tf), ([], 0))
        htf = last_closed(htf_group, entry_time, htf_minutes) if htf_group else None
        btc_candle = last_closed(btc, entry_time, 240) if btc else None
        row = {
            "account_id": trade["account_id"], "request_id": trade.get("request_id"), "strategy": "VPTR3",
            "symbol": symbol, "tf": tf, "side": side, "entry_time": entry_time.isoformat(),
            "candle_time": candle["time"].isoformat() if candle else "", "has_real_candle": bool(candle),
            "lookahead_safe": bool(candle and candle["time"].timestamp() + minutes * 60 <= entry_time.timestamp()),
            "entry_hour_utc": entry_time.hour, "weekday_utc": entry_time.strftime("%A"),
        }
        if candle:
            close = candle["close"]
            row.update({
                "rsi14": candle.get("rsi14"),
                "rsi14_dir_score": direction * ((candle.get("rsi14") or 50) - 50),
                "adx14": candle.get("adx14"), "plus_di14": candle.get("plus_di14"), "minus_di14": candle.get("minus_di14"),
                "di_spread_dir": direction * ((candle.get("plus_di14") or 0) - (candle.get("minus_di14") or 0)),
                "ema20_slope_5_pct": candle.get("ema20_slope_5_pct"), "ema50_slope_5_pct": candle.get("ema50_slope_5_pct"),
                "close_vs_ema20_dir_pct": direction * (close - candle["ema20"]) / close * 100 if candle.get("ema20") else None,
                "close_vs_ema50_dir_pct": direction * (close - candle["ema50"]) / close * 100 if candle.get("ema50") else None,
                "trend_efficiency_10": candle.get("trend_efficiency_10"), "realized_vol_20_pct": candle.get("realized_vol_20_pct"),
                "range_pct": candle.get("range_pct"), "volume_zscore_20": candle.get("volume_zscore_20"),
                "bb_width_percentile_100": candle.get("bb_width_percentile_100"), "atr_percentile_100": candle.get("atr_percentile_100"),
            })
            adx = candle.get("adx14") or 0
            ema_slope = candle.get("ema50_slope_5_pct") or 0
            row["market_regime"] = "TREND_UP" if adx >= 20 and ema_slope > 0 else ("TREND_DOWN" if adx >= 20 and ema_slope < 0 else "RANGE")
            atr_rank = candle.get("atr_percentile_100")
            row["volatility_regime"] = "HIGH" if atr_rank is not None and atr_rank >= 70 else ("LOW" if atr_rank is not None and atr_rank <= 30 else "NORMAL")
        if htf:
            htf_close = htf["close"]
            htf_return = (htf_close - htf_group[max(0, htf_group.index(htf) - 3)]["close"]) / htf_group[max(0, htf_group.index(htf) - 3)]["close"] * 100
            row["htf_return_3_dir_pct"] = direction * htf_return
            row["htf_close_vs_ema20_dir_pct"] = direction * (htf_close - htf["ema20"]) / htf_close * 100 if htf.get("ema20") else None
            row["htf_adx14"] = htf.get("adx14")
            row["mtf_alignment"] = int((row.get("close_vs_ema20_dir_pct") or 0) > 0 and (row.get("htf_close_vs_ema20_dir_pct") or 0) > 0)
        if btc_candle:
            btc_index = btc.index(btc_candle)
            for lookback in (3, 10):
                previous = btc[max(0, btc_index - lookback)]["close"]
                row[f"btc_return_{lookback}_dir_pct"] = direction * (btc_candle["close"] - previous) / previous * 100 if previous else None
            row["btc_rsi14_dir_score"] = direction * ((btc_candle.get("rsi14") or 50) - 50)
            row["btc_adx14"] = btc_candle.get("adx14")
        output.append(row)

    columns = [
        "account_id", "request_id", "strategy", "symbol", "tf", "side", "entry_time", "candle_time",
        "has_real_candle", "lookahead_safe", *NUMERIC_FEATURES, *TEXT_FEATURES,
    ]
    OUT.mkdir(parents=True, exist_ok=True)
    temp = CSV_OUT.with_suffix(".csv.tmp")
    with temp.open("w", newline="", encoding="utf-8-sig") as fh:
        writer = csv.DictWriter(fh, fieldnames=columns, extrasaction="ignore")
        writer.writeheader()
        writer.writerows(output)
    temp.replace(CSV_OUT)
    completeness = {
        feature: round(100 * sum(finite(row.get(feature)) is not None for row in output) / len(output), 2)
        for feature in NUMERIC_FEATURES
    }
    summary = {
        "status": "OK", "records": len(output), "real_candles": sum(bool(row["has_real_candle"]) for row in output),
        "lookahead_safe": sum(bool(row["lookahead_safe"]) for row in output),
        "accounts": {account: sum(row["account_id"] == account for row in output) for account in ("1", "2")},
        "numeric_features": len(NUMERIC_FEATURES), "text_features": len(TEXT_FEATURES),
        "feature_completeness_pct": completeness, "automatic_blocking": False,
    }
    SUMMARY.write_text(json.dumps(summary, indent=2), encoding="utf-8")
    print(json.dumps(summary, indent=2))


if __name__ == "__main__":
    main()