bifitnex-trading/main.py

#!/usr/bin/env python3
"""LLM-driven cryptocurrency trading system — single-run mode for crontab."""

import json
import logging
import os
import sys
import time
from datetime import datetime, timezone, timedelta

import config
import data_fetcher
import indicators
import llm_analyzer
import portfolio as pf
import risk_manager
import slack_notifier
import trade_logger
import trader

# ---------------------------------------------------------------------------
# Logging setup
# ---------------------------------------------------------------------------
_TZ_UTC8 = timezone(timedelta(hours=8))

logging.basicConfig(
    level=logging.INFO,
    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
    handlers=[
        logging.StreamHandler(sys.stdout),
        logging.FileHandler("trading.log"),
    ],
)
logging.Formatter.converter = lambda *args: datetime.now(_TZ_UTC8).timetuple()
logger = logging.getLogger("main")

STATE_FILE = "bot_state.json"
STOP_ORDERS_FILE = "stop_orders.json"


def _calc_atr_stop_price(entry_price: float, atr: float) -> float:
    """根據 ATR 動態計算止損價，帶上下限。"""
    atr_pct = (atr * config.ATR_SL_MULTIPLIER) / entry_price
    bounded_pct = max(config.ATR_SL_MIN_PCT, min(config.ATR_SL_MAX_PCT, atr_pct))
    return round(entry_price * (1 - bounded_pct), 8)


def _load_state() -> dict:
    if os.path.exists(STATE_FILE):
        try:
            with open(STATE_FILE) as f:
                return json.load(f)
        except (json.JSONDecodeError, IOError):
            pass
    return {"last_status_report": 0, "run_count": 0}


def _save_state(state: dict):
    with open(STATE_FILE, "w") as f:
        json.dump(state, f, indent=2)


def _load_tracked_stops() -> dict:
    """Load tracked stop orders from file. Returns {symbol: {order_id, stop_price, entry_price, amount}}."""
    if os.path.exists(STOP_ORDERS_FILE):
        try:
            with open(STOP_ORDERS_FILE) as f:
                return json.load(f)
        except (json.JSONDecodeError, IOError):
            pass
    return {}


def _save_tracked_stops(stops: dict):
    with open(STOP_ORDERS_FILE, "w") as f:
        json.dump(stops, f, indent=2)


def run_cycle():
    """Execute one full trading cycle."""
    state = _load_state()
    state["run_count"] = state.get("run_count", 0) + 1
    logger.info("=" * 60)
    logger.info("Trading cycle #%d at %s", state["run_count"], datetime.now(_TZ_UTC8).isoformat())

    # 1. Load portfolio state
    port = pf.load_positions()

    # 2. Fetch account status from exchange
    account_status = {"wallets": [], "positions": []}
    try:
        account_status = data_fetcher.fetch_account_status()
        port = pf.sync_with_exchange(port, account_status)
        logger.info(
            "Account: %.2f USDT total, %.2f USDT available",
            port.get("total_balance_usdt", 0),
            port.get("available_usdt", 0),
        )
    except Exception as e:
        logger.error("Failed to fetch account status: %s", e)
        if config.PAPER_TRADING and port.get("total_balance_usdt", 0) == 0:
            port["total_balance_usdt"] = 10000
            port["available_usdt"] = 10000
            logger.info("Paper trading: using default 10000 USDT balance")

    # 2a. Fetch initial capital from deposit history
    try:
        port["initial_capital"] = data_fetcher.fetch_total_deposits()
    except Exception as e:
        logger.warning("Failed to fetch deposit history: %s", e)

    # 2b. Sync stop-loss orders with exchange (real-time, not local memory)
    try:
        active_orders = data_fetcher.fetch_active_orders()
    except Exception as e:
        logger.error("Failed to fetch active orders: %s", e)
        active_orders = []
    # Build map: symbol → list of EXCHANGE STOP orders
    stop_orders_by_sym = {}
    for o in active_orders:
        if o.get("type") == "EXCHANGE STOP":
            stop_orders_by_sym.setdefault(o["symbol"], []).append(o)

    # Build map: currency → wallet available balance (for accurate stop-loss amounts)
    # Use "available" (not "balance") to exclude coins locked by existing orders
    wallet_balances = {}
    for w in account_status.get("wallets", []):
        if w.get("type") == "exchange":
            wallet_balances[w["currency"]] = w.get("available", 0) or w.get("balance", 0)

    # Load tracked stops early — needed for both stop sync and fill detection
    tracked_stops = _load_tracked_stops()

    for sym, pos in port.get("positions", {}).items():
        if pos.get("amount", 0) <= 0:
            continue
        entry = pos.get("entry_price", 0)
        if entry <= 0:
            continue

        # Use exchange wallet balance as the authoritative amount
        currency = sym[1:].replace(":UST", "").replace("UST", "")
        amount = wallet_balances.get(currency, 0)
        if amount <= 0:
            continue

        # Skip stop-loss for positions below exchange minimum order size
        min_amt = config.MIN_ORDER_AMOUNT.get(sym, 0)
        if min_amt > 0 and amount < min_amt:
            continue

        # Use stored stop price if available (preserves ATR-calculated value);
        # fall back to fixed % for legacy positions without a stored price
        tracked = tracked_stops.get(sym, {})
        expected_stop = tracked.get("stop_price") or round(entry * (1 - config.STOP_LOSS_PCT), 8)
        existing_stops = stop_orders_by_sym.get(sym, [])

        if existing_stops:
            # Check if existing stop matches expected amount & price
            stop = existing_stops[0]
            stop_amt_ok = abs(abs(stop["amount"]) - amount) / amount < 0.01  # 1% tolerance
            stop_px_ok = abs(stop["price"] - expected_stop) / expected_stop < 0.01
            if stop_amt_ok and stop_px_ok:
                # Ensure this stop is tracked for fill detection
                tracked_stops[sym] = {
                    "order_id": stop["id"], "stop_price": stop["price"],
                    "entry_price": entry, "amount": amount,
                }
                continue  # Stop order is correct
            # Wrong amount or price — cancel all existing stops and re-place
            for s in existing_stops:
                trader.cancel_order(s["id"])
                logger.info("Cancelled outdated stop %s for %s (amt=%.6f, px=%.6g)",
                            s["id"], sym, abs(s["amount"]), s["price"])
            tracked_stops.pop(sym, None)
            time.sleep(1.0)  # Wait for Bitfinex to release locked balance

        logger.warning("Position %s missing/outdated stop-loss, placing now", sym)
        sl = trader.place_stop_loss_order(sym, amount, entry, stop_price=expected_stop)
        if sl is None and existing_stops:
            # Retry once — Bitfinex may need more time to release locked balance
            time.sleep(2.0)
            logger.info("Retrying stop-loss for %s after balance release delay", sym)
            sl = trader.place_stop_loss_order(sym, amount, entry, stop_price=expected_stop)
        if sl:
            # Update stop_orders_by_sym so step 10 sees the correct stop ID
            stop_orders_by_sym[sym] = [{"id": sl["order_id"], "symbol": sym,
                                        "amount": -amount, "price": sl["stop_price"],
                                        "type": "EXCHANGE STOP"}]
            # Track for fill detection
            tracked_stops[sym] = {
                "order_id": sl["order_id"], "stop_price": sl["stop_price"],
                "entry_price": entry, "amount": amount,
            }
            pf.save_positions(port)
            logger.info("Stop-loss for %s: %.6f @ %.6g", sym, amount, sl["stop_price"])
    _save_tracked_stops(tracked_stops)

    # 3. Fetch market data
    try:
        market_data = data_fetcher.fetch_all_market_data()
        logger.info("Fetched market data for %d symbols", len(market_data))
    except Exception as e:
        logger.error("Failed to fetch market data: %s", e)
        slack_notifier.send_error_alert(f"Market data fetch failed: {e}")
        return

    # 4. Calculate indicators (5m)
    indicators_by_symbol = {}
    current_prices = {}
    for sym, md in market_data.items():
        candles_df = md.get("candles")
        if candles_df is not None and not candles_df.empty:
            ind_df = indicators.calculate_indicators(candles_df)
            indicators_by_symbol[sym] = ind_df
        ticker = md.get("ticker", {})
        if ticker:
            current_prices[sym] = ticker.get("last_price", 0)

    # 4b. Calculate HTF indicators (1h)
    htf_by_symbol = {}
    pivots_by_symbol = {}
    for sym, md in market_data.items():
        candles_htf = md.get("candles_htf")
        if candles_htf is not None and not candles_htf.empty:
            htf_by_symbol[sym] = indicators.calculate_htf_indicators(candles_htf)
            pivots_by_symbol[sym] = indicators.calculate_pivot_points(candles_htf)

    # 5. Cache data (persisted for next crontab run)
    try:
        data_fetcher.cache_market_data(market_data, indicators_by_symbol)
    except Exception as e:
        logger.warning("Cache write failed: %s", e)

    # --- Detect filled stop-loss orders ---
    sl_filled = []
    active_order_ids = {str(o["id"]) for o in active_orders}
    for sym, sinfo in list(tracked_stops.items()):
        if str(sinfo.get("order_id")) in active_order_ids:
            continue  # still active, not filled
        # Stop order is gone — check if wallet balance is also gone (= filled, not cancelled by us)
        currency = sym[1:].replace(":UST", "").replace("UST", "")
        wallet_amt = wallet_balances.get(currency, 0)
        pos = port.get("positions", {}).get(sym, {})
        pos_amt = pos.get("amount", 0)
        # If wallet is empty or nearly empty but we had a position → stop was filled
        min_amt = config.MIN_ORDER_AMOUNT.get(sym, 0)
        if pos_amt > 0 and wallet_amt < max(min_amt, pos_amt * 0.05):
            stop_price = sinfo.get("stop_price", 0)
            entry_price = sinfo.get("entry_price", 0) or pos.get("entry_price", 0)
            amount = sinfo.get("amount", 0) or pos_amt
            amount_usdt = amount * stop_price if stop_price else 0
            realized_pnl = None
            realized_pnl_pct = None
            if entry_price and stop_price:
                realized_pnl = (stop_price - entry_price) * amount
                realized_pnl_pct = (stop_price - entry_price) / entry_price * 100

            sl_filled.append({
                "symbol": sym,
                "action": "SELL",
                "amount": amount,
                "price": stop_price,
                "amount_usdt": amount_usdt,
                "reason": "止損觸發",
                "confidence": 1.0,
                "mode": "live",
                "status": "filled",
                "is_stop_loss": True,
                "entry_price": entry_price,
                "realized_pnl": realized_pnl,
                "realized_pnl_pct": realized_pnl_pct,
                "stop_price": None,
            })

            # Update portfolio — position is closed
            port = pf.update_position(port, sym, "SELL", amount, stop_price, amount_usdt)
            pf.save_positions(port)
            trade_logger.log_trade(sl_filled[-1])
            logger.warning("Stop-loss FILLED for %s: %.6f @ %.6g, P&L: %.2f USDT (%.2f%%)",
                           sym, amount, stop_price, realized_pnl or 0, realized_pnl_pct or 0)

        # Remove from tracked stops (filled or cancelled)
        del tracked_stops[sym]
    _save_tracked_stops(tracked_stops)

    # --- Collect cycle results ---
    tp_closed = []   # take-profit closures
    executed = sl_filled  # start with filled stop-losses
    rejected = []    # signals rejected by risk manager

    # 6. Check take-profit on existing positions (stop-loss is handled by exchange stop orders)
    tp_signals = risk_manager.apply_stop_loss_take_profit(
        port.get("positions", {}), current_prices
    )
    for signal in tp_signals:
        sym = signal["symbol"]
        pos = port.get("positions", {}).get(sym, {})
        amount = pos.get("amount", 0)
        price = current_prices.get(sym, 0)
        if amount > 0 and price > 0:
            # Cancel exchange stop-loss orders before selling (from real-time data)
            for s in stop_orders_by_sym.get(sym, []):
                trader.cancel_order(s["id"])
                logger.info("Cancelled stop %s for %s before TP sell", s["id"], sym)
            tracked_stops.pop(sym, None)
            _save_tracked_stops(tracked_stops)

            # Use wallet balance as authoritative sell amount
            currency = sym[1:].replace(":UST", "").replace("UST", "")
            wallet_amt = wallet_balances.get(currency, 0)
            if wallet_amt > 0:
                amount = wallet_amt
            signal["sell_amount"] = amount
            signal["amount_usdt"] = amount * price
            # Capture cost basis before selling
            entry_price = pos.get("entry_price", 0)
            cost_basis = entry_price * amount if entry_price else 0

            result = trader.execute_trade(signal, current_prices)
            if result and result.get("status") in ("filled", "submitted"):
                sell_proceeds = amount * price
                realized_pnl = sell_proceeds - cost_basis if cost_basis else None
                realized_pnl_pct = ((price - entry_price) / entry_price * 100) if entry_price else None

                port = pf.update_position(port, sym, "SELL", amount, price, signal["amount_usdt"])
                pf.save_positions(port)
                trade_logger.log_trade(result)
                tp_closed.append({
                    **signal,
                    "amount_usdt": sell_proceeds,
                    "entry_price": entry_price,
                    "realized_pnl": realized_pnl,
                    "realized_pnl_pct": realized_pnl_pct,
                })
                logger.info("Take profit executed: %s, P&L: %.2f USDT (%.2f%%)",
                            sym, realized_pnl or 0, realized_pnl_pct or 0)

    # 7. Build indicator summary for LLM (5m + 1h context + pivots)
    indicator_summary = indicators.summarize_all(indicators_by_symbol)
    if htf_by_symbol:
        indicator_summary += "\n" + indicators.summarize_htf(htf_by_symbol)
    if pivots_by_symbol:
        indicator_summary += "\n" + indicators.summarize_pivots(pivots_by_symbol, current_prices)

    # 8. Build account status string for LLM
    account_str = _build_account_string(port, current_prices)

    # 9. Call LLM for analysis
    signals = []
    llm_ok = False
    try:
        signals = llm_analyzer.analyze_market(indicator_summary, account_str)
        llm_ok = True
        logger.info("LLM returned %d signals", len(signals))
        non_hold = [s for s in signals if s.get("action") != "HOLD"]
        if non_hold:
            for s in non_hold:
                logger.info("LLM signal: %s %s conf=%.2f reason=%s",
                            s.get("action"), s.get("symbol"), s.get("confidence", 0), s.get("reason", ""))
        else:
            # Debug: log ALL hold reasons (not just first 3) to diagnose all-HOLD cycles
            for s in signals:
                logger.info("LLM HOLD: %s conf=%.2f reason=%s",
                            s.get("symbol"), s.get("confidence", 0), s.get("reason", ""))
    except Exception as e:
        logger.error("LLM analysis failed: %s", e)
        slack_notifier.send_error_alert(f"LLM analysis failed: {e}")

    # 10. Validate and execute trades
    for signal in signals:
        action = signal.get("action", "HOLD")
        if action == "HOLD":
            continue

        ind_df = indicators_by_symbol.get(signal.get("symbol"))
        validated, reject_reason = risk_manager.validate_trade(signal, port, ind_df, current_prices)
        if validated is None:
            logger.info("Signal rejected by risk manager: %s %s — %s", action, signal.get("symbol"), reject_reason)
            rejected.append({**signal, "reject_reason": reject_reason})
            continue

        sym = validated["symbol"]

        if action == "SELL":
            # Cancel exchange stop-loss orders BEFORE selling (free up locked balance)
            for s in stop_orders_by_sym.get(sym, []):
                trader.cancel_order(s["id"])
                logger.info("Cancelled stop %s for %s before sell", s["id"], sym)
            tracked_stops.pop(sym, None)
            _save_tracked_stops(tracked_stops)
            # Use wallet balance as authoritative sell amount
            currency = sym[1:].replace(":UST", "").replace("UST", "")
            wallet_amt = wallet_balances.get(currency, 0)
            if wallet_amt > 0:
                validated["sell_amount"] = wallet_amt

        result = trader.execute_trade(validated, current_prices)
        if result and result.get("status") == "failed":
            logger.warning("Order failed at exchange: %s %s — %s", action, validated["symbol"], result.get("error", ""))
            rejected.append({**validated, "reject_reason": f"交易所錯誤: {result.get('error', 'unknown')}"})
            # BUY failed at exchange — mark balance as unreliable to skip remaining BUYs
            if action == "BUY" and "not enough" in result.get("error", "").lower():
                port["available_usdt"] = 0
                logger.info("Marked available_usdt=0 to prevent further BUY attempts this cycle")
            continue

        if result and result.get("status") in ("filled", "submitted"):
            amount = result.get("amount", 0)
            price = result.get("price", 0)
            amount_usdt = result.get("amount_usdt", 0)

            # Calculate realized P&L for SELL trades
            realized_pnl = None
            realized_pnl_pct = None
            if action == "SELL":
                pos = port.get("positions", {}).get(sym, {})
                entry_price = pos.get("entry_price", 0)
                if entry_price:
                    cost_basis = entry_price * amount
                    realized_pnl = amount * price - cost_basis
                    realized_pnl_pct = (price - entry_price) / entry_price * 100

            port = pf.update_position(port, sym, action, amount, price, amount_usdt)

            stop_price = None
            if action == "BUY":
                # Cancel existing exchange stop orders before placing new one
                pos = port.get("positions", {}).get(sym, {})
                cancelled_any = False
                for s in stop_orders_by_sym.get(sym, []):
                    trader.cancel_order(s["id"])
                    cancelled_any = True
                    logger.info("Cancelled old stop %s for %s (position size changed)", s["id"], sym)
                if cancelled_any:
                    time.sleep(1.0)  # Wait for Bitfinex to release locked balance

                # Place new stop-loss for TOTAL position amount at new avg entry
                # Refresh wallet balance to get actual available amount (handles partial fills, fees)
                try:
                    refreshed_status = data_fetcher.fetch_account_status()
                    currency = sym[1:].replace(":UST", "").replace("UST", "")
                    wallet_amt = 0
                    for w in refreshed_status.get("wallets", []):
                        if w.get("type") == "exchange" and w.get("currency") == currency:
                            wallet_amt = w.get("available", 0) or w.get("balance", 0)
                            break
                    total_amount = wallet_amt if wallet_amt > 0 else pos.get("amount", amount)
                except Exception:
                    total_amount = pos.get("amount", amount)
                entry_price = pos.get("entry_price", price)

                # ATR-based dynamic stop price
                atr_stop = None
                ind_df = indicators_by_symbol.get(sym)
                if ind_df is not None and "atr" in ind_df.columns:
                    atr_val = ind_df["atr"].dropna().iloc[-1] if not ind_df["atr"].dropna().empty else None
                    if atr_val and atr_val > 0:
                        atr_stop = _calc_atr_stop_price(entry_price, atr_val)
                        atr_pct = (1 - atr_stop / entry_price) * 100
                        logger.info("ATR stop for %s: ATR=%.6g, stop=%.6g (%.1f%%)",
                                    sym, atr_val, atr_stop, atr_pct)
                if atr_stop is None:
                    logger.info("ATR unavailable for %s, using fixed %.0f%% stop", sym, config.STOP_LOSS_PCT * 100)

                sl = trader.place_stop_loss_order(sym, total_amount, entry_price, stop_price=atr_stop)
                if sl is None and cancelled_any:
                    # Retry once — Bitfinex may need more time to release locked balance
                    time.sleep(2.0)
                    logger.info("Retrying stop-loss for %s after balance release delay", sym)
                    sl = trader.place_stop_loss_order(sym, total_amount, entry_price, stop_price=atr_stop)
                if sl:
                    stop_price = sl["stop_price"]
                    # Update stop_orders_by_sym so subsequent actions in this cycle see it
                    stop_orders_by_sym[sym] = [{"id": sl["order_id"], "symbol": sym,
                                                "amount": -total_amount, "price": stop_price,
                                                "type": "EXCHANGE STOP"}]
                    # Track for fill detection
                    tracked_stops[sym] = {
                        "order_id": sl["order_id"], "stop_price": stop_price,
                        "entry_price": entry_price, "amount": total_amount,
                    }
                    _save_tracked_stops(tracked_stops)
                    logger.info("Stop-loss set for %s: %.6f @ %.6g", sym, total_amount, stop_price)

            pf.save_positions(port)
            trade_logger.log_trade(result)
            executed.append({**result, "stop_price": stop_price,
                             "realized_pnl": realized_pnl, "realized_pnl_pct": realized_pnl_pct})

    # 10b. Post-trade wallet refresh — ensure next trade uses latest balances
    if executed or tp_closed:
        try:
            refreshed = data_fetcher.fetch_account_status()
            port = pf.sync_with_exchange(port, refreshed)
            # Rebuild wallet_balances for any remaining logic
            wallet_balances = {}
            for w in refreshed.get("wallets", []):
                if w.get("type") == "exchange":
                    wallet_balances[w["currency"]] = w.get("balance", 0)
            logger.info("Post-trade wallet refresh: %.2f USDT available", port.get("available_usdt", 0))
        except Exception as e:
            logger.warning("Post-trade wallet refresh failed: %s", e)

    # 11. Send unified cycle report to Slack
    portfolio_summary = _build_portfolio_one_liner(port, current_prices)
    slack_notifier.send_cycle_report(
        cycle_number=state["run_count"],
        signals=signals,
        executed=executed,
        rejected=rejected,
        tp_closed=tp_closed,
        portfolio_summary=portfolio_summary,
        llm_ok=llm_ok,
    )

    # 12. Save final state
    pf.save_positions(port)

    # 13. Hourly status report (check elapsed time across crontab runs)
    now = time.time()
    last_report = state.get("last_status_report", 0)
    if now - last_report >= config.STATUS_REPORT_INTERVAL_MINUTES * 60:
        summary = pf.get_portfolio_summary(port, current_prices)
        slack_notifier.send_status_update(summary)
        logger.info("Hourly status report sent")
        state["last_status_report"] = now

    state["last_run"] = now
    _save_state(state)
    logger.info("Cycle complete")


def _build_portfolio_one_liner(port: dict, current_prices: dict) -> str:
    """Build a one-line portfolio summary for the cycle report."""
    available = port.get("available_usdt", 0)
    positions = port.get("positions", {})
    pos_count = 0
    total_cost = 0.0
    total_market_value = 0.0
    for sym, pos in positions.items():
        amount = pos.get("amount", 0)
        entry = pos.get("entry_price", 0)
        current = current_prices.get(sym, 0)
        min_amt = config.MIN_ORDER_AMOUNT.get(sym, 0)
        if amount > 0 and entry > 0 and amount >= min_amt:
            pos_count += 1
            total_cost += amount * entry
            if current > 0:
                total_market_value += amount * current
    total_value = available + total_market_value
    # 總收益率（基於入金）
    initial_capital = port.get("initial_capital", 0)
    total_return_pct = ((total_value / initial_capital - 1) * 100) if initial_capital > 0 else 0.0
    # 變動收益率（基於持倉成本）
    unrealized = total_market_value - total_cost
    total_capital = available + total_cost
    change_pct = ((total_value / total_capital - 1) * 100) if total_capital > 0 else 0.0
    return (
        f"總值 {total_value:.2f} USDT | 總收益 {total_return_pct:+.2f}% | "
        f"變動 {change_pct:+.2f}% | {available:.2f} 可用 | 持倉 {pos_count} 筆"
    )


def _build_account_string(port: dict, current_prices: dict) -> str:
    """Build a concise account status string for the LLM prompt."""
    lines = []
    lines.append(f"Total Balance: {port.get('total_balance_usdt', 0):.2f} USDT")
    lines.append(f"Available: {port.get('available_usdt', 0):.2f} USDT")

    positions = port.get("positions", {})
    if positions:
        lines.append("\nOpen Positions:")
        for sym, pos in positions.items():
            amount = pos.get("amount", 0)
            entry = pos.get("entry_price", 0)
            current = current_prices.get(sym, 0)
            pnl_pct = ((current - entry) / entry * 100) if entry > 0 and current > 0 else 0
            name = config.SYMBOL_NAMES.get(sym, sym)
            lines.append(
                f"  {name}: {amount:.6f} @ {entry:.6g} "
                f"(now {current:.6g}, {pnl_pct:+.2f}%)"
            )
    else:
        lines.append("\nNo open positions.")

    return "\n".join(lines)


if __name__ == "__main__":
    mode = "PAPER" if config.PAPER_TRADING else "LIVE"
    logger.info("Running in %s mode, monitoring %d symbols", mode, len(config.TOP_15_SYMBOLS))
    run_cycle()