Add hourly Slack trend report, log all HOLD reasons, whale correlation analysis

- hourly_trend_report.py: standalone cron script (XX:00:30) sends 1h bullish/bearish status - slack_notifier.py: add send_market_trend_report() — simple bullish/bearish only, no entry signals - main.py: log all 15 HOLD reasons (not just first 3) for debugging all-HOLD cycles - backtest/whale_correlation.py: blockchain.com on-chain correlation analysis (result: no signal) - memory/: update project memory with architecture split, cron layout, feedback Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
2026-03-18 13:45:52 +00:00 · 2026-03-18 13:45:52 +00:00 · d261b36460
commit d261b36460
parent 32aa6e40cd
10 changed files with 397 additions and 14 deletions
--- a/backtest/whale_correlation.py
+++ b/backtest/whale_correlation.py
@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""Fetch blockchain.com on-chain metrics and analyze correlation with BTC price changes.
+
+Whale proxy metrics (all free, daily granularity):
+- estimated-transaction-volume (BTC): total estimated tx volume
+- n-transactions: daily confirmed transaction count
+- Derived: avg_tx_size = volume / n_transactions (whale activity proxy)
+- output-volume (BTC): total output value
+
+Correlation targets:
+- BTC next-day return
+- BTC next-3-day return
+"""
+
+import os
+import sys
+import time
+
+import pandas as pd
+import numpy as np
+import requests
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), ".."))
+
+CACHE_DIR = os.path.join(os.path.dirname(__file__), "..", "cache", "backtest")
+BTC_1H_CACHE = os.path.join(CACHE_DIR, "tBTCUST_1h.csv")
+
+BLOCKCHAIN_API = "https://api.blockchain.info/charts"
+METRICS = [
+    "estimated-transaction-volume",      # BTC total est. tx volume
+    "estimated-transaction-volume-usd",  # USD total est. tx volume
+    "n-transactions",                    # daily confirmed tx count
+    "output-volume",                     # total output value (BTC)
+    "n-unique-addresses",                # unique addresses per day
+]
+
+
+def fetch_blockchain_metric(name: str, start: str, end: str) -> pd.DataFrame:
+    """Fetch a single blockchain.com chart metric."""
+    start_ts = int(pd.Timestamp(start).timestamp())
+    end_ts = int(pd.Timestamp(end).timestamp())
+    # timespan is calculated from end; we use start param to set beginning
+    url = f"{BLOCKCHAIN_API}/{name}"
+    params = {
+        "format": "json",
+        "start": start_ts,
+        "timespan": "1year",  # large enough window
+    }
+    resp = requests.get(url, params=params, timeout=30)
+    resp.raise_for_status()
+    data = resp.json()
+
+    values = data.get("values", [])
+    if not values:
+        return pd.DataFrame()
+
+    df = pd.DataFrame(values)
+    df.columns = ["timestamp", name]
+    df["date"] = pd.to_datetime(df["timestamp"], unit="s").dt.date
+    df = df[["date", name]]
+
+    # Filter to requested range
+    start_date = pd.Timestamp(start).date()
+    end_date = pd.Timestamp(end).date()
+    df = df[(df["date"] >= start_date) & (df["date"] <= end_date)]
+
+    return df
+
+
+def load_btc_daily_prices() -> pd.DataFrame:
+    """Load BTC 1h cache and resample to daily OHLC."""
+    if not os.path.exists(BTC_1H_CACHE):
+        print(f"ERROR: BTC 1h cache not found at {BTC_1H_CACHE}")
+        print("Run backtest first to populate the cache.")
+        sys.exit(1)
+
+    df = pd.read_csv(BTC_1H_CACHE, parse_dates=["timestamp"])
+    df["date"] = df["timestamp"].dt.date
+    daily = df.groupby("date").agg(
+        open=("open", "first"),
+        high=("high", "max"),
+        low=("low", "min"),
+        close=("close", "last"),
+        volume=("volume", "sum"),
+    ).reset_index()
+    return daily
+
+
+def main():
+    start = "2025-07-01"
+    end = "2026-03-17"
+
+    print("=== Whale Activity ↔ BTC Price Correlation Analysis ===\n")
+
+    # Step 1: Fetch on-chain metrics
+    print("Fetching blockchain.com metrics...")
+    metrics_dfs = []
+    for metric in METRICS:
+        print(f"  {metric}...", end=" ", flush=True)
+        try:
+            df = fetch_blockchain_metric(metric, start, end)
+            print(f"{len(df)} days")
+            metrics_dfs.append(df)
+        except Exception as e:
+            print(f"FAILED: {e}")
+        time.sleep(2)  # rate limit: 1 req / 10 sec (be conservative)
+
+    if not metrics_dfs:
+        print("ERROR: No metrics fetched")
+        return
+
+    # Merge all metrics on date
+    onchain = metrics_dfs[0]
+    for df in metrics_dfs[1:]:
+        onchain = onchain.merge(df, on="date", how="outer")
+    onchain = onchain.sort_values("date").reset_index(drop=True)
+
+    # Derived metrics
+    if "estimated-transaction-volume" in onchain.columns and "n-transactions" in onchain.columns:
+        onchain["avg_tx_size_btc"] = onchain["estimated-transaction-volume"] / onchain["n-transactions"]
+    if "estimated-transaction-volume-usd" in onchain.columns and "n-transactions" in onchain.columns:
+        onchain["avg_tx_size_usd"] = onchain["estimated-transaction-volume-usd"] / onchain["n-transactions"]
+
+    print(f"\nOn-chain data: {len(onchain)} days")
+
+    # Step 2: Load BTC prices
+    print("Loading BTC daily prices from cache...")
+    btc = load_btc_daily_prices()
+    print(f"BTC daily data: {len(btc)} days")
+
+    # Step 3: Merge and compute returns
+    merged = onchain.merge(btc[["date", "close", "volume"]], on="date", how="inner")
+    merged = merged.rename(columns={"close": "btc_close", "volume": "btc_volume"})
+    merged = merged.sort_values("date").reset_index(drop=True)
+
+    # Price returns (forward-looking)
+    merged["ret_1d"] = merged["btc_close"].pct_change().shift(-1)   # next-day return
+    merged["ret_3d"] = merged["btc_close"].pct_change(3).shift(-3)  # next-3-day return
+    merged["ret_5d"] = merged["btc_close"].pct_change(5).shift(-5)  # next-5-day return
+
+    # Z-score normalization for on-chain metrics (rolling 30-day)
+    onchain_cols = [c for c in merged.columns if c not in
+                    ["date", "btc_close", "btc_volume", "ret_1d", "ret_3d", "ret_5d"]]
+
+    for col in onchain_cols:
+        roll_mean = merged[col].rolling(30, min_periods=10).mean()
+        roll_std = merged[col].rolling(30, min_periods=10).std()
+        merged[f"{col}_zscore"] = (merged[col] - roll_mean) / roll_std.replace(0, np.nan)
+
+    # Step 4: Correlation analysis
+    print(f"\nMerged dataset: {len(merged)} days")
+    print(f"Date range: {merged['date'].iloc[0]} to {merged['date'].iloc[-1]}")
+
+    # Raw correlations
+    zscore_cols = [c for c in merged.columns if c.endswith("_zscore")]
+    target_cols = ["ret_1d", "ret_3d", "ret_5d"]
+
+    print("\n" + "=" * 70)
+    print("  PEARSON CORRELATION: On-Chain Metrics ↔ BTC Forward Returns")
+    print("=" * 70)
+
+    valid = merged.dropna(subset=target_cols + zscore_cols)
+    print(f"  (Using {len(valid)} complete observations)\n")
+
+    results = []
+    for oc_col in zscore_cols:
+        for target in target_cols:
+            corr = valid[oc_col].corr(valid[target])
+            results.append({"metric": oc_col, "target": target, "corr": corr})
+
+    results_df = pd.DataFrame(results)
+
+    # Print as pivot table
+    pivot = results_df.pivot(index="metric", columns="target", values="corr")
+    pivot = pivot[target_cols]  # order columns
+
+    # Sort by absolute correlation with ret_1d
+    pivot["abs_ret_1d"] = pivot["ret_1d"].abs()
+    pivot = pivot.sort_values("abs_ret_1d", ascending=False)
+    pivot = pivot.drop(columns="abs_ret_1d")
+
+    for metric in pivot.index:
+        name = metric.replace("_zscore", "")
+        vals = "  ".join(f"{pivot.loc[metric, t]:+.4f}" for t in target_cols)
+        print(f"  {name:<35s} {vals}")
+
+    print(f"\n  {'':35s} {'ret_1d':>8s}  {'ret_3d':>8s}  {'ret_5d':>8s}")
+
+    # Step 5: Highlight significant correlations
+    print("\n" + "=" * 70)
+    print("  NOTABLE CORRELATIONS (|r| > 0.10)")
+    print("=" * 70)
+
+    notable = results_df[results_df["corr"].abs() > 0.10].sort_values("corr", key=abs, ascending=False)
+    if notable.empty:
+        print("  None found — on-chain metrics show weak correlation with BTC returns.")
+    else:
+        for _, row in notable.iterrows():
+            direction = "↑↑" if row["corr"] > 0 else "↓↑" if row["corr"] < 0 else "  "
+            name = row["metric"].replace("_zscore", "")
+            print(f"  {direction} {name:<35s} → {row['target']}: r={row['corr']:+.4f}")
+
+    # Step 6: Extreme value analysis (whale spikes)
+    print("\n" + "=" * 70)
+    print("  EXTREME VALUE ANALYSIS (Top/Bottom 10% Days)")
+    print("=" * 70)
+
+    for col_name in ["avg_tx_size_btc", "estimated-transaction-volume", "avg_tx_size_usd"]:
+        zscore_col = f"{col_name}_zscore"
+        if zscore_col not in merged.columns:
+            continue
+
+        valid_ext = merged.dropna(subset=[zscore_col, "ret_1d"])
+        if len(valid_ext) < 20:
+            continue
+
+        q10 = valid_ext[zscore_col].quantile(0.10)
+        q90 = valid_ext[zscore_col].quantile(0.90)
+
+        low_days = valid_ext[valid_ext[zscore_col] <= q10]
+        high_days = valid_ext[valid_ext[zscore_col] >= q90]
+        all_avg = valid_ext["ret_1d"].mean()
+
+        print(f"\n  {col_name}:")
+        print(f"    Low activity days (bottom 10%):  avg next-day ret = {low_days['ret_1d'].mean():+.4f}  (n={len(low_days)})")
+        print(f"    High activity days (top 10%):    avg next-day ret = {high_days['ret_1d'].mean():+.4f}  (n={len(high_days)})")
+        print(f"    All days average:                avg next-day ret = {all_avg:+.4f}  (n={len(valid_ext)})")
+
+    # Save merged data for further analysis
+    out_path = os.path.join(CACHE_DIR, "whale_correlation_data.csv")
+    merged.to_csv(out_path, index=False)
+    print(f"\nSaved merged dataset to {out_path}")
+    print("Done.")
+
+
+if __name__ == "__main__":
+    main()
--- a/hourly_trend_report.py
+++ b/hourly_trend_report.py
@ -0,0 +1,31 @@
+#!/usr/bin/env python3
+"""Send hourly 1h market trend report to Slack."""
+
+import sys
+import os
+
+sys.path.insert(0, os.path.dirname(__file__))
+
+import data_fetcher
+import indicators
+import slack_notifier
+
+
+def main():
+    market_data = data_fetcher.fetch_all_market_data()
+
+    htf_by_symbol = {}
+    current_prices = {}
+    for sym, md in market_data.items():
+        ticker = md.get("ticker", {})
+        if ticker:
+            current_prices[sym] = ticker.get("last_price", 0)
+        candles_htf = md.get("candles_htf")
+        if candles_htf is not None and not candles_htf.empty:
+            htf_by_symbol[sym] = indicators.calculate_htf_indicators(candles_htf)
+
+    slack_notifier.send_market_trend_report(htf_by_symbol, current_prices)
+
+
+if __name__ == "__main__":
+    main()
--- a/main.py
+++ b/main.py
@ -358,10 +358,9 @@ def run_cycle():
                logger.info("LLM signal: %s %s conf=%.2f reason=%s",
                            s.get("action"), s.get("symbol"), s.get("confidence", 0), s.get("reason", ""))
        else:
-            # Debug: log a sample of HOLD reasons to diagnose all-HOLD cycles
-            samples = signals[:3]
-            for s in samples:
-                logger.info("LLM HOLD sample: %s conf=%.2f reason=%s",
+            # Debug: log ALL hold reasons (not just first 3) to diagnose all-HOLD cycles
+            for s in signals:
+                logger.info("LLM HOLD: %s conf=%.2f reason=%s",
                            s.get("symbol"), s.get("confidence", 0), s.get("reason", ""))
    except Exception as e:
        logger.error("LLM analysis failed: %s", e)
--- a/memory/MEMORY.md
+++ b/memory/MEMORY.md
@ -1,8 +1,17 @@
 # Memory Index

- [user_profile.md](user_profile.md) — User role: crypto trader running Bitfinex bot, communicates in Traditional Chinese
- [project_trading_bot.md](project_trading_bot.md) — Key architecture: stop-loss sync, sell logic, order sizing, exposure, post-trade refresh, report format
- [project_cost_basis_sync.md](project_cost_basis_sync.md) — sync_cost_basis.py: order history cost calculation, wallet sync, Bitfinex API quirks
- [project_cron_timing.md](project_cron_timing.md) — Crontab timing: main.py :01/:06, sync :02/:32, offset from candle close
- [feedback_trading.md](feedback_trading.md) — User feedback: real-time stop-loss, no exposure limit, cost-basis order sizing
- [feedback_api_errors.md](feedback_api_errors.md) — Bitfinex 500 error patterns: stale stop IDs, min order size, balance locking, cancel not-found
+## User
+- [user_profile.md](user_profile.md) — 繁中溝通、Bitfinex 現貨交易者
+
+## Feedback
+- [feedback_trading.md](feedback_trading.md) — 止損用即時資料、無總曝險上限、成本基礎下單
+- [feedback_api_errors.md](feedback_api_errors.md) — Bitfinex 500 錯誤模式與修正
+- [feedback_no_misleading_signals.md](feedback_no_misleading_signals.md) — 報告不要暗示可進場，只報市場狀態
+
+## Project
+- [project_trading_bot.md](project_trading_bot.md) — 核心架構：止損、SELL、下單、曝險、報告格式
+- [project_architecture_split.md](project_architecture_split.md) — Production 用 LLM，Backtest 用規則引擎，兩者獨立
+- [project_cost_basis_sync.md](project_cost_basis_sync.md) — sync_cost_basis.py：訂單歷史成本計算
+- [project_cron_timing.md](project_cron_timing.md) — Cron 排程：交易 cycle、趨勢報告、成本同步、錯誤監控
+- [project_backtest_v3.md](project_backtest_v3.md) — V3 回測：加 context filters，return -19%→-13%
+- [project_whale_correlation.md](project_whale_correlation.md) — 免費鏈上數據與 BTC 無顯著相關性
--- a/memory/feedback_no_misleading_signals.md
+++ b/memory/feedback_no_misleading_signals.md
@ -0,0 +1,11 @@
+---
+name: No misleading entry signals in reports
+description: Trend report should not imply entry readiness — only show market state (bullish/bearish)
+type: feedback
+---
+
+趨勢報告不要顯示「可進場」之類的判斷字眼，只報告多頭/空頭。
+
+**Why:** 趨勢報告顯示「可進場」但 LLM 沒進場，造成混淆。Production 進場完全由 LLM 判斷，程式邏輯判斷與 LLM 不一致。
+
+**How to apply:** Slack 報告只呈現客觀市場數據，不做進場/出場建議。
--- a/memory/project_architecture_split.md
+++ b/memory/project_architecture_split.md
@ -0,0 +1,20 @@
+---
+name: Production vs Backtest architecture
+description: Production uses LLM for signals, backtest uses rule-based signal_generator — they are independent
+type: project
+---
+
+Production 和 Backtest 是兩條獨立路線：
+
+**Production (main.py):**
+- 進場/出場完全由 LLM (Claude CLI) 判斷
+- LLM prompt 包含策略規則，但 LLM 自行決定是否遵守
+- risk_manager 只做風控驗證（倉位大小、最大持倉數）
+
+**Backtest (backtest/):**
+- 用硬編碼規則的 signal_generator.py 判斷
+- 確定性、可重複，不跑 LLM
+- V3 加入 context 參數（BTC 趨勢、buy_pressure、funding sentiment）
+- `--no-context` flag 可關閉做 A/B 比較
+
+**How to apply:** 改 backtest 不影響 production。改 LLM prompt 才影響 production 行為。
--- a/memory/project_backtest_v3.md
+++ b/memory/project_backtest_v3.md
@ -0,0 +1,13 @@
+---
+name: Backtest V3 context filters
+description: V3 added BTC trend, buy pressure, funding sentiment — return improved from -19% to -13%
+type: project
+---
+
+V3 回測 (2025-07-01 ~ 2026-03-17, $10k):
+- Return: -19.07% → -13.48%
+- Max DD: -27.19% → -18.25%
+- BUYs: 385 → 189 (-51%)
+
+新增 context 參數：BTC 趨勢過濾、buy_pressure (OHLCV proxy)、funding sentiment (perp basis)。
+只影響 backtest，不影響 production。
--- a/memory/project_cron_timing.md
+++ b/memory/project_cron_timing.md
@ -6,9 +6,13 @@ type: project

 ## Crontab 排程

- `main.py`：`*/5 * * * * sleep 30 && ...`（:00:30, :05:30, :10:30...）
- `sync_cost_basis.py`：`2,32 * * * *`（:02, :32）
+| 排程 | 腳本 | 用途 |
+|------|------|------|
+| `*/5 * * * *` (sleep 30) | main.py | 交易 cycle（LLM 分析 + 執行） |
+| `0 * * * *` (sleep 30) | hourly_trend_report.py | 每小時 Slack 1h 趨勢報告（多頭/空頭） |
+| `2,32 * * * *` | sync_cost_basis.py | 成本基礎同步 |
+| `7 * * * *` | check_errors.py | 錯誤監控 |

-**Why:** Bitfinex 5 分鐘 K 線在整點收盤（:00, :05, :10...），延遲 30 秒確保數據到位。sync_cost_basis 在 :02/:32 避免衝突。
+**Why:** Bitfinex 5 分鐘 K 線在整點收盤（:00, :05, :10...），延遲 30 秒確保數據到位。sync_cost_basis 在 :02/:32 避免衝突。趨勢報告在 :00:30 發送。

-**How to apply:** crontab 不支援秒，用 `sleep 30 &&` 實現。修改排程時維持此偏移策略。
+**How to apply:** crontab 不支援秒，用 `sleep 30 &&` 實現。修改排程時維持此偏移策略，注意避免 Bitfinex API rate limit。
--- a/memory/project_whale_correlation.md
+++ b/memory/project_whale_correlation.md
@ -0,0 +1,12 @@
+---
+name: Whale data correlation analysis
+description: Free on-chain metrics show near-zero correlation with BTC price — not useful for signals
+type: project
+---
+
+2026-03-18 用 backtest/whale_correlation.py 分析 blockchain.com 免費鏈上指標。
+
+結果：所有指標跟 BTC 回報相關性 < 0.11（噪音）。
+真正有用的 whale 指標（exchange inflow/outflow）需要 CryptoQuant ($99/月) 或 Glassnode ($799/月)。
+
+**How to apply:** 不要再花時間在免費鏈上數據做交易信號。
--- a/slack_notifier.py
+++ b/slack_notifier.py
@ -151,6 +151,53 @@ def send_cycle_report(
    _send({"text": text})


+def send_market_trend_report(htf_by_symbol: dict, current_prices: dict,
+                             indicators_5m: dict | None = None):
+    """Send hourly market trend summary to Slack."""
+    import pandas as pd
+
+    lines = ["📈 *每小時 1h 趨勢報告*\n"]
+
+    bullish = []
+    bearish = []
+
+    for sym in sorted(htf_by_symbol):
+        df = htf_by_symbol[sym]
+        if df.empty or len(df) < 2:
+            continue
+        last = df.iloc[-1]
+        name = config.SYMBOL_NAMES.get(sym, sym)
+        price = current_prices.get(sym, 0)
+
+        ema9 = last.get("ema9", 0)
+        ema21 = last.get("ema21", 0)
+        adx_val = last.get("adx", 0)
+        rsi_1h = last.get("rsi", 50)
+
+        is_bullish = ema9 > ema21 if pd.notna(ema9) and pd.notna(ema21) else False
+        adx_val = adx_val if pd.notna(adx_val) else 0
+        rsi_1h = rsi_1h if pd.notna(rsi_1h) else 50
+
+        price_str = f"{price:.6g}" if price > 0 else "N/A"
+        info = f"{name}: ADX={adx_val:.0f} | RSI={rsi_1h:.0f} | ${price_str}"
+
+        if is_bullish:
+            bullish.append(f"🟢 {info}")
+        else:
+            bearish.append(f"🔴 {info}")
+
+    if bullish:
+        lines.append(f"*多頭 ({len(bullish)}):*")
+        lines.extend(f"  {s}" for s in bullish)
+        lines.append("")
+
+    if bearish:
+        lines.append(f"*空頭 ({len(bearish)}):*")
+        lines.extend(f"  {s}" for s in bearish)
+
+    _send({"text": "\n".join(lines)})
+
+
 def send_startup_message():
    """Notify that the bot has started."""
    mode = "PAPER" if config.PAPER_TRADING else "LIVE"